1b3e8e0991bc6a9fc732b4ab2217bd89e815d6c5
2 Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3 Microsoft Visual C++ functions. Originally developed for the ReactOS
4 (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
7 Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
9 Permission is hereby granted, free of charge, to any person obtaining a
10 copy of this software and associated documentation files (the "Software"),
11 to deal in the Software without restriction, including without limitation
12 the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 and/or sell copies of the Software, and to permit persons to whom the
14 Software is furnished to do so, subject to the following conditions:
16 The above copyright notice and this permission notice shall be included in
17 all copies or substantial portions of the Software.
19 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 DEALINGS IN THE SOFTWARE.
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
32 FIXME: review all "memory" clobbers, add/remove to match Visual C++
33 behavior: some "obvious" memory barriers are not present in the Visual C++
34 implementation - e.g. __stosX; on the other hand, some memory barriers that
35 *are* present could have been missed
39 NOTE: this is a *compatibility* header. Some functions may look wrong at
40 first, but they're only "as wrong" as they would be on Visual C++. Our
41 priority is compatibility
43 NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44 constraints and the uses of __volatile__ out of my... hat. Do not touch
45 them. I hate cargo cult programming
47 NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48 barriers aren't there in Visual C++ (e.g. __stosX)
50 NOTE: review all intrinsics with a return value, add/remove __volatile__
51 where necessary. If an intrinsic whose value is ignored generates a no-op
52 under Visual C++, __volatile__ must be omitted; if it always generates code
53 (for example, if it has side effects), __volatile__ must be specified. GCC
54 will only optimize out non-volatile asm blocks with outputs, so input-only
55 blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56 and follow Visual C++ behavior
58 NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59 atomic operations. Test the version like this:
61 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
64 Pay attention to the type of barrier. Make it match with what Visual C++
65 would use in the same case
72 /*** memcopy must be memmove ***/
73 void* __cdecl
memmove(void* dest
, const void* source
, size_t num
);
74 __INTRIN_INLINE
void* __cdecl
memcpy(void* dest
, const void* source
, size_t num
)
76 return memmove(dest
, source
, num
);
80 /*** Stack frame juggling ***/
81 #define _ReturnAddress() (__builtin_return_address(0))
82 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
83 /* TODO: __getcallerseflags but how??? */
85 /* Maybe the same for x86? */
87 #define _alloca(s) __builtin_alloca(s)
90 /*** Memory barriers ***/
92 __INTRIN_INLINE
void _ReadWriteBarrier(void)
94 __asm__
__volatile__("" : : : "memory");
97 /* GCC only supports full barriers */
98 #define _ReadBarrier _ReadWriteBarrier
99 #define _WriteBarrier _ReadWriteBarrier
101 __INTRIN_INLINE
void _mm_mfence(void)
103 __asm__
__volatile__("mfence" : : : "memory");
106 __INTRIN_INLINE
void _mm_lfence(void)
109 __asm__
__volatile__("lfence");
113 __INTRIN_INLINE
void _mm_sfence(void)
116 __asm__
__volatile__("sfence");
121 __INTRIN_INLINE
void __faststorefence(void)
124 __asm__
__volatile__("lock; orl $0, %0;" : : "m"(local
));
129 /*** Atomic operations ***/
131 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
133 __INTRIN_INLINE
char _InterlockedCompareExchange8(volatile char * Destination
, char Exchange
, char Comperand
)
135 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
138 __INTRIN_INLINE
short _InterlockedCompareExchange16(volatile short * Destination
, short Exchange
, short Comperand
)
140 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
145 __INTRIN_INLINE
long __cdecl
_InterlockedCompareExchange(volatile long * Destination
, long Exchange
, long Comperand
)
147 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
150 __INTRIN_INLINE
void * _InterlockedCompareExchangePointer(void * volatile * Destination
, void * Exchange
, void * Comperand
)
152 return (void *)__sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
157 __INTRIN_INLINE
char _InterlockedExchange8(volatile char * Target
, char Value
)
159 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
160 __sync_synchronize();
161 return __sync_lock_test_and_set(Target
, Value
);
164 __INTRIN_INLINE
short _InterlockedExchange16(volatile short * Target
, short Value
)
166 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
167 __sync_synchronize();
168 return __sync_lock_test_and_set(Target
, Value
);
173 __INTRIN_INLINE
long __cdecl
_InterlockedExchange(volatile long * Target
, long Value
)
175 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
176 __sync_synchronize();
177 return __sync_lock_test_and_set(Target
, Value
);
180 __INTRIN_INLINE
void * _InterlockedExchangePointer(void * volatile * Target
, void * Value
)
182 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
183 __sync_synchronize();
184 return (void *)__sync_lock_test_and_set(Target
, Value
);
189 #if defined(__x86_64__)
190 __INTRIN_INLINE
long long _InterlockedExchange64(volatile long long * Target
, long long Value
)
192 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
193 __sync_synchronize();
194 return __sync_lock_test_and_set(Target
, Value
);
198 __INTRIN_INLINE
char _InterlockedExchangeAdd8(char volatile * Addend
, char Value
)
200 return __sync_fetch_and_add(Addend
, Value
);
203 __INTRIN_INLINE
short _InterlockedExchangeAdd16(volatile short * Addend
, short Value
)
205 return __sync_fetch_and_add(Addend
, Value
);
209 __INTRIN_INLINE
long __cdecl
_InterlockedExchangeAdd(volatile long * Addend
, long Value
)
211 return __sync_fetch_and_add(Addend
, Value
);
215 #if defined(__x86_64__)
216 __INTRIN_INLINE
long long _InterlockedExchangeAdd64(volatile long long * Addend
, long long Value
)
218 return __sync_fetch_and_add(Addend
, Value
);
222 __INTRIN_INLINE
char _InterlockedAnd8(volatile char * value
, char mask
)
224 return __sync_fetch_and_and(value
, mask
);
227 __INTRIN_INLINE
short _InterlockedAnd16(volatile short * value
, short mask
)
229 return __sync_fetch_and_and(value
, mask
);
232 __INTRIN_INLINE
long _InterlockedAnd(volatile long * value
, long mask
)
234 return __sync_fetch_and_and(value
, mask
);
237 #if defined(__x86_64__)
238 __INTRIN_INLINE
long long _InterlockedAnd64(volatile long long * value
, long long mask
)
240 return __sync_fetch_and_and(value
, mask
);
244 __INTRIN_INLINE
char _InterlockedOr8(volatile char * value
, char mask
)
246 return __sync_fetch_and_or(value
, mask
);
249 __INTRIN_INLINE
short _InterlockedOr16(volatile short * value
, short mask
)
251 return __sync_fetch_and_or(value
, mask
);
254 __INTRIN_INLINE
long _InterlockedOr(volatile long * value
, long mask
)
256 return __sync_fetch_and_or(value
, mask
);
259 #if defined(__x86_64__)
260 __INTRIN_INLINE
long long _InterlockedOr64(volatile long long * value
, long long mask
)
262 return __sync_fetch_and_or(value
, mask
);
266 __INTRIN_INLINE
char _InterlockedXor8(volatile char * value
, char mask
)
268 return __sync_fetch_and_xor(value
, mask
);
271 __INTRIN_INLINE
short _InterlockedXor16(volatile short * value
, short mask
)
273 return __sync_fetch_and_xor(value
, mask
);
276 __INTRIN_INLINE
long _InterlockedXor(volatile long * value
, long mask
)
278 return __sync_fetch_and_xor(value
, mask
);
281 #if defined(__x86_64__)
282 __INTRIN_INLINE
long long _InterlockedXor64(volatile long long * value
, long long mask
)
284 return __sync_fetch_and_xor(value
, mask
);
289 __INTRIN_INLINE
long __cdecl
_InterlockedDecrement(volatile long * lpAddend
)
291 return __sync_sub_and_fetch(lpAddend
, 1);
294 __INTRIN_INLINE
long __cdecl
_InterlockedIncrement(volatile long * lpAddend
)
296 return __sync_add_and_fetch(lpAddend
, 1);
300 __INTRIN_INLINE
short _InterlockedDecrement16(volatile short * lpAddend
)
302 return __sync_sub_and_fetch(lpAddend
, 1);
305 __INTRIN_INLINE
short _InterlockedIncrement16(volatile short * lpAddend
)
307 return __sync_add_and_fetch(lpAddend
, 1);
310 #if defined(__x86_64__)
311 __INTRIN_INLINE
long long _InterlockedDecrement64(volatile long long * lpAddend
)
313 return __sync_sub_and_fetch(lpAddend
, 1);
316 __INTRIN_INLINE
long long _InterlockedIncrement64(volatile long long * lpAddend
)
318 return __sync_add_and_fetch(lpAddend
, 1);
322 #else /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
324 __INTRIN_INLINE
char _InterlockedCompareExchange8(volatile char * Destination
, char Exchange
, char Comperand
)
326 char retval
= Comperand
;
327 __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
) : "memory");
331 __INTRIN_INLINE
short _InterlockedCompareExchange16(volatile short * Destination
, short Exchange
, short Comperand
)
333 short retval
= Comperand
;
334 __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
): "memory");
338 __INTRIN_INLINE
long _InterlockedCompareExchange(volatile long * Destination
, long Exchange
, long Comperand
)
340 long retval
= Comperand
;
341 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
): "memory");
345 __INTRIN_INLINE
void * _InterlockedCompareExchangePointer(void * volatile * Destination
, void * Exchange
, void * Comperand
)
347 void * retval
= (void *)Comperand
;
348 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval
] "=a" (retval
) : "[retval]" (retval
), [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
) : "memory");
352 __INTRIN_INLINE
char _InterlockedExchange8(volatile char * Target
, char Value
)
355 __asm__("xchgb %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
359 __INTRIN_INLINE
short _InterlockedExchange16(volatile short * Target
, short Value
)
361 short retval
= Value
;
362 __asm__("xchgw %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
366 __INTRIN_INLINE
long _InterlockedExchange(volatile long * Target
, long Value
)
369 __asm__("xchgl %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
373 __INTRIN_INLINE
void * _InterlockedExchangePointer(void * volatile * Target
, void * Value
)
375 void * retval
= Value
;
376 __asm__("xchgl %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
380 __INTRIN_INLINE
char _InterlockedExchangeAdd8(char volatile * Addend
, char Value
)
383 __asm__("lock; xaddb %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
387 __INTRIN_INLINE
short _InterlockedExchangeAdd16(volatile short * Addend
, short Value
)
389 short retval
= Value
;
390 __asm__("lock; xaddw %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
394 __INTRIN_INLINE
long _InterlockedExchangeAdd(volatile long * Addend
, long Value
)
397 __asm__("lock; xaddl %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
401 __INTRIN_INLINE
char _InterlockedAnd8(volatile char * value
, char mask
)
411 y
= _InterlockedCompareExchange8(value
, x
& mask
, x
);
418 __INTRIN_INLINE
short _InterlockedAnd16(volatile short * value
, short mask
)
428 y
= _InterlockedCompareExchange16(value
, x
& mask
, x
);
435 __INTRIN_INLINE
long _InterlockedAnd(volatile long * value
, long mask
)
445 y
= _InterlockedCompareExchange(value
, x
& mask
, x
);
452 __INTRIN_INLINE
char _InterlockedOr8(volatile char * value
, char mask
)
462 y
= _InterlockedCompareExchange8(value
, x
| mask
, x
);
469 __INTRIN_INLINE
short _InterlockedOr16(volatile short * value
, short mask
)
479 y
= _InterlockedCompareExchange16(value
, x
| mask
, x
);
486 __INTRIN_INLINE
long _InterlockedOr(volatile long * value
, long mask
)
496 y
= _InterlockedCompareExchange(value
, x
| mask
, x
);
503 __INTRIN_INLINE
char _InterlockedXor8(volatile char * value
, char mask
)
513 y
= _InterlockedCompareExchange8(value
, x
^ mask
, x
);
520 __INTRIN_INLINE
short _InterlockedXor16(volatile short * value
, short mask
)
530 y
= _InterlockedCompareExchange16(value
, x
^ mask
, x
);
537 __INTRIN_INLINE
long _InterlockedXor(volatile long * value
, long mask
)
547 y
= _InterlockedCompareExchange(value
, x
^ mask
, x
);
554 __INTRIN_INLINE
long _InterlockedDecrement(volatile long * lpAddend
)
556 return _InterlockedExchangeAdd(lpAddend
, -1) - 1;
559 __INTRIN_INLINE
long _InterlockedIncrement(volatile long * lpAddend
)
561 return _InterlockedExchangeAdd(lpAddend
, 1) + 1;
564 __INTRIN_INLINE
short _InterlockedDecrement16(volatile short * lpAddend
)
566 return _InterlockedExchangeAdd16(lpAddend
, -1) - 1;
569 __INTRIN_INLINE
short _InterlockedIncrement16(volatile short * lpAddend
)
571 return _InterlockedExchangeAdd16(lpAddend
, 1) + 1;
574 #if defined(__x86_64__)
575 __INTRIN_INLINE
long long _InterlockedDecrement64(volatile long long * lpAddend
)
577 return _InterlockedExchangeAdd64(lpAddend
, -1) - 1;
580 __INTRIN_INLINE
long long _InterlockedIncrement64(volatile long long * lpAddend
)
582 return _InterlockedExchangeAdd64(lpAddend
, 1) + 1;
586 #endif /* (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 */
588 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
590 __INTRIN_INLINE
long long _InterlockedCompareExchange64(volatile long long * Destination
, long long Exchange
, long long Comperand
)
592 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
597 __INTRIN_INLINE
long long _InterlockedCompareExchange64(volatile long long * Destination
, long long Exchange
, long long Comperand
)
599 long long retval
= Comperand
;
603 "lock; cmpxchg8b %[Destination]" :
604 [retval
] "+A" (retval
) :
605 [Destination
] "m" (*Destination
),
606 "b" ((unsigned long)((Exchange
>> 0) & 0xFFFFFFFF)),
607 "c" ((unsigned long)((Exchange
>> 32) & 0xFFFFFFFF)) :
617 __INTRIN_INLINE
long _InterlockedAddLargeStatistic(volatile long long * Addend
, long Value
)
621 "lock; addl %[Value], %[Lo32];"
623 "lock; adcl $0, %[Hi32];"
625 [Lo32
] "+m" (*((volatile long *)(Addend
) + 0)), [Hi32
] "+m" (*((volatile long *)(Addend
) + 1)) :
626 [Value
] "ir" (Value
) :
632 #endif /* __i386__ */
634 __INTRIN_INLINE
unsigned char _interlockedbittestandreset(volatile long * a
, long b
)
636 unsigned char retval
;
637 __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
641 #if defined(__x86_64__)
642 __INTRIN_INLINE
unsigned char _interlockedbittestandreset64(volatile long long * a
, long long b
)
644 unsigned char retval
;
645 __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval
] "=r" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
650 __INTRIN_INLINE
unsigned char _interlockedbittestandset(volatile long * a
, long b
)
652 unsigned char retval
;
653 __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval
] "=q" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
657 #if defined(__x86_64__)
658 __INTRIN_INLINE
unsigned char _interlockedbittestandset64(volatile long long * a
, long long b
)
660 unsigned char retval
;
661 __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval
] "=r" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
666 /*** String operations ***/
668 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
669 __INTRIN_INLINE
void __stosb(unsigned char * Dest
, unsigned char Data
, size_t Count
)
674 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
675 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
679 __INTRIN_INLINE
void __stosw(unsigned short * Dest
, unsigned short Data
, size_t Count
)
684 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
685 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
689 __INTRIN_INLINE
void __stosd(unsigned long * Dest
, unsigned long Data
, size_t Count
)
694 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
695 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
700 __INTRIN_INLINE
void __stosq(unsigned long long * Dest
, unsigned long long Data
, size_t Count
)
705 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
706 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
711 __INTRIN_INLINE
void __movsb(unsigned char * Destination
, const unsigned char * Source
, size_t Count
)
716 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
717 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
721 __INTRIN_INLINE
void __movsw(unsigned short * Destination
, const unsigned short * Source
, size_t Count
)
726 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
727 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
731 __INTRIN_INLINE
void __movsd(unsigned long * Destination
, const unsigned long * Source
, size_t Count
)
736 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
737 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
742 __INTRIN_INLINE
void __movsq(unsigned long * Destination
, const unsigned long * Source
, size_t Count
)
747 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
748 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
753 #if defined(__x86_64__)
755 /*** GS segment addressing ***/
757 __INTRIN_INLINE
void __writegsbyte(unsigned long Offset
, unsigned char Data
)
759 __asm__
__volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
762 __INTRIN_INLINE
void __writegsword(unsigned long Offset
, unsigned short Data
)
764 __asm__
__volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
767 __INTRIN_INLINE
void __writegsdword(unsigned long Offset
, unsigned long Data
)
769 __asm__
__volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
772 __INTRIN_INLINE
void __writegsqword(unsigned long Offset
, unsigned long long Data
)
774 __asm__
__volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
777 __INTRIN_INLINE
unsigned char __readgsbyte(unsigned long Offset
)
780 __asm__
__volatile__("movb %%gs:%a[Offset], %b[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
784 __INTRIN_INLINE
unsigned short __readgsword(unsigned long Offset
)
786 unsigned short value
;
787 __asm__
__volatile__("movw %%gs:%a[Offset], %w[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
791 __INTRIN_INLINE
unsigned long __readgsdword(unsigned long Offset
)
794 __asm__
__volatile__("movl %%gs:%a[Offset], %k[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
798 __INTRIN_INLINE
unsigned long long __readgsqword(unsigned long Offset
)
800 unsigned long long value
;
801 __asm__
__volatile__("movq %%gs:%a[Offset], %q[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
805 __INTRIN_INLINE
void __incgsbyte(unsigned long Offset
)
807 __asm__
__volatile__("incb %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
810 __INTRIN_INLINE
void __incgsword(unsigned long Offset
)
812 __asm__
__volatile__("incw %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
815 __INTRIN_INLINE
void __incgsdword(unsigned long Offset
)
817 __asm__
__volatile__("incl %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
820 __INTRIN_INLINE
void __incgsqword(unsigned long Offset
)
822 __asm__
__volatile__("incq %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
825 __INTRIN_INLINE
void __addgsbyte(unsigned long Offset
, unsigned char Data
)
827 __asm__
__volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
830 __INTRIN_INLINE
void __addgsword(unsigned long Offset
, unsigned short Data
)
832 __asm__
__volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
835 __INTRIN_INLINE
void __addgsdword(unsigned long Offset
, unsigned int Data
)
837 __asm__
__volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
840 __INTRIN_INLINE
void __addgsqword(unsigned long Offset
, unsigned long long Data
)
842 __asm__
__volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
845 #else /* defined(__x86_64__) */
847 /*** FS segment addressing ***/
849 __INTRIN_INLINE
void __writefsbyte(unsigned long Offset
, unsigned char Data
)
851 __asm__
__volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
854 __INTRIN_INLINE
void __writefsword(unsigned long Offset
, unsigned short Data
)
856 __asm__
__volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
859 __INTRIN_INLINE
void __writefsdword(unsigned long Offset
, unsigned long Data
)
861 __asm__
__volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "ir" (Data
) : "memory");
864 __INTRIN_INLINE
unsigned char __readfsbyte(unsigned long Offset
)
867 __asm__
__volatile__("movb %%fs:%a[Offset], %b[value]" : [value
] "=q" (value
) : [Offset
] "ir" (Offset
));
871 __INTRIN_INLINE
unsigned short __readfsword(unsigned long Offset
)
873 unsigned short value
;
874 __asm__
__volatile__("movw %%fs:%a[Offset], %w[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
878 __INTRIN_INLINE
unsigned long __readfsdword(unsigned long Offset
)
881 __asm__
__volatile__("movl %%fs:%a[Offset], %k[value]" : [value
] "=r" (value
) : [Offset
] "ir" (Offset
));
885 __INTRIN_INLINE
void __incfsbyte(unsigned long Offset
)
887 __asm__
__volatile__("incb %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
890 __INTRIN_INLINE
void __incfsword(unsigned long Offset
)
892 __asm__
__volatile__("incw %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
895 __INTRIN_INLINE
void __incfsdword(unsigned long Offset
)
897 __asm__
__volatile__("incl %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
900 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
901 __INTRIN_INLINE
void __addfsbyte(unsigned long Offset
, unsigned char Data
)
903 if(!__builtin_constant_p(Offset
))
904 __asm__
__volatile__("addb %b[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
) : "memory");
906 __asm__
__volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
909 __INTRIN_INLINE
void __addfsword(unsigned long Offset
, unsigned short Data
)
911 if(!__builtin_constant_p(Offset
))
912 __asm__
__volatile__("addw %w[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
) : "memory");
914 __asm__
__volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
917 __INTRIN_INLINE
void __addfsdword(unsigned long Offset
, unsigned long Data
)
919 if(!__builtin_constant_p(Offset
))
920 __asm__
__volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
) : "memory");
922 __asm__
__volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
925 #endif /* defined(__x86_64__) */
928 /*** Bit manipulation ***/
930 __INTRIN_INLINE
unsigned char _BitScanForward(unsigned long * Index
, unsigned long Mask
)
932 __asm__("bsfl %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
936 __INTRIN_INLINE
unsigned char _BitScanReverse(unsigned long * Index
, unsigned long Mask
)
938 __asm__("bsrl %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
942 /* NOTE: again, the bizarre implementation follows Visual C++ */
943 __INTRIN_INLINE
unsigned char _bittest(const long * a
, long b
)
945 unsigned char retval
;
947 if(__builtin_constant_p(b
))
948 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*(a
+ (b
/ 32))), [b
] "Ir" (b
% 32));
950 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "m" (*a
), [b
] "r" (b
));
956 __INTRIN_INLINE
unsigned char _BitScanForward64(unsigned long * Index
, unsigned long long Mask
)
958 unsigned long long Index64
;
959 __asm__("bsfq %[Mask], %[Index]" : [Index
] "=r" (Index64
) : [Mask
] "mr" (Mask
));
964 __INTRIN_INLINE
unsigned char _BitScanReverse64(unsigned long * Index
, unsigned long long Mask
)
966 unsigned long long Index64
;
967 __asm__("bsrq %[Mask], %[Index]" : [Index
] "=r" (Index64
) : [Mask
] "mr" (Mask
));
972 __INTRIN_INLINE
unsigned char _bittest64(const long long * a
, long long b
)
974 unsigned char retval
;
976 if(__builtin_constant_p(b
))
977 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*(a
+ (b
/ 64))), [b
] "Ir" (b
% 64));
979 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "m" (*a
), [b
] "r" (b
));
985 __INTRIN_INLINE
unsigned char _bittestandcomplement(long * a
, long b
)
987 unsigned char retval
;
989 if(__builtin_constant_p(b
))
990 __asm__("btc %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 32))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 32));
992 __asm__("btc %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
997 __INTRIN_INLINE
unsigned char _bittestandreset(long * a
, long b
)
999 unsigned char retval
;
1001 if(__builtin_constant_p(b
))
1002 __asm__("btr %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 32))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 32));
1004 __asm__("btr %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1009 __INTRIN_INLINE
unsigned char _bittestandset(long * a
, long b
)
1011 unsigned char retval
;
1013 if(__builtin_constant_p(b
))
1014 __asm__("bts %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 32))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 32));
1016 __asm__("bts %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1023 __INTRIN_INLINE
unsigned char _bittestandset64(long long * a
, long long b
)
1025 unsigned char retval
;
1027 if(__builtin_constant_p(b
))
1028 __asm__("btsq %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 64))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 64));
1030 __asm__("btsq %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1035 __INTRIN_INLINE
unsigned char _bittestandreset64(long long * a
, long long b
)
1037 unsigned char retval
;
1039 if(__builtin_constant_p(b
))
1040 __asm__("btrq %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 64))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 64));
1042 __asm__("btrq %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1047 __INTRIN_INLINE
unsigned char _bittestandcomplement64(long long * a
, long long b
)
1049 unsigned char retval
;
1051 if(__builtin_constant_p(b
))
1052 __asm__("btcq %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 64))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 64));
1054 __asm__("btcq %[b], %[a]; setb %b[retval]" : [a
] "+m" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
1061 __INTRIN_INLINE
unsigned char __cdecl
_rotl8(unsigned char value
, unsigned char shift
)
1063 unsigned char retval
;
1064 __asm__("rolb %b[shift], %b[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1068 __INTRIN_INLINE
unsigned short __cdecl
_rotl16(unsigned short value
, unsigned char shift
)
1070 unsigned short retval
;
1071 __asm__("rolw %b[shift], %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1075 __INTRIN_INLINE
unsigned int __cdecl
_rotl(unsigned int value
, int shift
)
1077 unsigned int retval
;
1078 __asm__("roll %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1083 __INTRIN_INLINE
unsigned long long _rotl64(unsigned long long value
, int shift
)
1085 unsigned long long retval
;
1086 __asm__("rolq %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1090 __INTRIN_INLINE
unsigned long long __cdecl
_rotl64(unsigned long long value
, int shift
)
1092 /* FIXME: this is probably not optimal */
1093 return (value
<< shift
) | (value
>> (64 - shift
));
1097 __INTRIN_INLINE
unsigned int __cdecl
_rotr(unsigned int value
, int shift
)
1099 unsigned int retval
;
1100 __asm__("rorl %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1104 __INTRIN_INLINE
unsigned char __cdecl
_rotr8(unsigned char value
, unsigned char shift
)
1106 unsigned char retval
;
1107 __asm__("rorb %b[shift], %b[retval]" : [retval
] "=qm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1111 __INTRIN_INLINE
unsigned short __cdecl
_rotr16(unsigned short value
, unsigned char shift
)
1113 unsigned short retval
;
1114 __asm__("rorw %b[shift], %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1119 __INTRIN_INLINE
unsigned long long _rotr64(unsigned long long value
, int shift
)
1121 unsigned long long retval
;
1122 __asm__("rorq %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1126 __INTRIN_INLINE
unsigned long long __cdecl
_rotr64(unsigned long long value
, int shift
)
1128 /* FIXME: this is probably not optimal */
1129 return (value
>> shift
) | (value
<< (64 - shift
));
1133 __INTRIN_INLINE
unsigned long __cdecl
_lrotl(unsigned long value
, int shift
)
1135 unsigned long retval
;
1136 __asm__("roll %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1140 __INTRIN_INLINE
unsigned long __cdecl
_lrotr(unsigned long value
, int shift
)
1142 unsigned long retval
;
1143 __asm__("rorl %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
1148 NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
1149 constraint (edx:eax) for the Mask argument, because it's the only way GCC
1150 can pass 64-bit operands around - passing the two 32 bit parts separately
1151 just confuses it. Also we declare Bit as an int and then truncate it to
1152 match Visual C++ behavior
1154 __INTRIN_INLINE
unsigned long long __ll_lshift(unsigned long long Mask
, int Bit
)
1156 unsigned long long retval
= Mask
;
1160 "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
1162 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
1168 __INTRIN_INLINE
long long __ll_rshift(long long Mask
, int Bit
)
1170 long long retval
= Mask
;
1174 "shrdl %b[Bit], %%edx, %%eax; sarl %b[Bit], %%edx" :
1176 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
1182 __INTRIN_INLINE
unsigned long long __ull_rshift(unsigned long long Mask
, int Bit
)
1184 unsigned long long retval
= Mask
;
1188 "shrdl %b[Bit], %%edx, %%eax; shrl %b[Bit], %%edx" :
1190 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
1196 __INTRIN_INLINE
unsigned short __cdecl
_byteswap_ushort(unsigned short value
)
1198 unsigned short retval
;
1199 __asm__("rorw $8, %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
));
1203 __INTRIN_INLINE
unsigned long __cdecl
_byteswap_ulong(unsigned long value
)
1205 unsigned long retval
;
1206 __asm__("bswapl %[retval]" : [retval
] "=r" (retval
) : "[retval]" (value
));
1211 __INTRIN_INLINE
unsigned long long _byteswap_uint64(unsigned long long value
)
1213 unsigned long long retval
;
1214 __asm__("bswapq %[retval]" : [retval
] "=r" (retval
) : "[retval]" (value
));
1218 __INTRIN_INLINE
unsigned long long __cdecl
_byteswap_uint64(unsigned long long value
)
1221 unsigned long long int64part
;
1223 unsigned long lowpart
;
1224 unsigned long hipart
;
1227 retval
.int64part
= value
;
1228 __asm__("bswapl %[lowpart]\n"
1229 "bswapl %[hipart]\n"
1230 : [lowpart
] "=r" (retval
.hipart
), [hipart
] "=r" (retval
.lowpart
) : "[lowpart]" (retval
.lowpart
), "[hipart]" (retval
.hipart
) );
1231 return retval
.int64part
;
1235 __INTRIN_INLINE
unsigned int __lzcnt(unsigned int value
)
1237 return __builtin_clz(value
);
1240 __INTRIN_INLINE
unsigned short __lzcnt16(unsigned short value
)
1242 return __builtin_clz(value
);
1245 __INTRIN_INLINE
unsigned int __popcnt(unsigned int value
)
1247 return __builtin_popcount(value
);
1250 __INTRIN_INLINE
unsigned short __popcnt16(unsigned short value
)
1252 return __builtin_popcount(value
);
1256 __INTRIN_INLINE
unsigned long long __lzcnt64(unsigned long long value
)
1258 return __builtin_clzll(value
);
1261 __INTRIN_INLINE
unsigned long long __popcnt64(unsigned long long value
)
1263 return __builtin_popcountll(value
);
1267 /*** 64-bit math ***/
1269 __INTRIN_INLINE
long long __emul(int a
, int b
)
1272 __asm__("imull %[b]" : "=A" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
1276 __INTRIN_INLINE
unsigned long long __emulu(unsigned int a
, unsigned int b
)
1278 unsigned long long retval
;
1279 __asm__("mull %[b]" : "=A" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
1283 __INTRIN_INLINE
long long __cdecl
_abs64(long long value
)
1285 return (value
>= 0) ? value
: -value
;
1290 __INTRIN_INLINE
long long __mulh(long long a
, long long b
)
1293 __asm__("imulq %[b]" : "=d" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
1297 __INTRIN_INLINE
unsigned long long __umulh(unsigned long long a
, unsigned long long b
)
1299 unsigned long long retval
;
1300 __asm__("mulq %[b]" : "=d" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
1308 __INTRIN_INLINE
unsigned char __inbyte(unsigned short Port
)
1311 __asm__
__volatile__("inb %w[Port], %b[byte]" : [byte
] "=a" (byte
) : [Port
] "Nd" (Port
));
1315 __INTRIN_INLINE
unsigned short __inword(unsigned short Port
)
1317 unsigned short word
;
1318 __asm__
__volatile__("inw %w[Port], %w[word]" : [word
] "=a" (word
) : [Port
] "Nd" (Port
));
1322 __INTRIN_INLINE
unsigned long __indword(unsigned short Port
)
1324 unsigned long dword
;
1325 __asm__
__volatile__("inl %w[Port], %k[dword]" : [dword
] "=a" (dword
) : [Port
] "Nd" (Port
));
1329 __INTRIN_INLINE
void __inbytestring(unsigned short Port
, unsigned char * Buffer
, unsigned long Count
)
1331 __asm__ __volatile__
1334 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
1335 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
1340 __INTRIN_INLINE
void __inwordstring(unsigned short Port
, unsigned short * Buffer
, unsigned long Count
)
1342 __asm__ __volatile__
1345 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
1346 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
1351 __INTRIN_INLINE
void __indwordstring(unsigned short Port
, unsigned long * Buffer
, unsigned long Count
)
1353 __asm__ __volatile__
1356 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
1357 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
1362 __INTRIN_INLINE
void __outbyte(unsigned short Port
, unsigned char Data
)
1364 __asm__
__volatile__("outb %b[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
1367 __INTRIN_INLINE
void __outword(unsigned short Port
, unsigned short Data
)
1369 __asm__
__volatile__("outw %w[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
1372 __INTRIN_INLINE
void __outdword(unsigned short Port
, unsigned long Data
)
1374 __asm__
__volatile__("outl %k[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
1377 __INTRIN_INLINE
void __outbytestring(unsigned short Port
, unsigned char * Buffer
, unsigned long Count
)
1379 __asm__
__volatile__("rep; outsb" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
1382 __INTRIN_INLINE
void __outwordstring(unsigned short Port
, unsigned short * Buffer
, unsigned long Count
)
1384 __asm__
__volatile__("rep; outsw" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
1387 __INTRIN_INLINE
void __outdwordstring(unsigned short Port
, unsigned long * Buffer
, unsigned long Count
)
1389 __asm__
__volatile__("rep; outsl" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
1392 __INTRIN_INLINE
int __cdecl
_inp(unsigned short Port
)
1394 return __inbyte(Port
);
1397 __INTRIN_INLINE
unsigned short __cdecl
_inpw(unsigned short Port
)
1399 return __inword(Port
);
1402 __INTRIN_INLINE
unsigned long __cdecl
_inpd(unsigned short Port
)
1404 return __indword(Port
);
1407 __INTRIN_INLINE
int __cdecl
_outp(unsigned short Port
, int databyte
)
1409 __outbyte(Port
, (unsigned char)databyte
);
1413 __INTRIN_INLINE
unsigned short __cdecl
_outpw(unsigned short Port
, unsigned short dataword
)
1415 __outword(Port
, dataword
);
1419 __INTRIN_INLINE
unsigned long __cdecl
_outpd(unsigned short Port
, unsigned long dataword
)
1421 __outdword(Port
, dataword
);
1426 /*** System information ***/
1428 __INTRIN_INLINE
void __cpuid(int CPUInfo
[4], int InfoType
)
1430 __asm__
__volatile__("cpuid" : "=a" (CPUInfo
[0]), "=b" (CPUInfo
[1]), "=c" (CPUInfo
[2]), "=d" (CPUInfo
[3]) : "a" (InfoType
));
1433 __INTRIN_INLINE
void __cpuidex(int CPUInfo
[4], int InfoType
, int ECXValue
)
1435 __asm__
__volatile__("cpuid" : "=a" (CPUInfo
[0]), "=b" (CPUInfo
[1]), "=c" (CPUInfo
[2]), "=d" (CPUInfo
[3]) : "a" (InfoType
), "c" (ECXValue
));
1438 __INTRIN_INLINE
unsigned long long __rdtsc(void)
1441 unsigned long long low
, high
;
1442 __asm__
__volatile__("rdtsc" : "=a"(low
), "=d"(high
));
1443 return low
| (high
<< 32);
1445 unsigned long long retval
;
1446 __asm__
__volatile__("rdtsc" : "=A"(retval
));
1451 __INTRIN_INLINE
void __writeeflags(uintptr_t Value
)
1453 __asm__
__volatile__("push %0\n popf" : : "rim"(Value
));
1456 __INTRIN_INLINE
uintptr_t __readeflags(void)
1459 __asm__
__volatile__("pushf\n pop %0" : "=rm"(retval
));
1463 /*** Interrupts ***/
1466 #define __debugbreak() __asm__("int $3")
1468 __INTRIN_INLINE
void __cdecl
__debugbreak(void)
1474 __INTRIN_INLINE
void __ud2(void)
1479 __INTRIN_INLINE
void __int2c(void)
1481 __asm__("int $0x2c");
1484 __INTRIN_INLINE
void __cdecl
_disable(void)
1486 __asm__("cli" : : : "memory");
1489 __INTRIN_INLINE
void __cdecl
_enable(void)
1491 __asm__("sti" : : : "memory");
1494 __INTRIN_INLINE
void __halt(void)
1496 __asm__("hlt" : : : "memory");
1499 __declspec(noreturn
)
1500 __INTRIN_INLINE
void __fastfail(unsigned int Code
)
1502 __asm__("int $0x29" : : "c"(Code
) : "memory");
1503 __builtin_unreachable();
1506 /*** Protected memory management ***/
1510 __INTRIN_INLINE
void __writecr0(unsigned long long Data
)
1512 __asm__("mov %[Data], %%cr0" : : [Data
] "r" (Data
) : "memory");
1515 __INTRIN_INLINE
void __writecr3(unsigned long long Data
)
1517 __asm__("mov %[Data], %%cr3" : : [Data
] "r" (Data
) : "memory");
1520 __INTRIN_INLINE
void __writecr4(unsigned long long Data
)
1522 __asm__("mov %[Data], %%cr4" : : [Data
] "r" (Data
) : "memory");
1525 __INTRIN_INLINE
void __writecr8(unsigned long long Data
)
1527 __asm__("mov %[Data], %%cr8" : : [Data
] "r" (Data
) : "memory");
1530 __INTRIN_INLINE
unsigned long long __readcr0(void)
1532 unsigned long long value
;
1533 __asm__
__volatile__("mov %%cr0, %[value]" : [value
] "=r" (value
));
1537 __INTRIN_INLINE
unsigned long long __readcr2(void)
1539 unsigned long long value
;
1540 __asm__
__volatile__("mov %%cr2, %[value]" : [value
] "=r" (value
));
1544 __INTRIN_INLINE
unsigned long long __readcr3(void)
1546 unsigned long long value
;
1547 __asm__
__volatile__("mov %%cr3, %[value]" : [value
] "=r" (value
));
1551 __INTRIN_INLINE
unsigned long long __readcr4(void)
1553 unsigned long long value
;
1554 __asm__
__volatile__("mov %%cr4, %[value]" : [value
] "=r" (value
));
1558 __INTRIN_INLINE
unsigned long long __readcr8(void)
1560 unsigned long long value
;
1561 __asm__
__volatile__("movq %%cr8, %q[value]" : [value
] "=r" (value
));
1565 #else /* __x86_64__ */
1567 __INTRIN_INLINE
void __writecr0(unsigned int Data
)
1569 __asm__("mov %[Data], %%cr0" : : [Data
] "r" (Data
) : "memory");
1572 __INTRIN_INLINE
void __writecr3(unsigned int Data
)
1574 __asm__("mov %[Data], %%cr3" : : [Data
] "r" (Data
) : "memory");
1577 __INTRIN_INLINE
void __writecr4(unsigned int Data
)
1579 __asm__("mov %[Data], %%cr4" : : [Data
] "r" (Data
) : "memory");
1582 __INTRIN_INLINE
void __writecr8(unsigned int Data
)
1584 __asm__("mov %[Data], %%cr8" : : [Data
] "r" (Data
) : "memory");
1587 __INTRIN_INLINE
unsigned long __readcr0(void)
1589 unsigned long value
;
1590 __asm__
__volatile__("mov %%cr0, %[value]" : [value
] "=r" (value
));
1594 __INTRIN_INLINE
unsigned long __readcr2(void)
1596 unsigned long value
;
1597 __asm__
__volatile__("mov %%cr2, %[value]" : [value
] "=r" (value
));
1601 __INTRIN_INLINE
unsigned long __readcr3(void)
1603 unsigned long value
;
1604 __asm__
__volatile__("mov %%cr3, %[value]" : [value
] "=r" (value
));
1608 __INTRIN_INLINE
unsigned long __readcr4(void)
1610 unsigned long value
;
1611 __asm__
__volatile__("mov %%cr4, %[value]" : [value
] "=r" (value
));
1615 __INTRIN_INLINE
unsigned long __readcr8(void)
1617 unsigned long value
;
1618 __asm__
__volatile__("mov %%cr8, %[value]" : [value
] "=r" (value
));
1622 #endif /* __x86_64__ */
1626 __INTRIN_INLINE
unsigned long long __readdr(unsigned int reg
)
1628 unsigned long long value
;
1632 __asm__
__volatile__("movq %%dr0, %q[value]" : [value
] "=r" (value
));
1635 __asm__
__volatile__("movq %%dr1, %q[value]" : [value
] "=r" (value
));
1638 __asm__
__volatile__("movq %%dr2, %q[value]" : [value
] "=r" (value
));
1641 __asm__
__volatile__("movq %%dr3, %q[value]" : [value
] "=r" (value
));
1644 __asm__
__volatile__("movq %%dr4, %q[value]" : [value
] "=r" (value
));
1647 __asm__
__volatile__("movq %%dr5, %q[value]" : [value
] "=r" (value
));
1650 __asm__
__volatile__("movq %%dr6, %q[value]" : [value
] "=r" (value
));
1653 __asm__
__volatile__("movq %%dr7, %q[value]" : [value
] "=r" (value
));
1659 __INTRIN_INLINE
void __writedr(unsigned reg
, unsigned long long value
)
1664 __asm__("movq %q[value], %%dr0" : : [value
] "r" (value
) : "memory");
1667 __asm__("movq %q[value], %%dr1" : : [value
] "r" (value
) : "memory");
1670 __asm__("movq %q[value], %%dr2" : : [value
] "r" (value
) : "memory");
1673 __asm__("movq %q[value], %%dr3" : : [value
] "r" (value
) : "memory");
1676 __asm__("movq %q[value], %%dr4" : : [value
] "r" (value
) : "memory");
1679 __asm__("movq %q[value], %%dr5" : : [value
] "r" (value
) : "memory");
1682 __asm__("movq %q[value], %%dr6" : : [value
] "r" (value
) : "memory");
1685 __asm__("movq %q[value], %%dr7" : : [value
] "r" (value
) : "memory");
1690 #else /* __x86_64__ */
1692 __INTRIN_INLINE
unsigned int __readdr(unsigned int reg
)
1698 __asm__
__volatile__("mov %%dr0, %[value]" : [value
] "=r" (value
));
1701 __asm__
__volatile__("mov %%dr1, %[value]" : [value
] "=r" (value
));
1704 __asm__
__volatile__("mov %%dr2, %[value]" : [value
] "=r" (value
));
1707 __asm__
__volatile__("mov %%dr3, %[value]" : [value
] "=r" (value
));
1710 __asm__
__volatile__("mov %%dr4, %[value]" : [value
] "=r" (value
));
1713 __asm__
__volatile__("mov %%dr5, %[value]" : [value
] "=r" (value
));
1716 __asm__
__volatile__("mov %%dr6, %[value]" : [value
] "=r" (value
));
1719 __asm__
__volatile__("mov %%dr7, %[value]" : [value
] "=r" (value
));
1725 __INTRIN_INLINE
void __writedr(unsigned reg
, unsigned int value
)
1730 __asm__("mov %[value], %%dr0" : : [value
] "r" (value
) : "memory");
1733 __asm__("mov %[value], %%dr1" : : [value
] "r" (value
) : "memory");
1736 __asm__("mov %[value], %%dr2" : : [value
] "r" (value
) : "memory");
1739 __asm__("mov %[value], %%dr3" : : [value
] "r" (value
) : "memory");
1742 __asm__("mov %[value], %%dr4" : : [value
] "r" (value
) : "memory");
1745 __asm__("mov %[value], %%dr5" : : [value
] "r" (value
) : "memory");
1748 __asm__("mov %[value], %%dr6" : : [value
] "r" (value
) : "memory");
1751 __asm__("mov %[value], %%dr7" : : [value
] "r" (value
) : "memory");
1756 #endif /* __x86_64__ */
1758 __INTRIN_INLINE
void __invlpg(void *Address
)
1760 __asm__
__volatile__ ("invlpg (%[Address])" : : [Address
] "b" (Address
) : "memory");
1764 /*** System operations ***/
1766 __INTRIN_INLINE
unsigned long long __readmsr(unsigned long reg
)
1769 unsigned long low
, high
;
1770 __asm__
__volatile__("rdmsr" : "=a" (low
), "=d" (high
) : "c" (reg
));
1771 return ((unsigned long long)high
<< 32) | low
;
1773 unsigned long long retval
;
1774 __asm__
__volatile__("rdmsr" : "=A" (retval
) : "c" (reg
));
1779 __INTRIN_INLINE
void __writemsr(unsigned long Register
, unsigned long long Value
)
1782 __asm__
__volatile__("wrmsr" : : "a" (Value
), "d" (Value
>> 32), "c" (Register
));
1784 __asm__
__volatile__("wrmsr" : : "A" (Value
), "c" (Register
));
1788 __INTRIN_INLINE
unsigned long long __readpmc(unsigned long counter
)
1790 unsigned long long retval
;
1791 __asm__
__volatile__("rdpmc" : "=A" (retval
) : "c" (counter
));
1795 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1796 __INTRIN_INLINE
unsigned long __segmentlimit(unsigned long a
)
1798 unsigned long retval
;
1799 __asm__
__volatile__("lsl %[a], %[retval]" : [retval
] "=r" (retval
) : [a
] "rm" (a
));
1803 __INTRIN_INLINE
void __wbinvd(void)
1805 __asm__
__volatile__("wbinvd" : : : "memory");
1808 __INTRIN_INLINE
void __lidt(void *Source
)
1810 __asm__
__volatile__("lidt %0" : : "m"(*(short*)Source
));
1813 __INTRIN_INLINE
void __sidt(void *Destination
)
1815 __asm__
__volatile__("sidt %0" : : "m"(*(short*)Destination
) : "memory");
1818 __INTRIN_INLINE
void _sgdt(void *Destination
)
1820 __asm__
__volatile__("sgdt %0" : : "m"(*(short*)Destination
) : "memory");
1823 /*** Misc operations ***/
1825 __INTRIN_INLINE
void _mm_pause(void)
1827 __asm__
__volatile__("pause" : : : "memory");
1830 __INTRIN_INLINE
void __nop(void)
1832 __asm__
__volatile__("nop");
1839 #endif /* KJK_INTRIN_X86_H_ */