2 Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3 Microsoft Visual C++ functions. Originally developed for the ReactOS
4 (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
7 Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
9 Permission is hereby granted, free of charge, to any person obtaining a
10 copy of this software and associated documentation files (the "Software"),
11 to deal in the Software without restriction, including without limitation
12 the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 and/or sell copies of the Software, and to permit persons to whom the
14 Software is furnished to do so, subject to the following conditions:
16 The above copyright notice and this permission notice shall be included in
17 all copies or substantial portions of the Software.
19 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 DEALINGS IN THE SOFTWARE.
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
32 FIXME: review all "memory" clobbers, add/remove to match Visual C++
33 behavior: some "obvious" memory barriers are not present in the Visual C++
34 implementation - e.g. __stosX; on the other hand, some memory barriers that
35 *are* present could have been missed
39 NOTE: this is a *compatibility* header. Some functions may look wrong at
40 first, but they're only "as wrong" as they would be on Visual C++. Our
41 priority is compatibility
43 NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44 constraints and the uses of __volatile__ out of my... hat. Do not touch
45 them. I hate cargo cult programming
47 NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48 barriers aren't there in Visual C++ (e.g. __stosX)
50 NOTE: review all intrinsics with a return value, add/remove __volatile__
51 where necessary. If an intrinsic whose value is ignored generates a no-op
52 under Visual C++, __volatile__ must be omitted; if it always generates code
53 (for example, if it has side effects), __volatile__ must be specified. GCC
54 will only optimize out non-volatile asm blocks with outputs, so input-only
55 blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56 and follow Visual C++ behavior
58 NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59 atomic operations. Test the version like this:
61 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
64 Pay attention to the type of barrier. Make it match with what Visual C++
65 would use in the same case
72 /*** Stack frame juggling ***/
73 #define _ReturnAddress() (__builtin_return_address(0))
74 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
75 /* TODO: __getcallerseflags but how??? */
78 /*** Atomic operations ***/
80 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
81 #define _ReadWriteBarrier() __sync_synchronize()
83 static void __inline__
__attribute__((always_inline
)) _MemoryBarrier(void)
85 __asm__
__volatile__("" : : : "memory");
87 #define _ReadWriteBarrier() _MemoryBarrier()
90 /* BUGBUG: GCC only supports full barriers */
91 #define _ReadBarrier _ReadWriteBarrier
92 #define _WriteBarrier _ReadWriteBarrier
94 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
96 static __inline__
__attribute__((always_inline
)) char _InterlockedCompareExchange8(volatile char * const Destination
, const char Exchange
, const char Comperand
)
98 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
101 static __inline__
__attribute__((always_inline
)) short _InterlockedCompareExchange16(volatile short * const Destination
, const short Exchange
, const short Comperand
)
103 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
106 static __inline__
__attribute__((always_inline
)) long _InterlockedCompareExchange(volatile long * const Destination
, const long Exchange
, const long Comperand
)
108 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
111 static __inline__
__attribute__((always_inline
)) void * _InterlockedCompareExchangePointer(void * volatile * const Destination
, void * const Exchange
, void * const Comperand
)
113 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
116 static __inline__
__attribute__((always_inline
)) long _InterlockedExchange(volatile long * const Target
, const long Value
)
118 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
119 __sync_synchronize();
120 return __sync_lock_test_and_set(Target
, Value
);
123 #if defined(_M_AMD64)
124 static __inline__
__attribute__((always_inline
)) long long _InterlockedExchange64(volatile long long * const Target
, const long long Value
)
126 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
127 __sync_synchronize();
128 return __sync_lock_test_and_set(Target
, Value
);
132 static __inline__
__attribute__((always_inline
)) void * _InterlockedExchangePointer(void * volatile * const Target
, void * const Value
)
135 __sync_synchronize();
136 return __sync_lock_test_and_set(Target
, Value
);
139 static __inline__
__attribute__((always_inline
)) long _InterlockedExchangeAdd16(volatile short * const Addend
, const short Value
)
141 return __sync_fetch_and_add(Addend
, Value
);
144 static __inline__
__attribute__((always_inline
)) long _InterlockedExchangeAdd(volatile long * const Addend
, const long Value
)
146 return __sync_fetch_and_add(Addend
, Value
);
149 #if defined(_M_AMD64)
150 static __inline__
__attribute__((always_inline
)) long long _InterlockedExchangeAdd64(volatile long long * const Addend
, const long long Value
)
152 return __sync_fetch_and_add(Addend
, Value
);
156 static __inline__
__attribute__((always_inline
)) char _InterlockedAnd8(volatile char * const value
, const char mask
)
158 return __sync_fetch_and_and(value
, mask
);
161 static __inline__
__attribute__((always_inline
)) short _InterlockedAnd16(volatile short * const value
, const short mask
)
163 return __sync_fetch_and_and(value
, mask
);
166 static __inline__
__attribute__((always_inline
)) long _InterlockedAnd(volatile long * const value
, const long mask
)
168 return __sync_fetch_and_and(value
, mask
);
171 #if defined(_M_AMD64)
172 static __inline__
__attribute__((always_inline
)) long _InterlockedAnd64(volatile long long * const value
, const long long mask
)
174 return __sync_fetch_and_and(value
, mask
);
178 static __inline__
__attribute__((always_inline
)) char _InterlockedOr8(volatile char * const value
, const char mask
)
180 return __sync_fetch_and_or(value
, mask
);
183 static __inline__
__attribute__((always_inline
)) short _InterlockedOr16(volatile short * const value
, const short mask
)
185 return __sync_fetch_and_or(value
, mask
);
188 static __inline__
__attribute__((always_inline
)) long _InterlockedOr(volatile long * const value
, const long mask
)
190 return __sync_fetch_and_or(value
, mask
);
193 #if defined(_M_AMD64)
194 static __inline__
__attribute__((always_inline
)) long _InterlockedOr64(volatile long long * const value
, const long long mask
)
196 return __sync_fetch_and_or(value
, mask
);
200 static __inline__
__attribute__((always_inline
)) char _InterlockedXor8(volatile char * const value
, const char mask
)
202 return __sync_fetch_and_xor(value
, mask
);
205 static __inline__
__attribute__((always_inline
)) short _InterlockedXor16(volatile short * const value
, const short mask
)
207 return __sync_fetch_and_xor(value
, mask
);
210 static __inline__
__attribute__((always_inline
)) long _InterlockedXor(volatile long * const value
, const long mask
)
212 return __sync_fetch_and_xor(value
, mask
);
217 static __inline__
__attribute__((always_inline
)) char _InterlockedCompareExchange8(volatile char * const Destination
, const char Exchange
, const char Comperand
)
219 char retval
= Comperand
;
220 __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
) : "memory");
224 static __inline__
__attribute__((always_inline
)) short _InterlockedCompareExchange16(volatile short * const Destination
, const short Exchange
, const short Comperand
)
226 short retval
= Comperand
;
227 __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
): "memory");
231 static __inline__
__attribute__((always_inline
)) long _InterlockedCompareExchange(volatile long * const Destination
, const long Exchange
, const long Comperand
)
233 long retval
= Comperand
;
234 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval
] "+a" (retval
) : [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
): "memory");
238 static __inline__
__attribute__((always_inline
)) void * _InterlockedCompareExchangePointer(void * volatile * const Destination
, void * const Exchange
, void * const Comperand
)
240 void * retval
= (void *)Comperand
;
241 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval
] "=a" (retval
) : "[retval]" (retval
), [Destination
] "m" (*Destination
), [Exchange
] "q" (Exchange
) : "memory");
245 static __inline__
__attribute__((always_inline
)) long _InterlockedExchange(volatile long * const Target
, const long Value
)
248 __asm__("xchgl %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
252 static __inline__
__attribute__((always_inline
)) void * _InterlockedExchangePointer(void * volatile * const Target
, void * const Value
)
254 void * retval
= Value
;
255 __asm__("xchgl %[retval], %[Target]" : [retval
] "+r" (retval
) : [Target
] "m" (*Target
) : "memory");
259 static __inline__
__attribute__((always_inline
)) long _InterlockedExchangeAdd16(volatile short * const Addend
, const short Value
)
262 __asm__("lock; xaddw %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
266 static __inline__
__attribute__((always_inline
)) long _InterlockedExchangeAdd(volatile long * const Addend
, const long Value
)
269 __asm__("lock; xaddl %[retval], %[Addend]" : [retval
] "+r" (retval
) : [Addend
] "m" (*Addend
) : "memory");
273 static __inline__
__attribute__((always_inline
)) char _InterlockedAnd8(volatile char * const value
, const char mask
)
283 y
= _InterlockedCompareExchange8(value
, x
& mask
, x
);
290 static __inline__
__attribute__((always_inline
)) short _InterlockedAnd16(volatile short * const value
, const short mask
)
300 y
= _InterlockedCompareExchange16(value
, x
& mask
, x
);
307 static __inline__
__attribute__((always_inline
)) long _InterlockedAnd(volatile long * const value
, const long mask
)
317 y
= _InterlockedCompareExchange(value
, x
& mask
, x
);
324 static __inline__
__attribute__((always_inline
)) char _InterlockedOr8(volatile char * const value
, const char mask
)
334 y
= _InterlockedCompareExchange8(value
, x
| mask
, x
);
341 static __inline__
__attribute__((always_inline
)) short _InterlockedOr16(volatile short * const value
, const short mask
)
351 y
= _InterlockedCompareExchange16(value
, x
| mask
, x
);
358 static __inline__
__attribute__((always_inline
)) long _InterlockedOr(volatile long * const value
, const long mask
)
368 y
= _InterlockedCompareExchange(value
, x
| mask
, x
);
375 static __inline__
__attribute__((always_inline
)) char _InterlockedXor8(volatile char * const value
, const char mask
)
385 y
= _InterlockedCompareExchange8(value
, x
^ mask
, x
);
392 static __inline__
__attribute__((always_inline
)) short _InterlockedXor16(volatile short * const value
, const short mask
)
402 y
= _InterlockedCompareExchange16(value
, x
^ mask
, x
);
409 static __inline__
__attribute__((always_inline
)) long _InterlockedXor(volatile long * const value
, const long mask
)
419 y
= _InterlockedCompareExchange(value
, x
^ mask
, x
);
428 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
430 static __inline__
__attribute__((always_inline
)) long long _InterlockedCompareExchange64(volatile long long * const Destination
, const long long Exchange
, const long long Comperand
)
432 return __sync_val_compare_and_swap(Destination
, Comperand
, Exchange
);
437 static __inline__
__attribute__((always_inline
)) long long _InterlockedCompareExchange64(volatile long long * const Destination
, const long long Exchange
, const long long Comperand
)
439 long long retval
= Comperand
;
443 "lock; cmpxchg8b %[Destination]" :
444 [retval
] "+A" (retval
) :
445 [Destination
] "m" (*Destination
),
446 "b" ((unsigned long)((Exchange
>> 0) & 0xFFFFFFFF)),
447 "c" ((unsigned long)((Exchange
>> 32) & 0xFFFFFFFF)) :
456 static __inline__
__attribute__((always_inline
)) long _InterlockedAddLargeStatistic(volatile long long * const Addend
, const long Value
)
460 "lock; add %[Value], %[Lo32];"
462 "lock; adc $0, %[Hi32];"
464 [Lo32
] "+m" (*((volatile long *)(Addend
) + 0)), [Hi32
] "+m" (*((volatile long *)(Addend
) + 1)) :
465 [Value
] "ir" (Value
) :
472 static __inline__
__attribute__((always_inline
)) long _InterlockedDecrement(volatile long * const lpAddend
)
474 return _InterlockedExchangeAdd(lpAddend
, -1) - 1;
477 static __inline__
__attribute__((always_inline
)) long _InterlockedIncrement(volatile long * const lpAddend
)
479 return _InterlockedExchangeAdd(lpAddend
, 1) + 1;
482 static __inline__
__attribute__((always_inline
)) long _InterlockedDecrement16(volatile short * const lpAddend
)
484 return _InterlockedExchangeAdd16(lpAddend
, -1) - 1;
487 static __inline__
__attribute__((always_inline
)) long _InterlockedIncrement16(volatile short * const lpAddend
)
489 return _InterlockedExchangeAdd16(lpAddend
, 1) + 1;
492 #if defined(_M_AMD64)
493 static __inline__
__attribute__((always_inline
)) long long _InterlockedDecrement64(volatile long long * const lpAddend
)
495 return _InterlockedExchangeAdd64(lpAddend
, -1) - 1;
498 static __inline__
__attribute__((always_inline
)) long long _InterlockedIncrement64(volatile long long * const lpAddend
)
500 return _InterlockedExchangeAdd64(lpAddend
, 1) + 1;
504 static __inline__
__attribute__((always_inline
)) unsigned char _interlockedbittestandreset(volatile long * a
, const long b
)
506 unsigned char retval
;
507 __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
511 #if defined(_M_AMD64)
512 static __inline__
__attribute__((always_inline
)) unsigned char _interlockedbittestandreset64(volatile long long * a
, const long long b
)
514 unsigned char retval
;
515 __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval
] "=r" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
520 static __inline__
__attribute__((always_inline
)) unsigned char _interlockedbittestandset(volatile long * a
, const long b
)
522 unsigned char retval
;
523 __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval
] "=q" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
527 #if defined(_M_AMD64)
528 static __inline__
__attribute__((always_inline
)) unsigned char _interlockedbittestandset64(volatile long long * a
, const long long b
)
530 unsigned char retval
;
531 __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval
] "=r" (retval
), [a
] "+m" (*a
) : [b
] "Ir" (b
) : "memory");
536 /*** String operations ***/
537 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
538 static __inline__
__attribute__((always_inline
)) void __stosb(unsigned char * Dest
, const unsigned char Data
, size_t Count
)
543 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
544 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
548 static __inline__
__attribute__((always_inline
)) void __stosw(unsigned short * Dest
, const unsigned short Data
, size_t Count
)
553 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
554 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
558 static __inline__
__attribute__((always_inline
)) void __stosd(unsigned long * Dest
, const unsigned long Data
, size_t Count
)
563 [Dest
] "=D" (Dest
), [Count
] "=c" (Count
) :
564 "[Dest]" (Dest
), "a" (Data
), "[Count]" (Count
)
568 static __inline__
__attribute__((always_inline
)) void __movsb(unsigned char * Destination
, const unsigned char * Source
, size_t Count
)
573 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
574 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
578 static __inline__
__attribute__((always_inline
)) void __movsw(unsigned short * Destination
, const unsigned short * Source
, size_t Count
)
583 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
584 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
588 static __inline__
__attribute__((always_inline
)) void __movsd(unsigned long * Destination
, const unsigned long * Source
, size_t Count
)
593 [Destination
] "=D" (Destination
), [Source
] "=S" (Source
), [Count
] "=c" (Count
) :
594 "[Destination]" (Destination
), "[Source]" (Source
), "[Count]" (Count
)
598 #if defined(_M_AMD64)
599 /*** GS segment addressing ***/
601 static __inline__
__attribute__((always_inline
)) void __writegsbyte(const unsigned long Offset
, const unsigned char Data
)
603 __asm__
__volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
606 static __inline__
__attribute__((always_inline
)) void __writegsword(const unsigned long Offset
, const unsigned short Data
)
608 __asm__
__volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
611 static __inline__
__attribute__((always_inline
)) void __writegsdword(const unsigned long Offset
, const unsigned long Data
)
613 __asm__
__volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
616 static __inline__
__attribute__((always_inline
)) void __writegsqword(const unsigned long Offset
, const unsigned __int64 Data
)
618 __asm__
__volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
621 static __inline__
__attribute__((always_inline
)) unsigned char __readgsbyte(const unsigned long Offset
)
624 __asm__
__volatile__("movb %%gs:%a[Offset], %b[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
628 static __inline__
__attribute__((always_inline
)) unsigned short __readgsword(const unsigned long Offset
)
630 unsigned short value
;
631 __asm__
__volatile__("movw %%gs:%a[Offset], %w[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
635 static __inline__
__attribute__((always_inline
)) unsigned long __readgsdword(const unsigned long Offset
)
638 __asm__
__volatile__("movl %%gs:%a[Offset], %k[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
642 static __inline__
__attribute__((always_inline
)) unsigned __int64
__readgsqword(const unsigned long Offset
)
644 unsigned __int64 value
;
645 __asm__
__volatile__("movq %%gs:%a[Offset], %q[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
649 static __inline__
__attribute__((always_inline
)) void __incgsbyte(const unsigned long Offset
)
651 __asm__
__volatile__("incb %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
654 static __inline__
__attribute__((always_inline
)) void __incgsword(const unsigned long Offset
)
656 __asm__
__volatile__("incw %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
659 static __inline__
__attribute__((always_inline
)) void __incgsdword(const unsigned long Offset
)
661 __asm__
__volatile__("incl %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
664 static __inline__
__attribute__((always_inline
)) void __addgsbyte(const unsigned long Offset
, const unsigned char Data
)
666 __asm__
__volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
669 static __inline__
__attribute__((always_inline
)) void __addgsword(const unsigned long Offset
, const unsigned short Data
)
671 __asm__
__volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
674 static __inline__
__attribute__((always_inline
)) void __addgsdword(const unsigned long Offset
, const unsigned int Data
)
676 __asm__
__volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
679 static __inline__
__attribute__((always_inline
)) void __addgsqword(const unsigned long Offset
, const unsigned __int64 Data
)
681 __asm__
__volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
685 /*** FS segment addressing ***/
686 static __inline__
__attribute__((always_inline
)) void __writefsbyte(const unsigned long Offset
, const unsigned char Data
)
688 __asm__
__volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
691 static __inline__
__attribute__((always_inline
)) void __writefsword(const unsigned long Offset
, const unsigned short Data
)
693 __asm__
__volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
696 static __inline__
__attribute__((always_inline
)) void __writefsdword(const unsigned long Offset
, const unsigned long Data
)
698 __asm__
__volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
701 static __inline__
__attribute__((always_inline
)) unsigned char __readfsbyte(const unsigned long Offset
)
704 __asm__
__volatile__("movb %%fs:%a[Offset], %b[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
) : "memory");
708 static __inline__
__attribute__((always_inline
)) unsigned short __readfsword(const unsigned long Offset
)
710 unsigned short value
;
711 __asm__
__volatile__("movw %%fs:%a[Offset], %w[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
715 static __inline__
__attribute__((always_inline
)) unsigned long __readfsdword(const unsigned long Offset
)
718 __asm__
__volatile__("movl %%fs:%a[Offset], %k[value]" : [value
] "=q" (value
) : [Offset
] "irm" (Offset
));
722 static __inline__
__attribute__((always_inline
)) void __incfsbyte(const unsigned long Offset
)
724 __asm__
__volatile__("incb %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
727 static __inline__
__attribute__((always_inline
)) void __incfsword(const unsigned long Offset
)
729 __asm__
__volatile__("incw %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
732 static __inline__
__attribute__((always_inline
)) void __incfsdword(const unsigned long Offset
)
734 __asm__
__volatile__("incl %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
) : "memory");
737 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
738 static __inline__
__attribute__((always_inline
)) void __addfsbyte(const unsigned long Offset
, const unsigned char Data
)
740 if(!__builtin_constant_p(Offset
))
741 __asm__
__volatile__("addb %k[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
) : "memory");
743 __asm__
__volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
746 static __inline__
__attribute__((always_inline
)) void __addfsword(const unsigned long Offset
, const unsigned short Data
)
748 if(!__builtin_constant_p(Offset
))
749 __asm__
__volatile__("addw %k[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
) : "memory");
751 __asm__
__volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
754 static __inline__
__attribute__((always_inline
)) void __addfsdword(const unsigned long Offset
, const unsigned int Data
)
756 if(!__builtin_constant_p(Offset
))
757 __asm__
__volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset
] "r" (Offset
) : "memory");
759 __asm__
__volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset
] "ir" (Offset
), [Data
] "iq" (Data
) : "memory");
764 /*** Bit manipulation ***/
765 static __inline__
__attribute__((always_inline
)) unsigned char _BitScanForward(unsigned long * const Index
, const unsigned long Mask
)
767 __asm__("bsfl %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
771 static __inline__
__attribute__((always_inline
)) unsigned char _BitScanReverse(unsigned long * const Index
, const unsigned long Mask
)
773 __asm__("bsrl %[Mask], %[Index]" : [Index
] "=r" (*Index
) : [Mask
] "mr" (Mask
));
777 /* NOTE: again, the bizarre implementation follows Visual C++ */
778 static __inline__
__attribute__((always_inline
)) unsigned char _bittest(const long * const a
, const long b
)
780 unsigned char retval
;
782 if(__builtin_constant_p(b
))
783 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*(a
+ (b
/ 32))), [b
] "Ir" (b
% 32));
785 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval
] "=q" (retval
) : [a
] "mr" (*a
), [b
] "r" (b
));
790 static __inline__
__attribute__((always_inline
)) unsigned char _bittestandcomplement(long * const a
, const long b
)
792 unsigned char retval
;
794 if(__builtin_constant_p(b
))
795 __asm__("btc %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 32))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 32));
797 __asm__("btc %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
802 static __inline__
__attribute__((always_inline
)) unsigned char _bittestandreset(long * const a
, const long b
)
804 unsigned char retval
;
806 if(__builtin_constant_p(b
))
807 __asm__("btr %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 32))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 32));
809 __asm__("btr %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
814 static __inline__
__attribute__((always_inline
)) unsigned char _bittestandset(long * const a
, const long b
)
816 unsigned char retval
;
818 if(__builtin_constant_p(b
))
819 __asm__("bts %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*(a
+ (b
/ 32))), [retval
] "=q" (retval
) : [b
] "Ir" (b
% 32));
821 __asm__("bts %[b], %[a]; setb %b[retval]" : [a
] "+mr" (*a
), [retval
] "=q" (retval
) : [b
] "r" (b
));
826 static __inline__
__attribute__((always_inline
)) unsigned char _rotl8(unsigned char value
, unsigned char shift
)
828 unsigned char retval
;
829 __asm__("rolb %b[shift], %b[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
833 static __inline__
__attribute__((always_inline
)) unsigned short _rotl16(unsigned short value
, unsigned char shift
)
835 unsigned short retval
;
836 __asm__("rolw %b[shift], %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
841 static __inline__
__attribute__((always_inline
)) unsigned int _rotl(unsigned int value
, int shift
)
843 unsigned long retval
;
844 __asm__("roll %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
848 static __inline__
__attribute__((always_inline
)) unsigned int _rotr(unsigned int value
, int shift
)
850 unsigned long retval
;
851 __asm__("rorl %b[shift], %k[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
856 static __inline__
__attribute__((always_inline
)) unsigned char _rotr8(unsigned char value
, unsigned char shift
)
858 unsigned char retval
;
859 __asm__("rorb %b[shift], %b[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
863 static __inline__
__attribute__((always_inline
)) unsigned short _rotr16(unsigned short value
, unsigned char shift
)
865 unsigned short retval
;
866 __asm__("rorw %b[shift], %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
), [shift
] "Nc" (shift
));
871 NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
872 constraint (edx:eax) for the Mask argument, because it's the only way GCC
873 can pass 64-bit operands around - passing the two 32 bit parts separately
874 just confuses it. Also we declare Bit as an int and then truncate it to
875 match Visual C++ behavior
877 static __inline__
__attribute__((always_inline
)) unsigned long long __ll_lshift(const unsigned long long Mask
, const int Bit
)
879 unsigned long long retval
= Mask
;
883 "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
885 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
891 static __inline__
__attribute__((always_inline
)) long long __ll_rshift(const long long Mask
, const int Bit
)
893 unsigned long long retval
= Mask
;
897 "shldl %b[Bit], %%eax, %%edx; sarl %b[Bit], %%eax" :
899 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
905 static __inline__
__attribute__((always_inline
)) unsigned long long __ull_rshift(const unsigned long long Mask
, int Bit
)
907 unsigned long long retval
= Mask
;
911 "shrdl %b[Bit], %%eax, %%edx; shrl %b[Bit], %%eax" :
913 [Bit
] "Nc" ((unsigned char)((unsigned long)Bit
) & 0xFF)
919 static __inline__
__attribute__((always_inline
)) unsigned short _byteswap_ushort(unsigned short value
)
921 unsigned short retval
;
922 __asm__("rorw $8, %w[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
));
926 static __inline__
__attribute__((always_inline
)) unsigned long _byteswap_ulong(unsigned long value
)
928 unsigned long retval
;
929 __asm__("bswapl %[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
));
934 static __inline__
__attribute__((always_inline
)) unsigned __int64
_byteswap_uint64(unsigned __int64 value
)
936 unsigned __int64 retval
;
937 __asm__("bswapq %[retval]" : [retval
] "=rm" (retval
) : "[retval]" (value
));
941 static __inline__
__attribute__((always_inline
)) unsigned __int64
_byteswap_uint64(unsigned __int64 value
)
946 unsigned long lowpart
;
947 unsigned long hipart
;
950 retval
.int64part
= value
;
951 __asm__("bswapl %[lowpart]\n"
953 : [lowpart
] "=rm" (retval
.hipart
), [hipart
] "=rm" (retval
.lowpart
) : "[lowpart]" (retval
.lowpart
), "[hipart]" (retval
.hipart
) );
954 return retval
.int64part
;
958 /*** 64-bit math ***/
959 static __inline__
__attribute__((always_inline
)) long long __emul(const int a
, const int b
)
962 __asm__("imull %[b]" : "=A" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
966 static __inline__
__attribute__((always_inline
)) unsigned long long __emulu(const unsigned int a
, const unsigned int b
)
968 unsigned long long retval
;
969 __asm__("mull %[b]" : "=A" (retval
) : [a
] "a" (a
), [b
] "rm" (b
));
975 static __inline__
__attribute__((always_inline
)) unsigned char __inbyte(const unsigned short Port
)
978 __asm__
__volatile__("inb %w[Port], %b[byte]" : [byte
] "=a" (byte
) : [Port
] "Nd" (Port
));
982 static __inline__
__attribute__((always_inline
)) unsigned short __inword(const unsigned short Port
)
985 __asm__
__volatile__("inw %w[Port], %w[word]" : [word
] "=a" (word
) : [Port
] "Nd" (Port
));
989 static __inline__
__attribute__((always_inline
)) unsigned long __indword(const unsigned short Port
)
992 __asm__
__volatile__("inl %w[Port], %k[dword]" : [dword
] "=a" (dword
) : [Port
] "Nd" (Port
));
996 static __inline__
__attribute__((always_inline
)) void __inbytestring(unsigned short Port
, unsigned char * Buffer
, unsigned long Count
)
1001 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
1002 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
1007 static __inline__
__attribute__((always_inline
)) void __inwordstring(unsigned short Port
, unsigned short * Buffer
, unsigned long Count
)
1009 __asm__ __volatile__
1012 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
1013 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
1018 static __inline__
__attribute__((always_inline
)) void __indwordstring(unsigned short Port
, unsigned long * Buffer
, unsigned long Count
)
1020 __asm__ __volatile__
1023 [Buffer
] "=D" (Buffer
), [Count
] "=c" (Count
) :
1024 "d" (Port
), "[Buffer]" (Buffer
), "[Count]" (Count
) :
1029 static __inline__
__attribute__((always_inline
)) void __outbyte(unsigned short const Port
, const unsigned char Data
)
1031 __asm__
__volatile__("outb %b[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
1034 static __inline__
__attribute__((always_inline
)) void __outword(unsigned short const Port
, const unsigned short Data
)
1036 __asm__
__volatile__("outw %w[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
1039 static __inline__
__attribute__((always_inline
)) void __outdword(unsigned short const Port
, const unsigned long Data
)
1041 __asm__
__volatile__("outl %k[Data], %w[Port]" : : [Port
] "Nd" (Port
), [Data
] "a" (Data
));
1044 static __inline__
__attribute__((always_inline
)) void __outbytestring(unsigned short const Port
, const unsigned char * const Buffer
, const unsigned long Count
)
1046 __asm__
__volatile__("rep; outsb" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
1049 static __inline__
__attribute__((always_inline
)) void __outwordstring(unsigned short const Port
, const unsigned short * const Buffer
, const unsigned long Count
)
1051 __asm__
__volatile__("rep; outsw" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
1054 static __inline__
__attribute__((always_inline
)) void __outdwordstring(unsigned short const Port
, const unsigned long * const Buffer
, const unsigned long Count
)
1056 __asm__
__volatile__("rep; outsl" : : [Port
] "d" (Port
), [Buffer
] "S" (Buffer
), "c" (Count
));
1060 /*** System information ***/
1061 static __inline__
__attribute__((always_inline
)) void __cpuid(int CPUInfo
[], const int InfoType
)
1063 __asm__
__volatile__("cpuid" : "=a" (CPUInfo
[0]), "=b" (CPUInfo
[1]), "=c" (CPUInfo
[2]), "=d" (CPUInfo
[3]) : "a" (InfoType
));
1066 static __inline__
__attribute__((always_inline
)) unsigned long long __rdtsc(void)
1069 unsigned long long low
, high
;
1070 __asm__
__volatile__("rdtsc" : "=a"(low
), "=d"(high
));
1071 return low
| (high
<< 32);
1073 unsigned long long retval
;
1074 __asm__
__volatile__("rdtsc" : "=A"(retval
));
1079 static __inline__
__attribute__((always_inline
)) void __writeeflags(uintptr_t Value
)
1081 __asm__
__volatile__("push %0\n popf" : : "rim"(Value
));
1084 static __inline__
__attribute__((always_inline
)) uintptr_t __readeflags(void)
1087 __asm__
__volatile__("pushf\n pop %0" : "=rm"(retval
));
1091 /*** Interrupts ***/
1092 static __inline__
__attribute__((always_inline
)) void __debugbreak(void)
1097 static __inline__
__attribute__((always_inline
)) void __int2c(void)
1099 __asm__("int $0x2c");
1102 static __inline__
__attribute__((always_inline
)) void _disable(void)
1107 static __inline__
__attribute__((always_inline
)) void _enable(void)
1113 /*** Protected memory management ***/
1115 static __inline__
__attribute__((always_inline
)) void __writecr0(const unsigned __int64 Data
)
1117 __asm__("mov %[Data], %%cr0" : : [Data
] "q" (Data
) : "memory");
1120 static __inline__
__attribute__((always_inline
)) void __writecr3(const unsigned __int64 Data
)
1122 __asm__("mov %[Data], %%cr3" : : [Data
] "q" (Data
) : "memory");
1125 static __inline__
__attribute__((always_inline
)) void __writecr4(const unsigned __int64 Data
)
1127 __asm__("mov %[Data], %%cr4" : : [Data
] "q" (Data
) : "memory");
1131 static __inline__
__attribute__((always_inline
)) void __writecr8(const unsigned __int64 Data
)
1133 __asm__("mov %[Data], %%cr8" : : [Data
] "q" (Data
) : "memory");
1137 static __inline__
__attribute__((always_inline
)) unsigned __int64
__readcr0(void)
1139 unsigned __int64 value
;
1140 __asm__
__volatile__("mov %%cr0, %[value]" : [value
] "=q" (value
));
1144 static __inline__
__attribute__((always_inline
)) unsigned __int64
__readcr2(void)
1146 unsigned __int64 value
;
1147 __asm__
__volatile__("mov %%cr2, %[value]" : [value
] "=q" (value
));
1151 static __inline__
__attribute__((always_inline
)) unsigned __int64
__readcr3(void)
1153 unsigned __int64 value
;
1154 __asm__
__volatile__("mov %%cr3, %[value]" : [value
] "=q" (value
));
1158 static __inline__
__attribute__((always_inline
)) unsigned __int64
__readcr4(void)
1160 unsigned __int64 value
;
1161 __asm__
__volatile__("mov %%cr4, %[value]" : [value
] "=q" (value
));
1166 static __inline__
__attribute__((always_inline
)) unsigned __int64
__readcr8(void)
1168 unsigned __int64 value
;
1169 __asm__
__volatile__("movq %%cr8, %q[value]" : [value
] "=q" (value
));
1175 static __inline__
__attribute__((always_inline
)) unsigned __int64
__readdr(unsigned int reg
)
1177 unsigned __int64 value
;
1181 __asm__
__volatile__("movq %%dr0, %q[value]" : [value
] "=q" (value
));
1184 __asm__
__volatile__("movq %%dr1, %q[value]" : [value
] "=q" (value
));
1187 __asm__
__volatile__("movq %%dr2, %q[value]" : [value
] "=q" (value
));
1190 __asm__
__volatile__("movq %%dr3, %q[value]" : [value
] "=q" (value
));
1193 __asm__
__volatile__("movq %%dr4, %q[value]" : [value
] "=q" (value
));
1196 __asm__
__volatile__("movq %%dr5, %q[value]" : [value
] "=q" (value
));
1199 __asm__
__volatile__("movq %%dr6, %q[value]" : [value
] "=q" (value
));
1202 __asm__
__volatile__("movq %%dr7, %q[value]" : [value
] "=q" (value
));
1208 static __inline__
__attribute__((always_inline
)) void __writedr(unsigned reg
, unsigned __int64 value
)
1213 __asm__("movq %q[value], %%dr0" : : [value
] "q" (value
) : "memory");
1216 __asm__("movq %q[value], %%dr1" : : [value
] "q" (value
) : "memory");
1219 __asm__("movq %q[value], %%dr2" : : [value
] "q" (value
) : "memory");
1222 __asm__("movq %q[value], %%dr3" : : [value
] "q" (value
) : "memory");
1225 __asm__("movq %q[value], %%dr4" : : [value
] "q" (value
) : "memory");
1228 __asm__("movq %q[value], %%dr5" : : [value
] "q" (value
) : "memory");
1231 __asm__("movq %q[value], %%dr6" : : [value
] "q" (value
) : "memory");
1234 __asm__("movq %q[value], %%dr7" : : [value
] "q" (value
) : "memory");
1240 static __inline__
__attribute__((always_inline
)) void __invlpg(void * const Address
)
1242 __asm__("invlpg %[Address]" : : [Address
] "m" (*((unsigned char *)(Address
))));
1246 /*** System operations ***/
1247 static __inline__
__attribute__((always_inline
)) unsigned long long __readmsr(const int reg
)
1250 unsigned long low
, high
;
1251 __asm__
__volatile__("rdmsr" : "=a" (low
), "=d" (high
) : "c" (reg
));
1252 return ((unsigned long long)high
<< 32) | low
;
1254 unsigned long long retval
;
1255 __asm__
__volatile__("rdmsr" : "=A" (retval
) : "c" (reg
));
1260 static __inline__
__attribute__((always_inline
)) void __writemsr(const unsigned long Register
, const unsigned long long Value
)
1263 __asm__
__volatile__("wrmsr" : : "a" (Value
), "d" (Value
>> 32), "c" (Register
));
1265 __asm__
__volatile__("wrmsr" : : "A" (Value
), "c" (Register
));
1269 static __inline__
__attribute__((always_inline
)) unsigned long long __readpmc(const int counter
)
1271 unsigned long long retval
;
1272 __asm__
__volatile__("rdpmc" : "=A" (retval
) : "c" (counter
));
1276 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1277 static __inline__
__attribute__((always_inline
)) unsigned long __segmentlimit(const unsigned long a
)
1279 unsigned long retval
;
1280 __asm__
__volatile__("lsl %[a], %[retval]" : [retval
] "=r" (retval
) : [a
] "rm" (a
));
1284 static __inline__
__attribute__((always_inline
)) void __wbinvd(void)
1286 __asm__
__volatile__("wbinvd");
1289 static __inline__
__attribute__((always_inline
)) void __lidt(void *Source
)
1291 __asm__
__volatile__("lidt %0" : : "m"(*(short*)Source
));
1294 static __inline__
__attribute__((always_inline
)) void __sidt(void *Destination
)
1296 __asm__
__volatile__("sidt %0" : : "m"(*(short*)Destination
) : "memory");
1303 #endif /* KJK_INTRIN_X86_H_ */