[INTRIN]
[reactos.git] / include / crt / mingw32 / intrin_x86.h
1 /*
2 Compatibility <intrin_x86.h> header for GCC -- GCC equivalents of intrinsic
3 Microsoft Visual C++ functions. Originally developed for the ReactOS
4 (<http://www.reactos.org/>) and TinyKrnl (<http://www.tinykrnl.org/>)
5 projects.
6
7 Copyright (c) 2006 KJK::Hyperion <hackbunny@reactos.com>
8
9 Permission is hereby granted, free of charge, to any person obtaining a
10 copy of this software and associated documentation files (the "Software"),
11 to deal in the Software without restriction, including without limitation
12 the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 and/or sell copies of the Software, and to permit persons to whom the
14 Software is furnished to do so, subject to the following conditions:
15
16 The above copyright notice and this permission notice shall be included in
17 all copies or substantial portions of the Software.
18
19 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 DEALINGS IN THE SOFTWARE.
26 */
27
28 #ifndef KJK_INTRIN_X86_H_
29 #define KJK_INTRIN_X86_H_
30
31 /*
32 FIXME: review all "memory" clobbers, add/remove to match Visual C++
33 behavior: some "obvious" memory barriers are not present in the Visual C++
34 implementation - e.g. __stosX; on the other hand, some memory barriers that
35 *are* present could have been missed
36 */
37
38 /*
39 NOTE: this is a *compatibility* header. Some functions may look wrong at
40 first, but they're only "as wrong" as they would be on Visual C++. Our
41 priority is compatibility
42
43 NOTE: unlike most people who write inline asm for GCC, I didn't pull the
44 constraints and the uses of __volatile__ out of my... hat. Do not touch
45 them. I hate cargo cult programming
46
47 NOTE: be very careful with declaring "memory" clobbers. Some "obvious"
48 barriers aren't there in Visual C++ (e.g. __stosX)
49
50 NOTE: review all intrinsics with a return value, add/remove __volatile__
51 where necessary. If an intrinsic whose value is ignored generates a no-op
52 under Visual C++, __volatile__ must be omitted; if it always generates code
53 (for example, if it has side effects), __volatile__ must be specified. GCC
54 will only optimize out non-volatile asm blocks with outputs, so input-only
55 blocks are safe. Oddities such as the non-volatile 'rdmsr' are intentional
56 and follow Visual C++ behavior
57
58 NOTE: on GCC 4.1.0, please use the __sync_* built-ins for barriers and
59 atomic operations. Test the version like this:
60
61 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
62 ...
63
64 Pay attention to the type of barrier. Make it match with what Visual C++
65 would use in the same case
66 */
67
68 #ifdef __cplusplus
69 extern "C" {
70 #endif
71
72 /*** Stack frame juggling ***/
73 #define _ReturnAddress() (__builtin_return_address(0))
74 #define _AddressOfReturnAddress() (&(((void **)(__builtin_frame_address(0)))[1]))
75 /* TODO: __getcallerseflags but how??? */
76
77 /* Maybe the same for x86? */
78 #ifdef _x86_64
79 #define _alloca(s) __builtin_alloca(s)
80 #endif
81
82 /*** Atomic operations ***/
83
84 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
85 #define _ReadWriteBarrier() __sync_synchronize()
86 #else
87 __INTRIN_INLINE void _MemoryBarrier(void)
88 {
89 __asm__ __volatile__("" : : : "memory");
90 }
91 #define _ReadWriteBarrier() _MemoryBarrier()
92 #endif
93
94 /* BUGBUG: GCC only supports full barriers */
95 #define _ReadBarrier _ReadWriteBarrier
96 #define _WriteBarrier _ReadWriteBarrier
97
98 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100
99
100 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * const Destination, const char Exchange, const char Comperand)
101 {
102 return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
103 }
104
105 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * const Destination, const short Exchange, const short Comperand)
106 {
107 return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
108 }
109
110 __INTRIN_INLINE long _InterlockedCompareExchange(volatile long * const Destination, const long Exchange, const long Comperand)
111 {
112 return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
113 }
114
115 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * const Destination, void * const Exchange, void * const Comperand)
116 {
117 return (void *)__sync_val_compare_and_swap(Destination, Comperand, Exchange);
118 }
119
120 __INTRIN_INLINE long _InterlockedExchange(volatile long * const Target, const long Value)
121 {
122 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
123 __sync_synchronize();
124 return __sync_lock_test_and_set(Target, Value);
125 }
126
127 #if defined(_M_AMD64)
128 __INTRIN_INLINE long long _InterlockedExchange64(volatile long long * const Target, const long long Value)
129 {
130 /* NOTE: __sync_lock_test_and_set would be an acquire barrier, so we force a full barrier */
131 __sync_synchronize();
132 return __sync_lock_test_and_set(Target, Value);
133 }
134 #endif
135
136 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * const Target, void * const Value)
137 {
138 /* NOTE: ditto */
139 __sync_synchronize();
140 return (void *)__sync_lock_test_and_set(Target, Value);
141 }
142
143 __INTRIN_INLINE long _InterlockedExchangeAdd16(volatile short * const Addend, const short Value)
144 {
145 return __sync_fetch_and_add(Addend, Value);
146 }
147
148 __INTRIN_INLINE long _InterlockedExchangeAdd(volatile long * const Addend, const long Value)
149 {
150 return __sync_fetch_and_add(Addend, Value);
151 }
152
153 #if defined(_M_AMD64)
154 __INTRIN_INLINE long long _InterlockedExchangeAdd64(volatile long long * const Addend, const long long Value)
155 {
156 return __sync_fetch_and_add(Addend, Value);
157 }
158 #endif
159
160 __INTRIN_INLINE char _InterlockedAnd8(volatile char * const value, const char mask)
161 {
162 return __sync_fetch_and_and(value, mask);
163 }
164
165 __INTRIN_INLINE short _InterlockedAnd16(volatile short * const value, const short mask)
166 {
167 return __sync_fetch_and_and(value, mask);
168 }
169
170 __INTRIN_INLINE long _InterlockedAnd(volatile long * const value, const long mask)
171 {
172 return __sync_fetch_and_and(value, mask);
173 }
174
175 #if defined(_M_AMD64)
176 __INTRIN_INLINE long _InterlockedAnd64(volatile long long * const value, const long long mask)
177 {
178 return __sync_fetch_and_and(value, mask);
179 }
180 #endif
181
182 __INTRIN_INLINE char _InterlockedOr8(volatile char * const value, const char mask)
183 {
184 return __sync_fetch_and_or(value, mask);
185 }
186
187 __INTRIN_INLINE short _InterlockedOr16(volatile short * const value, const short mask)
188 {
189 return __sync_fetch_and_or(value, mask);
190 }
191
192 __INTRIN_INLINE long _InterlockedOr(volatile long * const value, const long mask)
193 {
194 return __sync_fetch_and_or(value, mask);
195 }
196
197 #if defined(_M_AMD64)
198 __INTRIN_INLINE long _InterlockedOr64(volatile long long * const value, const long long mask)
199 {
200 return __sync_fetch_and_or(value, mask);
201 }
202 #endif
203
204 __INTRIN_INLINE char _InterlockedXor8(volatile char * const value, const char mask)
205 {
206 return __sync_fetch_and_xor(value, mask);
207 }
208
209 __INTRIN_INLINE short _InterlockedXor16(volatile short * const value, const short mask)
210 {
211 return __sync_fetch_and_xor(value, mask);
212 }
213
214 __INTRIN_INLINE long _InterlockedXor(volatile long * const value, const long mask)
215 {
216 return __sync_fetch_and_xor(value, mask);
217 }
218
219 #else
220
221 __INTRIN_INLINE char _InterlockedCompareExchange8(volatile char * const Destination, const char Exchange, const char Comperand)
222 {
223 char retval = Comperand;
224 __asm__("lock; cmpxchgb %b[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
225 return retval;
226 }
227
228 __INTRIN_INLINE short _InterlockedCompareExchange16(volatile short * const Destination, const short Exchange, const short Comperand)
229 {
230 short retval = Comperand;
231 __asm__("lock; cmpxchgw %w[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
232 return retval;
233 }
234
235 __INTRIN_INLINE long _InterlockedCompareExchange(volatile long * const Destination, const long Exchange, const long Comperand)
236 {
237 long retval = Comperand;
238 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "+a" (retval) : [Destination] "m" (*Destination), [Exchange] "q" (Exchange): "memory");
239 return retval;
240 }
241
242 __INTRIN_INLINE void * _InterlockedCompareExchangePointer(void * volatile * const Destination, void * const Exchange, void * const Comperand)
243 {
244 void * retval = (void *)Comperand;
245 __asm__("lock; cmpxchgl %k[Exchange], %[Destination]" : [retval] "=a" (retval) : "[retval]" (retval), [Destination] "m" (*Destination), [Exchange] "q" (Exchange) : "memory");
246 return retval;
247 }
248
249 __INTRIN_INLINE long _InterlockedExchange(volatile long * const Target, const long Value)
250 {
251 long retval = Value;
252 __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
253 return retval;
254 }
255
256 __INTRIN_INLINE void * _InterlockedExchangePointer(void * volatile * const Target, void * const Value)
257 {
258 void * retval = Value;
259 __asm__("xchgl %[retval], %[Target]" : [retval] "+r" (retval) : [Target] "m" (*Target) : "memory");
260 return retval;
261 }
262
263 __INTRIN_INLINE long _InterlockedExchangeAdd16(volatile short * const Addend, const short Value)
264 {
265 long retval = Value;
266 __asm__("lock; xaddw %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
267 return retval;
268 }
269
270 __INTRIN_INLINE long _InterlockedExchangeAdd(volatile long * const Addend, const long Value)
271 {
272 long retval = Value;
273 __asm__("lock; xaddl %[retval], %[Addend]" : [retval] "+r" (retval) : [Addend] "m" (*Addend) : "memory");
274 return retval;
275 }
276
277 __INTRIN_INLINE char _InterlockedAnd8(volatile char * const value, const char mask)
278 {
279 char x;
280 char y;
281
282 y = *value;
283
284 do
285 {
286 x = y;
287 y = _InterlockedCompareExchange8(value, x & mask, x);
288 }
289 while(y != x);
290
291 return y;
292 }
293
294 __INTRIN_INLINE short _InterlockedAnd16(volatile short * const value, const short mask)
295 {
296 short x;
297 short y;
298
299 y = *value;
300
301 do
302 {
303 x = y;
304 y = _InterlockedCompareExchange16(value, x & mask, x);
305 }
306 while(y != x);
307
308 return y;
309 }
310
311 __INTRIN_INLINE long _InterlockedAnd(volatile long * const value, const long mask)
312 {
313 long x;
314 long y;
315
316 y = *value;
317
318 do
319 {
320 x = y;
321 y = _InterlockedCompareExchange(value, x & mask, x);
322 }
323 while(y != x);
324
325 return y;
326 }
327
328 __INTRIN_INLINE char _InterlockedOr8(volatile char * const value, const char mask)
329 {
330 char x;
331 char y;
332
333 y = *value;
334
335 do
336 {
337 x = y;
338 y = _InterlockedCompareExchange8(value, x | mask, x);
339 }
340 while(y != x);
341
342 return y;
343 }
344
345 __INTRIN_INLINE short _InterlockedOr16(volatile short * const value, const short mask)
346 {
347 short x;
348 short y;
349
350 y = *value;
351
352 do
353 {
354 x = y;
355 y = _InterlockedCompareExchange16(value, x | mask, x);
356 }
357 while(y != x);
358
359 return y;
360 }
361
362 __INTRIN_INLINE long _InterlockedOr(volatile long * const value, const long mask)
363 {
364 long x;
365 long y;
366
367 y = *value;
368
369 do
370 {
371 x = y;
372 y = _InterlockedCompareExchange(value, x | mask, x);
373 }
374 while(y != x);
375
376 return y;
377 }
378
379 __INTRIN_INLINE char _InterlockedXor8(volatile char * const value, const char mask)
380 {
381 char x;
382 char y;
383
384 y = *value;
385
386 do
387 {
388 x = y;
389 y = _InterlockedCompareExchange8(value, x ^ mask, x);
390 }
391 while(y != x);
392
393 return y;
394 }
395
396 __INTRIN_INLINE short _InterlockedXor16(volatile short * const value, const short mask)
397 {
398 short x;
399 short y;
400
401 y = *value;
402
403 do
404 {
405 x = y;
406 y = _InterlockedCompareExchange16(value, x ^ mask, x);
407 }
408 while(y != x);
409
410 return y;
411 }
412
413 __INTRIN_INLINE long _InterlockedXor(volatile long * const value, const long mask)
414 {
415 long x;
416 long y;
417
418 y = *value;
419
420 do
421 {
422 x = y;
423 y = _InterlockedCompareExchange(value, x ^ mask, x);
424 }
425 while(y != x);
426
427 return y;
428 }
429
430 #endif
431
432 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) > 40100 && defined(__x86_64__)
433
434 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * const Destination, const long long Exchange, const long long Comperand)
435 {
436 return __sync_val_compare_and_swap(Destination, Comperand, Exchange);
437 }
438
439 #else
440
441 __INTRIN_INLINE long long _InterlockedCompareExchange64(volatile long long * const Destination, const long long Exchange, const long long Comperand)
442 {
443 long long retval = Comperand;
444
445 __asm__
446 (
447 "lock; cmpxchg8b %[Destination]" :
448 [retval] "+A" (retval) :
449 [Destination] "m" (*Destination),
450 "b" ((unsigned long)((Exchange >> 0) & 0xFFFFFFFF)),
451 "c" ((unsigned long)((Exchange >> 32) & 0xFFFFFFFF)) :
452 "memory"
453 );
454
455 return retval;
456 }
457
458 #endif
459
460 __INTRIN_INLINE long _InterlockedAddLargeStatistic(volatile long long * const Addend, const long Value)
461 {
462 __asm__
463 (
464 "lock; add %[Value], %[Lo32];"
465 "jae LABEL%=;"
466 "lock; adc $0, %[Hi32];"
467 "LABEL%=:;" :
468 [Lo32] "+m" (*((volatile long *)(Addend) + 0)), [Hi32] "+m" (*((volatile long *)(Addend) + 1)) :
469 [Value] "ir" (Value) :
470 "memory"
471 );
472
473 return Value;
474 }
475
476 __INTRIN_INLINE long _InterlockedDecrement(volatile long * const lpAddend)
477 {
478 return _InterlockedExchangeAdd(lpAddend, -1) - 1;
479 }
480
481 __INTRIN_INLINE long _InterlockedIncrement(volatile long * const lpAddend)
482 {
483 return _InterlockedExchangeAdd(lpAddend, 1) + 1;
484 }
485
486 __INTRIN_INLINE short _InterlockedDecrement16(volatile short * const lpAddend)
487 {
488 return _InterlockedExchangeAdd16(lpAddend, -1) - 1;
489 }
490
491 __INTRIN_INLINE short _InterlockedIncrement16(volatile short * const lpAddend)
492 {
493 return _InterlockedExchangeAdd16(lpAddend, 1) + 1;
494 }
495
496 #if defined(_M_AMD64)
497 __INTRIN_INLINE long long _InterlockedDecrement64(volatile long long * const lpAddend)
498 {
499 return _InterlockedExchangeAdd64(lpAddend, -1) - 1;
500 }
501
502 __INTRIN_INLINE long long _InterlockedIncrement64(volatile long long * const lpAddend)
503 {
504 return _InterlockedExchangeAdd64(lpAddend, 1) + 1;
505 }
506 #endif
507
508 __INTRIN_INLINE unsigned char _interlockedbittestandreset(volatile long * a, const long b)
509 {
510 unsigned char retval;
511 __asm__("lock; btrl %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
512 return retval;
513 }
514
515 #if defined(_M_AMD64)
516 __INTRIN_INLINE unsigned char _interlockedbittestandreset64(volatile long long * a, const long long b)
517 {
518 unsigned char retval;
519 __asm__("lock; btrq %[b], %[a]; setb %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
520 return retval;
521 }
522 #endif
523
524 __INTRIN_INLINE unsigned char _interlockedbittestandset(volatile long * a, const long b)
525 {
526 unsigned char retval;
527 __asm__("lock; btsl %[b], %[a]; setc %b[retval]" : [retval] "=q" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
528 return retval;
529 }
530
531 #if defined(_M_AMD64)
532 __INTRIN_INLINE unsigned char _interlockedbittestandset64(volatile long long * a, const long long b)
533 {
534 unsigned char retval;
535 __asm__("lock; btsq %[b], %[a]; setc %b[retval]" : [retval] "=r" (retval), [a] "+m" (*a) : [b] "Ir" (b) : "memory");
536 return retval;
537 }
538 #endif
539
540 /*** String operations ***/
541 /* NOTE: we don't set a memory clobber in the __stosX functions because Visual C++ doesn't */
542 __INTRIN_INLINE void __stosb(unsigned char * Dest, const unsigned char Data, size_t Count)
543 {
544 __asm__ __volatile__
545 (
546 "rep; stosb" :
547 [Dest] "=D" (Dest), [Count] "=c" (Count) :
548 "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
549 );
550 }
551
552 __INTRIN_INLINE void __stosw(unsigned short * Dest, const unsigned short Data, size_t Count)
553 {
554 __asm__ __volatile__
555 (
556 "rep; stosw" :
557 [Dest] "=D" (Dest), [Count] "=c" (Count) :
558 "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
559 );
560 }
561
562 __INTRIN_INLINE void __stosd(unsigned long * Dest, const unsigned long Data, size_t Count)
563 {
564 __asm__ __volatile__
565 (
566 "rep; stosl" :
567 [Dest] "=D" (Dest), [Count] "=c" (Count) :
568 "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
569 );
570 }
571
572 #ifdef _M_AMD64
573 __INTRIN_INLINE void __stosq(unsigned __int64 * Dest, const unsigned __int64 Data, size_t Count)
574 {
575 __asm__ __volatile__
576 (
577 "rep; stosq" :
578 [Dest] "=D" (Dest), [Count] "=c" (Count) :
579 "[Dest]" (Dest), "a" (Data), "[Count]" (Count)
580 );
581 }
582 #endif
583
584 __INTRIN_INLINE void __movsb(unsigned char * Destination, const unsigned char * Source, size_t Count)
585 {
586 __asm__ __volatile__
587 (
588 "rep; movsb" :
589 [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
590 "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
591 );
592 }
593
594 __INTRIN_INLINE void __movsw(unsigned short * Destination, const unsigned short * Source, size_t Count)
595 {
596 __asm__ __volatile__
597 (
598 "rep; movsw" :
599 [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
600 "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
601 );
602 }
603
604 __INTRIN_INLINE void __movsd(unsigned long * Destination, const unsigned long * Source, size_t Count)
605 {
606 __asm__ __volatile__
607 (
608 "rep; movsd" :
609 [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
610 "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
611 );
612 }
613
614 #ifdef _M_AMD64
615 __INTRIN_INLINE void __movsq(unsigned long * Destination, const unsigned long * Source, size_t Count)
616 {
617 __asm__ __volatile__
618 (
619 "rep; movsq" :
620 [Destination] "=D" (Destination), [Source] "=S" (Source), [Count] "=c" (Count) :
621 "[Destination]" (Destination), "[Source]" (Source), "[Count]" (Count)
622 );
623 }
624 #endif
625
626 #if defined(_M_AMD64)
627 /*** GS segment addressing ***/
628
629 __INTRIN_INLINE void __writegsbyte(const unsigned long Offset, const unsigned char Data)
630 {
631 __asm__ __volatile__("movb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
632 }
633
634 __INTRIN_INLINE void __writegsword(const unsigned long Offset, const unsigned short Data)
635 {
636 __asm__ __volatile__("movw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
637 }
638
639 __INTRIN_INLINE void __writegsdword(const unsigned long Offset, const unsigned long Data)
640 {
641 __asm__ __volatile__("movl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
642 }
643
644 __INTRIN_INLINE void __writegsqword(const unsigned long Offset, const unsigned __int64 Data)
645 {
646 __asm__ __volatile__("movq %q[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
647 }
648
649 __INTRIN_INLINE unsigned char __readgsbyte(const unsigned long Offset)
650 {
651 unsigned char value;
652 __asm__ __volatile__("movb %%gs:%a[Offset], %b[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
653 return value;
654 }
655
656 __INTRIN_INLINE unsigned short __readgsword(const unsigned long Offset)
657 {
658 unsigned short value;
659 __asm__ __volatile__("movw %%gs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
660 return value;
661 }
662
663 __INTRIN_INLINE unsigned long __readgsdword(const unsigned long Offset)
664 {
665 unsigned long value;
666 __asm__ __volatile__("movl %%gs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
667 return value;
668 }
669
670 __INTRIN_INLINE unsigned __int64 __readgsqword(const unsigned long Offset)
671 {
672 unsigned __int64 value;
673 __asm__ __volatile__("movq %%gs:%a[Offset], %q[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
674 return value;
675 }
676
677 __INTRIN_INLINE void __incgsbyte(const unsigned long Offset)
678 {
679 __asm__ __volatile__("incb %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
680 }
681
682 __INTRIN_INLINE void __incgsword(const unsigned long Offset)
683 {
684 __asm__ __volatile__("incw %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
685 }
686
687 __INTRIN_INLINE void __incgsdword(const unsigned long Offset)
688 {
689 __asm__ __volatile__("incl %%gs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
690 }
691
692 __INTRIN_INLINE void __addgsbyte(const unsigned long Offset, const unsigned char Data)
693 {
694 __asm__ __volatile__("addb %b[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
695 }
696
697 __INTRIN_INLINE void __addgsword(const unsigned long Offset, const unsigned short Data)
698 {
699 __asm__ __volatile__("addw %w[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
700 }
701
702 __INTRIN_INLINE void __addgsdword(const unsigned long Offset, const unsigned int Data)
703 {
704 __asm__ __volatile__("addl %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
705 }
706
707 __INTRIN_INLINE void __addgsqword(const unsigned long Offset, const unsigned __int64 Data)
708 {
709 __asm__ __volatile__("addq %k[Data], %%gs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
710 }
711
712 #else
713 /*** FS segment addressing ***/
714 __INTRIN_INLINE void __writefsbyte(const unsigned long Offset, const unsigned char Data)
715 {
716 __asm__ __volatile__("movb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
717 }
718
719 __INTRIN_INLINE void __writefsword(const unsigned long Offset, const unsigned short Data)
720 {
721 __asm__ __volatile__("movw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
722 }
723
724 __INTRIN_INLINE void __writefsdword(const unsigned long Offset, const unsigned long Data)
725 {
726 __asm__ __volatile__("movl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "ir" (Data) : "memory");
727 }
728
729 __INTRIN_INLINE unsigned char __readfsbyte(const unsigned long Offset)
730 {
731 unsigned char value;
732 __asm__ __volatile__("movb %%fs:%a[Offset], %b[value]" : [value] "=q" (value) : [Offset] "ir" (Offset));
733 return value;
734 }
735
736 __INTRIN_INLINE unsigned short __readfsword(const unsigned long Offset)
737 {
738 unsigned short value;
739 __asm__ __volatile__("movw %%fs:%a[Offset], %w[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
740 return value;
741 }
742
743 __INTRIN_INLINE unsigned long __readfsdword(const unsigned long Offset)
744 {
745 unsigned long value;
746 __asm__ __volatile__("movl %%fs:%a[Offset], %k[value]" : [value] "=r" (value) : [Offset] "ir" (Offset));
747 return value;
748 }
749
750 __INTRIN_INLINE void __incfsbyte(const unsigned long Offset)
751 {
752 __asm__ __volatile__("incb %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
753 }
754
755 __INTRIN_INLINE void __incfsword(const unsigned long Offset)
756 {
757 __asm__ __volatile__("incw %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
758 }
759
760 __INTRIN_INLINE void __incfsdword(const unsigned long Offset)
761 {
762 __asm__ __volatile__("incl %%fs:%a[Offset]" : : [Offset] "ir" (Offset) : "memory");
763 }
764
765 /* NOTE: the bizarre implementation of __addfsxxx mimics the broken Visual C++ behavior */
766 __INTRIN_INLINE void __addfsbyte(const unsigned long Offset, const unsigned char Data)
767 {
768 if(!__builtin_constant_p(Offset))
769 __asm__ __volatile__("addb %b[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
770 else
771 __asm__ __volatile__("addb %b[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
772 }
773
774 __INTRIN_INLINE void __addfsword(const unsigned long Offset, const unsigned short Data)
775 {
776 if(!__builtin_constant_p(Offset))
777 __asm__ __volatile__("addw %w[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
778 else
779 __asm__ __volatile__("addw %w[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
780 }
781
782 __INTRIN_INLINE void __addfsdword(const unsigned long Offset, const unsigned int Data)
783 {
784 if(!__builtin_constant_p(Offset))
785 __asm__ __volatile__("addl %k[Offset], %%fs:%a[Offset]" : : [Offset] "r" (Offset) : "memory");
786 else
787 __asm__ __volatile__("addl %k[Data], %%fs:%a[Offset]" : : [Offset] "ir" (Offset), [Data] "iq" (Data) : "memory");
788 }
789 #endif
790
791
792 /*** Bit manipulation ***/
793 __INTRIN_INLINE unsigned char _BitScanForward(unsigned long * const Index, const unsigned long Mask)
794 {
795 __asm__("bsfl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
796 return Mask ? 1 : 0;
797 }
798
799 __INTRIN_INLINE unsigned char _BitScanReverse(unsigned long * const Index, const unsigned long Mask)
800 {
801 __asm__("bsrl %[Mask], %[Index]" : [Index] "=r" (*Index) : [Mask] "mr" (Mask));
802 return Mask ? 1 : 0;
803 }
804
805 /* NOTE: again, the bizarre implementation follows Visual C++ */
806 __INTRIN_INLINE unsigned char _bittest(const long * const a, const long b)
807 {
808 unsigned char retval;
809
810 if(__builtin_constant_p(b))
811 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 32))), [b] "Ir" (b % 32));
812 else
813 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*a), [b] "r" (b));
814
815 return retval;
816 }
817
818 #ifdef _M_AMD64
819 __INTRIN_INLINE unsigned char _bittest64(const __int64 * const a, const __int64 b)
820 {
821 unsigned char retval;
822
823 if(__builtin_constant_p(b))
824 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*(a + (b / 64))), [b] "Ir" (b % 64));
825 else
826 __asm__("bt %[b], %[a]; setb %b[retval]" : [retval] "=q" (retval) : [a] "mr" (*a), [b] "r" (b));
827
828 return retval;
829 }
830 #endif
831
832 __INTRIN_INLINE unsigned char _bittestandcomplement(long * const a, const long b)
833 {
834 unsigned char retval;
835
836 if(__builtin_constant_p(b))
837 __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
838 else
839 __asm__("btc %[b], %[a]; setb %b[retval]" : [a] "+mr" (*a), [retval] "=q" (retval) : [b] "r" (b));
840
841 return retval;
842 }
843
844 __INTRIN_INLINE unsigned char _bittestandreset(long * const a, const long b)
845 {
846 unsigned char retval;
847
848 if(__builtin_constant_p(b))
849 __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
850 else
851 __asm__("btr %[b], %[a]; setb %b[retval]" : [a] "+mr" (*a), [retval] "=q" (retval) : [b] "r" (b));
852
853 return retval;
854 }
855
856 __INTRIN_INLINE unsigned char _bittestandset(long * const a, const long b)
857 {
858 unsigned char retval;
859
860 if(__builtin_constant_p(b))
861 __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+mr" (*(a + (b / 32))), [retval] "=q" (retval) : [b] "Ir" (b % 32));
862 else
863 __asm__("bts %[b], %[a]; setb %b[retval]" : [a] "+mr" (*a), [retval] "=q" (retval) : [b] "r" (b));
864
865 return retval;
866 }
867
868 __INTRIN_INLINE unsigned char _rotl8(unsigned char value, unsigned char shift)
869 {
870 unsigned char retval;
871 __asm__("rolb %b[shift], %b[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
872 return retval;
873 }
874
875 __INTRIN_INLINE unsigned short _rotl16(unsigned short value, unsigned char shift)
876 {
877 unsigned short retval;
878 __asm__("rolw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
879 return retval;
880 }
881
882 __INTRIN_INLINE unsigned int _rotl(unsigned int value, int shift)
883 {
884 unsigned long retval;
885 __asm__("roll %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
886 return retval;
887 }
888
889 __INTRIN_INLINE unsigned int _rotr(unsigned int value, int shift)
890 {
891 unsigned long retval;
892 __asm__("rorl %b[shift], %k[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
893 return retval;
894 }
895
896 __INTRIN_INLINE unsigned char _rotr8(unsigned char value, unsigned char shift)
897 {
898 unsigned char retval;
899 __asm__("rorb %b[shift], %b[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
900 return retval;
901 }
902
903 __INTRIN_INLINE unsigned short _rotr16(unsigned short value, unsigned char shift)
904 {
905 unsigned short retval;
906 __asm__("rorw %b[shift], %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value), [shift] "Nc" (shift));
907 return retval;
908 }
909
910 /*
911 NOTE: in __ll_lshift, __ll_rshift and __ull_rshift we use the "A"
912 constraint (edx:eax) for the Mask argument, because it's the only way GCC
913 can pass 64-bit operands around - passing the two 32 bit parts separately
914 just confuses it. Also we declare Bit as an int and then truncate it to
915 match Visual C++ behavior
916 */
917 __INTRIN_INLINE unsigned long long __ll_lshift(const unsigned long long Mask, const int Bit)
918 {
919 unsigned long long retval = Mask;
920
921 __asm__
922 (
923 "shldl %b[Bit], %%eax, %%edx; sall %b[Bit], %%eax" :
924 "+A" (retval) :
925 [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
926 );
927
928 return retval;
929 }
930
931 __INTRIN_INLINE long long __ll_rshift(const long long Mask, const int Bit)
932 {
933 unsigned long long retval = Mask;
934
935 __asm__
936 (
937 "shldl %b[Bit], %%eax, %%edx; sarl %b[Bit], %%eax" :
938 "+A" (retval) :
939 [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
940 );
941
942 return retval;
943 }
944
945 __INTRIN_INLINE unsigned long long __ull_rshift(const unsigned long long Mask, int Bit)
946 {
947 unsigned long long retval = Mask;
948
949 __asm__
950 (
951 "shrdl %b[Bit], %%eax, %%edx; shrl %b[Bit], %%eax" :
952 "+A" (retval) :
953 [Bit] "Nc" ((unsigned char)((unsigned long)Bit) & 0xFF)
954 );
955
956 return retval;
957 }
958
959 __INTRIN_INLINE unsigned short _byteswap_ushort(unsigned short value)
960 {
961 unsigned short retval;
962 __asm__("rorw $8, %w[retval]" : [retval] "=rm" (retval) : "[retval]" (value));
963 return retval;
964 }
965
966 __INTRIN_INLINE unsigned long _byteswap_ulong(unsigned long value)
967 {
968 unsigned long retval;
969 __asm__("bswapl %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
970 return retval;
971 }
972
973 #ifdef _M_AMD64
974 __INTRIN_INLINE unsigned __int64 _byteswap_uint64(unsigned __int64 value)
975 {
976 unsigned __int64 retval;
977 __asm__("bswapq %[retval]" : [retval] "=r" (retval) : "[retval]" (value));
978 return retval;
979 }
980 #else
981 __INTRIN_INLINE unsigned __int64 _byteswap_uint64(unsigned __int64 value)
982 {
983 union {
984 __int64 int64part;
985 struct {
986 unsigned long lowpart;
987 unsigned long hipart;
988 };
989 } retval;
990 retval.int64part = value;
991 __asm__("bswapl %[lowpart]\n"
992 "bswapl %[hipart]\n"
993 : [lowpart] "=r" (retval.hipart), [hipart] "=r" (retval.lowpart) : "[lowpart]" (retval.lowpart), "[hipart]" (retval.hipart) );
994 return retval.int64part;
995 }
996 #endif
997
998 /*** 64-bit math ***/
999 __INTRIN_INLINE long long __emul(const int a, const int b)
1000 {
1001 long long retval;
1002 __asm__("imull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1003 return retval;
1004 }
1005
1006 __INTRIN_INLINE unsigned long long __emulu(const unsigned int a, const unsigned int b)
1007 {
1008 unsigned long long retval;
1009 __asm__("mull %[b]" : "=A" (retval) : [a] "a" (a), [b] "rm" (b));
1010 return retval;
1011 }
1012
1013 #ifdef _M_AMD64
1014
1015 __INTRIN_INLINE __int64 __mulh(__int64 a, __int64 b)
1016 {
1017 __int64 retval;
1018 __asm__("imulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1019 return retval;
1020 }
1021
1022 __INTRIN_INLINE unsigned __int64 __umulh(unsigned __int64 a, unsigned __int64 b)
1023 {
1024 unsigned __int64 retval;
1025 __asm__("mulq %[b]" : "=d" (retval) : [a] "a" (a), [b] "rm" (b));
1026 return retval;
1027 }
1028
1029 #endif
1030
1031 /*** Port I/O ***/
1032 __INTRIN_INLINE unsigned char __inbyte(const unsigned short Port)
1033 {
1034 unsigned char byte;
1035 __asm__ __volatile__("inb %w[Port], %b[byte]" : [byte] "=a" (byte) : [Port] "Nd" (Port));
1036 return byte;
1037 }
1038
1039 __INTRIN_INLINE unsigned short __inword(const unsigned short Port)
1040 {
1041 unsigned short word;
1042 __asm__ __volatile__("inw %w[Port], %w[word]" : [word] "=a" (word) : [Port] "Nd" (Port));
1043 return word;
1044 }
1045
1046 __INTRIN_INLINE unsigned long __indword(const unsigned short Port)
1047 {
1048 unsigned long dword;
1049 __asm__ __volatile__("inl %w[Port], %k[dword]" : [dword] "=a" (dword) : [Port] "Nd" (Port));
1050 return dword;
1051 }
1052
1053 __INTRIN_INLINE void __inbytestring(unsigned short Port, unsigned char * Buffer, unsigned long Count)
1054 {
1055 __asm__ __volatile__
1056 (
1057 "rep; insb" :
1058 [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1059 "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1060 "memory"
1061 );
1062 }
1063
1064 __INTRIN_INLINE void __inwordstring(unsigned short Port, unsigned short * Buffer, unsigned long Count)
1065 {
1066 __asm__ __volatile__
1067 (
1068 "rep; insw" :
1069 [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1070 "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1071 "memory"
1072 );
1073 }
1074
1075 __INTRIN_INLINE void __indwordstring(unsigned short Port, unsigned long * Buffer, unsigned long Count)
1076 {
1077 __asm__ __volatile__
1078 (
1079 "rep; insl" :
1080 [Buffer] "=D" (Buffer), [Count] "=c" (Count) :
1081 "d" (Port), "[Buffer]" (Buffer), "[Count]" (Count) :
1082 "memory"
1083 );
1084 }
1085
1086 __INTRIN_INLINE void __outbyte(unsigned short const Port, const unsigned char Data)
1087 {
1088 __asm__ __volatile__("outb %b[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1089 }
1090
1091 __INTRIN_INLINE void __outword(unsigned short const Port, const unsigned short Data)
1092 {
1093 __asm__ __volatile__("outw %w[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1094 }
1095
1096 __INTRIN_INLINE void __outdword(unsigned short const Port, const unsigned long Data)
1097 {
1098 __asm__ __volatile__("outl %k[Data], %w[Port]" : : [Port] "Nd" (Port), [Data] "a" (Data));
1099 }
1100
1101 __INTRIN_INLINE void __outbytestring(unsigned short const Port, const unsigned char * const Buffer, const unsigned long Count)
1102 {
1103 __asm__ __volatile__("rep; outsb" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1104 }
1105
1106 __INTRIN_INLINE void __outwordstring(unsigned short const Port, const unsigned short * const Buffer, const unsigned long Count)
1107 {
1108 __asm__ __volatile__("rep; outsw" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1109 }
1110
1111 __INTRIN_INLINE void __outdwordstring(unsigned short const Port, const unsigned long * const Buffer, const unsigned long Count)
1112 {
1113 __asm__ __volatile__("rep; outsl" : : [Port] "d" (Port), [Buffer] "S" (Buffer), "c" (Count));
1114 }
1115
1116 __INTRIN_INLINE int _inp(unsigned short Port)
1117 {
1118 return __inbyte(Port);
1119 }
1120
1121 __INTRIN_INLINE unsigned short _inpw(unsigned short Port)
1122 {
1123 return __inword(Port);
1124 }
1125
1126 __INTRIN_INLINE unsigned long _inpd(unsigned short Port)
1127 {
1128 return __indword(Port);
1129 }
1130
1131 __INTRIN_INLINE int _outp(unsigned short Port, int databyte)
1132 {
1133 __outbyte(Port, databyte);
1134 return databyte;
1135 }
1136
1137 __INTRIN_INLINE unsigned short _outpw(unsigned short Port, unsigned short dataword)
1138 {
1139 __outword(Port, dataword);
1140 return dataword;
1141 }
1142
1143 __INTRIN_INLINE unsigned long _outpd(unsigned short Port, unsigned long dataword)
1144 {
1145 __outdword(Port, dataword);
1146 return dataword;
1147 }
1148
1149
1150 /*** System information ***/
1151 __INTRIN_INLINE void __cpuid(int CPUInfo[], const int InfoType)
1152 {
1153 __asm__ __volatile__("cpuid" : "=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3]) : "a" (InfoType));
1154 }
1155
1156 __INTRIN_INLINE unsigned long long __rdtsc(void)
1157 {
1158 #ifdef _M_AMD64
1159 unsigned long long low, high;
1160 __asm__ __volatile__("rdtsc" : "=a"(low), "=d"(high));
1161 return low | (high << 32);
1162 #else
1163 unsigned long long retval;
1164 __asm__ __volatile__("rdtsc" : "=A"(retval));
1165 return retval;
1166 #endif
1167 }
1168
1169 __INTRIN_INLINE void __writeeflags(uintptr_t Value)
1170 {
1171 __asm__ __volatile__("push %0\n popf" : : "rim"(Value));
1172 }
1173
1174 __INTRIN_INLINE uintptr_t __readeflags(void)
1175 {
1176 uintptr_t retval;
1177 __asm__ __volatile__("pushf\n pop %0" : "=rm"(retval));
1178 return retval;
1179 }
1180
1181 /*** Interrupts ***/
1182 __INTRIN_INLINE void __debugbreak(void)
1183 {
1184 __asm__("int $3");
1185 }
1186
1187 __INTRIN_INLINE void __int2c(void)
1188 {
1189 __asm__("int $0x2c");
1190 }
1191
1192 __INTRIN_INLINE void _disable(void)
1193 {
1194 __asm__("cli");
1195 }
1196
1197 __INTRIN_INLINE void _enable(void)
1198 {
1199 __asm__("sti");
1200 }
1201
1202 __INTRIN_INLINE void __halt(void)
1203 {
1204 __asm__("hlt\n\t");
1205 }
1206
1207 /*** Protected memory management ***/
1208
1209 __INTRIN_INLINE void __writecr0(const unsigned __int64 Data)
1210 {
1211 __asm__("mov %[Data], %%cr0" : : [Data] "r" (Data) : "memory");
1212 }
1213
1214 __INTRIN_INLINE void __writecr3(const unsigned __int64 Data)
1215 {
1216 __asm__("mov %[Data], %%cr3" : : [Data] "r" (Data) : "memory");
1217 }
1218
1219 __INTRIN_INLINE void __writecr4(const unsigned __int64 Data)
1220 {
1221 __asm__("mov %[Data], %%cr4" : : [Data] "r" (Data) : "memory");
1222 }
1223
1224 #ifdef _M_AMD64
1225 __INTRIN_INLINE void __writecr8(const unsigned __int64 Data)
1226 {
1227 __asm__("mov %[Data], %%cr8" : : [Data] "r" (Data) : "memory");
1228 }
1229
1230 __INTRIN_INLINE unsigned __int64 __readcr0(void)
1231 {
1232 unsigned __int64 value;
1233 __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1234 return value;
1235 }
1236
1237 __INTRIN_INLINE unsigned __int64 __readcr2(void)
1238 {
1239 unsigned __int64 value;
1240 __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1241 return value;
1242 }
1243
1244 __INTRIN_INLINE unsigned __int64 __readcr3(void)
1245 {
1246 unsigned __int64 value;
1247 __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1248 return value;
1249 }
1250
1251 __INTRIN_INLINE unsigned __int64 __readcr4(void)
1252 {
1253 unsigned __int64 value;
1254 __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1255 return value;
1256 }
1257
1258 __INTRIN_INLINE unsigned __int64 __readcr8(void)
1259 {
1260 unsigned __int64 value;
1261 __asm__ __volatile__("movq %%cr8, %q[value]" : [value] "=r" (value));
1262 return value;
1263 }
1264 #else
1265 __INTRIN_INLINE unsigned long __readcr0(void)
1266 {
1267 unsigned long value;
1268 __asm__ __volatile__("mov %%cr0, %[value]" : [value] "=r" (value));
1269 return value;
1270 }
1271
1272 __INTRIN_INLINE unsigned long __readcr2(void)
1273 {
1274 unsigned long value;
1275 __asm__ __volatile__("mov %%cr2, %[value]" : [value] "=r" (value));
1276 return value;
1277 }
1278
1279 __INTRIN_INLINE unsigned long __readcr3(void)
1280 {
1281 unsigned long value;
1282 __asm__ __volatile__("mov %%cr3, %[value]" : [value] "=r" (value));
1283 return value;
1284 }
1285
1286 __INTRIN_INLINE unsigned long __readcr4(void)
1287 {
1288 unsigned long value;
1289 __asm__ __volatile__("mov %%cr4, %[value]" : [value] "=r" (value));
1290 return value;
1291 }
1292 #endif
1293
1294 #ifdef _M_AMD64
1295 __INTRIN_INLINE unsigned __int64 __readdr(unsigned int reg)
1296 {
1297 unsigned __int64 value;
1298 switch (reg)
1299 {
1300 case 0:
1301 __asm__ __volatile__("movq %%dr0, %q[value]" : [value] "=r" (value));
1302 break;
1303 case 1:
1304 __asm__ __volatile__("movq %%dr1, %q[value]" : [value] "=r" (value));
1305 break;
1306 case 2:
1307 __asm__ __volatile__("movq %%dr2, %q[value]" : [value] "=r" (value));
1308 break;
1309 case 3:
1310 __asm__ __volatile__("movq %%dr3, %q[value]" : [value] "=r" (value));
1311 break;
1312 case 4:
1313 __asm__ __volatile__("movq %%dr4, %q[value]" : [value] "=r" (value));
1314 break;
1315 case 5:
1316 __asm__ __volatile__("movq %%dr5, %q[value]" : [value] "=r" (value));
1317 break;
1318 case 6:
1319 __asm__ __volatile__("movq %%dr6, %q[value]" : [value] "=r" (value));
1320 break;
1321 case 7:
1322 __asm__ __volatile__("movq %%dr7, %q[value]" : [value] "=r" (value));
1323 break;
1324 }
1325 return value;
1326 }
1327
1328 __INTRIN_INLINE void __writedr(unsigned reg, unsigned __int64 value)
1329 {
1330 switch (reg)
1331 {
1332 case 0:
1333 __asm__("movq %q[value], %%dr0" : : [value] "r" (value) : "memory");
1334 break;
1335 case 1:
1336 __asm__("movq %q[value], %%dr1" : : [value] "r" (value) : "memory");
1337 break;
1338 case 2:
1339 __asm__("movq %q[value], %%dr2" : : [value] "r" (value) : "memory");
1340 break;
1341 case 3:
1342 __asm__("movq %q[value], %%dr3" : : [value] "r" (value) : "memory");
1343 break;
1344 case 4:
1345 __asm__("movq %q[value], %%dr4" : : [value] "r" (value) : "memory");
1346 break;
1347 case 5:
1348 __asm__("movq %q[value], %%dr5" : : [value] "r" (value) : "memory");
1349 break;
1350 case 6:
1351 __asm__("movq %q[value], %%dr6" : : [value] "r" (value) : "memory");
1352 break;
1353 case 7:
1354 __asm__("movq %q[value], %%dr7" : : [value] "r" (value) : "memory");
1355 break;
1356 }
1357 }
1358 #else
1359 __INTRIN_INLINE unsigned int __readdr(unsigned int reg)
1360 {
1361 unsigned int value;
1362 switch (reg)
1363 {
1364 case 0:
1365 __asm__ __volatile__("mov %%dr0, %[value]" : [value] "=r" (value));
1366 break;
1367 case 1:
1368 __asm__ __volatile__("mov %%dr1, %[value]" : [value] "=r" (value));
1369 break;
1370 case 2:
1371 __asm__ __volatile__("mov %%dr2, %[value]" : [value] "=r" (value));
1372 break;
1373 case 3:
1374 __asm__ __volatile__("mov %%dr3, %[value]" : [value] "=r" (value));
1375 break;
1376 case 4:
1377 __asm__ __volatile__("mov %%dr4, %[value]" : [value] "=r" (value));
1378 break;
1379 case 5:
1380 __asm__ __volatile__("mov %%dr5, %[value]" : [value] "=r" (value));
1381 break;
1382 case 6:
1383 __asm__ __volatile__("mov %%dr6, %[value]" : [value] "=r" (value));
1384 break;
1385 case 7:
1386 __asm__ __volatile__("mov %%dr7, %[value]" : [value] "=r" (value));
1387 break;
1388 }
1389 return value;
1390 }
1391
1392 __INTRIN_INLINE void __writedr(unsigned reg, unsigned int value)
1393 {
1394 switch (reg)
1395 {
1396 case 0:
1397 __asm__("mov %[value], %%dr0" : : [value] "r" (value) : "memory");
1398 break;
1399 case 1:
1400 __asm__("mov %[value], %%dr1" : : [value] "r" (value) : "memory");
1401 break;
1402 case 2:
1403 __asm__("mov %[value], %%dr2" : : [value] "r" (value) : "memory");
1404 break;
1405 case 3:
1406 __asm__("mov %[value], %%dr3" : : [value] "r" (value) : "memory");
1407 break;
1408 case 4:
1409 __asm__("mov %[value], %%dr4" : : [value] "r" (value) : "memory");
1410 break;
1411 case 5:
1412 __asm__("mov %[value], %%dr5" : : [value] "r" (value) : "memory");
1413 break;
1414 case 6:
1415 __asm__("mov %[value], %%dr6" : : [value] "r" (value) : "memory");
1416 break;
1417 case 7:
1418 __asm__("mov %[value], %%dr7" : : [value] "r" (value) : "memory");
1419 break;
1420 }
1421 }
1422 #endif
1423
1424 __INTRIN_INLINE void __invlpg(void * const Address)
1425 {
1426 __asm__("invlpg %[Address]" : : [Address] "m" (*((unsigned char *)(Address))));
1427 }
1428
1429
1430 /*** System operations ***/
1431 __INTRIN_INLINE unsigned long long __readmsr(const int reg)
1432 {
1433 #ifdef _M_AMD64
1434 unsigned long low, high;
1435 __asm__ __volatile__("rdmsr" : "=a" (low), "=d" (high) : "c" (reg));
1436 return ((unsigned long long)high << 32) | low;
1437 #else
1438 unsigned long long retval;
1439 __asm__ __volatile__("rdmsr" : "=A" (retval) : "c" (reg));
1440 return retval;
1441 #endif
1442 }
1443
1444 __INTRIN_INLINE void __writemsr(const unsigned long Register, const unsigned long long Value)
1445 {
1446 #ifdef _M_AMD64
1447 __asm__ __volatile__("wrmsr" : : "a" (Value), "d" (Value >> 32), "c" (Register));
1448 #else
1449 __asm__ __volatile__("wrmsr" : : "A" (Value), "c" (Register));
1450 #endif
1451 }
1452
1453 __INTRIN_INLINE unsigned long long __readpmc(const int counter)
1454 {
1455 unsigned long long retval;
1456 __asm__ __volatile__("rdpmc" : "=A" (retval) : "c" (counter));
1457 return retval;
1458 }
1459
1460 /* NOTE: an immediate value for 'a' will raise an ICE in Visual C++ */
1461 __INTRIN_INLINE unsigned long __segmentlimit(const unsigned long a)
1462 {
1463 unsigned long retval;
1464 __asm__ __volatile__("lsl %[a], %[retval]" : [retval] "=r" (retval) : [a] "rm" (a));
1465 return retval;
1466 }
1467
1468 __INTRIN_INLINE void __wbinvd(void)
1469 {
1470 __asm__ __volatile__("wbinvd");
1471 }
1472
1473 __INTRIN_INLINE void __lidt(void *Source)
1474 {
1475 __asm__ __volatile__("lidt %0" : : "m"(*(short*)Source));
1476 }
1477
1478 __INTRIN_INLINE void __sidt(void *Destination)
1479 {
1480 __asm__ __volatile__("sidt %0" : : "m"(*(short*)Destination) : "memory");
1481 }
1482
1483 __INTRIN_INLINE void _mm_pause(void)
1484 {
1485 __asm__ __volatile__("pause");
1486 }
1487
1488 #ifdef __cplusplus
1489 }
1490 #endif
1491
1492 #endif /* KJK_INTRIN_X86_H_ */
1493
1494 /* EOF */