2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/i386/cpu.c
5 * PURPOSE: Routines for CPU-level support
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
9 /* INCLUDES *****************************************************************/
15 /* GLOBALS *******************************************************************/
17 /* The TSS to use for Double Fault Traps (INT 0x9) */
18 UCHAR KiDoubleFaultTSS
[KTSS_IO_MAPS
];
20 /* The TSS to use for NMI Fault Traps (INT 0x2) */
21 UCHAR KiNMITSS
[KTSS_IO_MAPS
];
23 /* CPU Features and Flags */
26 ULONG KeProcessorArchitecture
;
27 ULONG KeProcessorLevel
;
28 ULONG KeProcessorRevision
;
30 ULONG KiFastSystemCallDisable
;
31 ULONG KeI386NpxPresent
= 0;
32 ULONG KiMXCsrMask
= 0;
33 ULONG MxcsrFeatureMask
= 0;
34 ULONG KeI386XMMIPresent
= 0;
35 ULONG KeI386FxsrPresent
= 0;
36 ULONG KeI386MachineType
;
37 ULONG Ke386Pae
= FALSE
;
38 ULONG Ke386NoExecute
= FALSE
;
39 ULONG KeLargestCacheLine
= 0x40;
40 ULONG KeDcacheFlushCount
= 0;
41 ULONG KeIcacheFlushCount
= 0;
42 ULONG KiDmaIoCoherency
= 0;
43 ULONG KePrefetchNTAGranularity
= 32;
44 CHAR KeNumberProcessors
;
45 KAFFINITY KeActiveProcessors
= 1;
46 BOOLEAN KiI386PentiumLockErrataPresent
;
47 BOOLEAN KiSMTProcessorsPresent
;
49 /* The distance between SYSEXIT and IRETD return modes */
50 UCHAR KiSystemCallExitAdjust
;
52 /* The offset that was applied -- either 0 or the value above */
53 UCHAR KiSystemCallExitAdjusted
;
55 /* Whether the adjustment was already done once */
56 BOOLEAN KiFastCallCopyDoneOnce
;
59 volatile LONG KiTbFlushTimeStamp
;
62 static const CHAR CmpIntelID
[] = "GenuineIntel";
63 static const CHAR CmpAmdID
[] = "AuthenticAMD";
64 static const CHAR CmpCyrixID
[] = "CyrixInstead";
65 static const CHAR CmpTransmetaID
[] = "GenuineTMx86";
66 static const CHAR CmpCentaurID
[] = "CentaurHauls";
67 static const CHAR CmpRiseID
[] = "RiseRiseRise";
69 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
73 CPUID(IN ULONG InfoType
,
74 OUT PULONG CpuInfoEax
,
75 OUT PULONG CpuInfoEbx
,
76 OUT PULONG CpuInfoEcx
,
77 OUT PULONG CpuInfoEdx
)
81 /* Perform the CPUID Operation */
82 __cpuid((int*)CpuInfo
, InfoType
);
84 /* Return the results */
85 *CpuInfoEax
= CpuInfo
[0];
86 *CpuInfoEbx
= CpuInfo
[1];
87 *CpuInfoEcx
= CpuInfo
[2];
88 *CpuInfoEdx
= CpuInfo
[3];
93 WRMSR(IN ULONG Register
,
96 /* Write to the MSR */
97 __writemsr(Register
, Value
);
102 RDMSR(IN ULONG Register
)
104 /* Read from the MSR */
105 return __readmsr(Register
);
108 /* FUNCTIONS *****************************************************************/
112 KiSetProcessorType(VOID
)
114 ULONG EFlags
, NewEFlags
;
116 ULONG Stepping
, Type
;
118 /* Start by assuming no CPUID data */
119 KeGetCurrentPrcb()->CpuID
= 0;
122 EFlags
= __readeflags();
124 /* XOR out the ID bit and update EFlags */
125 NewEFlags
= EFlags
^ EFLAGS_ID
;
126 __writeeflags(NewEFlags
);
128 /* Get them back and see if they were modified */
129 NewEFlags
= __readeflags();
130 if (NewEFlags
!= EFlags
)
132 /* The modification worked, so CPUID exists. Set the ID Bit again. */
134 __writeeflags(EFlags
);
136 /* Peform CPUID 0 to see if CPUID 1 is supported */
137 CPUID(0, &Reg
, &Dummy
, &Dummy
, &Dummy
);
141 CPUID(1, &Reg
, &Dummy
, &Dummy
, &Dummy
);
144 * Get the Stepping and Type. The stepping contains both the
145 * Model and the Step, while the Type contains the returned Type.
146 * We ignore the family.
148 * For the stepping, we convert this: zzzzzzxy into this: x0y
150 Stepping
= Reg
& 0xF0;
152 Stepping
+= (Reg
& 0xFF);
157 /* Save them in the PRCB */
158 KeGetCurrentPrcb()->CpuID
= TRUE
;
159 KeGetCurrentPrcb()->CpuType
= (UCHAR
)Type
;
160 KeGetCurrentPrcb()->CpuStep
= (USHORT
)Stepping
;
164 DPRINT1("CPUID Support lacking\n");
169 DPRINT1("CPUID Support lacking\n");
173 __writeeflags(EFlags
);
180 PKPRCB Prcb
= KeGetCurrentPrcb();
184 /* Assume no Vendor ID and fail if no CPUID Support. */
185 Prcb
->VendorString
[0] = 0;
186 if (!Prcb
->CpuID
) return 0;
188 /* Get the Vendor ID and null-terminate it */
189 CPUID(0, &Vendor
[0], &Vendor
[1], &Vendor
[2], &Vendor
[3]);
192 /* Re-arrange vendor string */
194 Vendor
[2] = Vendor
[3];
197 /* Copy it to the PRCB and null-terminate it again */
198 RtlCopyMemory(Prcb
->VendorString
,
200 sizeof(Prcb
->VendorString
) - sizeof(CHAR
));
201 Prcb
->VendorString
[sizeof(Prcb
->VendorString
) - sizeof(CHAR
)] = ANSI_NULL
;
203 /* Now check the CPU Type */
204 if (!strcmp(Prcb
->VendorString
, CmpIntelID
))
208 else if (!strcmp(Prcb
->VendorString
, CmpAmdID
))
212 else if (!strcmp(Prcb
->VendorString
, CmpCyrixID
))
214 DPRINT1("Cyrix CPU support not fully tested!\n");
217 else if (!strcmp(Prcb
->VendorString
, CmpTransmetaID
))
219 DPRINT1("Transmeta CPU support not fully tested!\n");
220 return CPU_TRANSMETA
;
222 else if (!strcmp(Prcb
->VendorString
, CmpCentaurID
))
224 DPRINT1("Centaur CPU support not fully tested!\n");
227 else if (!strcmp(Prcb
->VendorString
, CmpRiseID
))
229 DPRINT1("Rise CPU support not fully tested!\n");
239 KiGetFeatureBits(VOID
)
241 PKPRCB Prcb
= KeGetCurrentPrcb();
243 ULONG FeatureBits
= KF_WORKING_PTE
;
245 BOOLEAN ExtendedCPUID
= TRUE
;
246 ULONG CpuFeatures
= 0;
248 /* Get the Vendor ID */
249 Vendor
= KiGetCpuVendor();
251 /* Make sure we got a valid vendor ID at least. */
252 if (!Vendor
) return FeatureBits
;
254 /* Get the CPUID Info. Features are in Reg[3]. */
255 CPUID(1, &Reg
[0], &Reg
[1], &Dummy
, &Reg
[3]);
257 /* Set the initial APIC ID */
258 Prcb
->InitialApicId
= (UCHAR
)(Reg
[1] >> 24);
265 /* Check if it's a P6 */
266 if (Prcb
->CpuType
== 6)
268 /* Perform the special sequence to get the MicroCode Signature */
270 CPUID(1, &Dummy
, &Dummy
, &Dummy
, &Dummy
);
271 Prcb
->UpdateSignature
.QuadPart
= RDMSR(0x8B);
273 else if (Prcb
->CpuType
== 5)
275 /* On P5, enable workaround for the LOCK errata. */
276 KiI386PentiumLockErrataPresent
= TRUE
;
279 /* Check for broken P6 with bad SMP PTE implementation */
280 if (((Reg
[0] & 0x0FF0) == 0x0610 && (Reg
[0] & 0x000F) <= 0x9) ||
281 ((Reg
[0] & 0x0FF0) == 0x0630 && (Reg
[0] & 0x000F) <= 0x4))
283 /* Remove support for correct PTE support. */
284 FeatureBits
&= ~KF_WORKING_PTE
;
287 /* Check if the CPU is too old to support SYSENTER */
288 if ((Prcb
->CpuType
< 6) ||
289 ((Prcb
->CpuType
== 6) && (Prcb
->CpuStep
< 0x0303)))
295 /* Set the current features */
296 CpuFeatures
= Reg
[3];
303 /* Check if this is a K5 or K6. (family 5) */
304 if ((Reg
[0] & 0x0F00) == 0x0500)
306 /* Get the Model Number */
307 switch (Reg
[0] & 0x00F0)
309 /* Model 1: K5 - 5k86 (initial models) */
312 /* Check if this is Step 0 or 1. They don't support PGE */
313 if ((Reg
[0] & 0x000F) > 0x03) break;
315 /* Model 0: K5 - SSA5 */
318 /* Model 0 doesn't support PGE at all. */
325 /* K6-2, Step 8 and over have support for MTRR. */
326 if ((Reg
[0] & 0x000F) >= 0x8) FeatureBits
|= KF_AMDK6MTRR
;
330 Model D: K6-2+, K6-III+ */
334 FeatureBits
|= KF_AMDK6MTRR
;
338 else if((Reg
[0] & 0x0F00) < 0x0500)
340 /* Families below 5 don't support PGE, PSE or CMOV at all */
341 Reg
[3] &= ~(0x08 | 0x2000 | 0x8000);
343 /* They also don't support advanced CPUID functions. */
344 ExtendedCPUID
= FALSE
;
347 /* Set the current features */
348 CpuFeatures
= Reg
[3];
355 /* FIXME: CMPXCGH8B */
362 /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
363 if ((Reg
[0] & 0x0FFF) >= 0x0542)
365 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
366 FeatureBits
|= KF_CMPXCHG8B
;
371 /* Centaur, IDT, Rise and VIA CPUs */
375 /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
376 However, this feature exists and operates properly without any additional steps. */
377 FeatureBits
|= KF_CMPXCHG8B
;
382 /* Convert all CPUID Feature bits into our format */
383 if (CpuFeatures
& 0x00000002) FeatureBits
|= KF_V86_VIS
| KF_CR4
;
384 if (CpuFeatures
& 0x00000008) FeatureBits
|= KF_LARGE_PAGE
| KF_CR4
;
385 if (CpuFeatures
& 0x00000010) FeatureBits
|= KF_RDTSC
;
386 if (CpuFeatures
& 0x00000100) FeatureBits
|= KF_CMPXCHG8B
;
387 if (CpuFeatures
& 0x00000800) FeatureBits
|= KF_FAST_SYSCALL
;
388 if (CpuFeatures
& 0x00001000) FeatureBits
|= KF_MTRR
;
389 if (CpuFeatures
& 0x00002000) FeatureBits
|= KF_GLOBAL_PAGE
| KF_CR4
;
390 if (CpuFeatures
& 0x00008000) FeatureBits
|= KF_CMOV
;
391 if (CpuFeatures
& 0x00010000) FeatureBits
|= KF_PAT
;
392 if (CpuFeatures
& 0x00200000) FeatureBits
|= KF_DTS
;
393 if (CpuFeatures
& 0x00800000) FeatureBits
|= KF_MMX
;
394 if (CpuFeatures
& 0x01000000) FeatureBits
|= KF_FXSR
;
395 if (CpuFeatures
& 0x02000000) FeatureBits
|= KF_XMMI
;
396 if (CpuFeatures
& 0x04000000) FeatureBits
|= KF_XMMI64
;
398 /* Check if the CPU has hyper-threading */
399 if (CpuFeatures
& 0x10000000)
401 /* Set the number of logical CPUs */
402 Prcb
->LogicalProcessorsPerPhysicalProcessor
= (UCHAR
)(Reg
[1] >> 16);
403 if (Prcb
->LogicalProcessorsPerPhysicalProcessor
> 1)
405 /* We're on dual-core */
406 KiSMTProcessorsPresent
= TRUE
;
411 /* We only have a single CPU */
412 Prcb
->LogicalProcessorsPerPhysicalProcessor
= 1;
415 /* Check if CPUID 0x80000000 is supported */
419 CPUID(0x80000000, &Reg
[0], &Dummy
, &Dummy
, &Dummy
);
420 if ((Reg
[0] & 0xffffff00) == 0x80000000)
422 /* Check if CPUID 0x80000001 is supported */
423 if (Reg
[0] >= 0x80000001)
425 /* Check which extended features are available. */
426 CPUID(0x80000001, &Dummy
, &Dummy
, &Dummy
, &Reg
[3]);
428 /* Check if NX-bit is supported */
429 if (Reg
[3] & 0x00100000) FeatureBits
|= KF_NX_BIT
;
431 /* Now handle each features for each CPU Vendor */
436 if (Reg
[3] & 0x80000000) FeatureBits
|= KF_3DNOW
;
443 /* Return the Feature Bits */
449 KiGetCacheInformation(VOID
)
451 PKIPCR Pcr
= (PKIPCR
)KeGetPcr();
453 ULONG Data
[4], Dummy
;
454 ULONG CacheRequests
= 0, i
;
455 ULONG CurrentRegister
;
457 ULONG Size
, Associativity
= 0, CacheLine
= 64, CurrentSize
= 0;
458 BOOLEAN FirstPass
= TRUE
;
460 /* Set default L2 size */
461 Pcr
->SecondLevelCacheSize
= 0;
463 /* Get the Vendor ID and make sure we support CPUID */
464 Vendor
= KiGetCpuVendor();
467 /* Check the Vendor ID */
470 /* Handle Intel case */
473 /*Check if we support CPUID 2 */
474 CPUID(0, &Data
[0], &Dummy
, &Dummy
, &Dummy
);
477 /* We need to loop for the number of times CPUID will tell us to */
480 /* Do the CPUID call */
481 CPUID(2, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
483 /* Check if it was the first call */
487 * The number of times to loop is the first byte. Read
488 * it and then destroy it so we don't get confused.
490 CacheRequests
= Data
[0] & 0xFF;
491 Data
[0] &= 0xFFFFFF00;
493 /* Don't go over this again */
497 /* Loop all 4 registers */
498 for (i
= 0; i
< 4; i
++)
500 /* Get the current register */
501 CurrentRegister
= Data
[i
];
504 * If the upper bit is set, then this register should
507 if (CurrentRegister
& 0x80000000) continue;
509 /* Keep looping for every byte inside this register */
510 while (CurrentRegister
)
512 /* Read a byte, skip a byte. */
513 RegisterByte
= (UCHAR
)(CurrentRegister
& 0xFF);
514 CurrentRegister
>>= 8;
515 if (!RegisterByte
) continue;
518 * Valid values are from 0x40 (0 bytes) to 0x49
519 * (32MB), or from 0x80 to 0x89 (same size but
522 if (((RegisterByte
> 0x40) && (RegisterByte
<= 0x47)) ||
523 ((RegisterByte
> 0x78) && (RegisterByte
<= 0x7C)) ||
524 ((RegisterByte
> 0x80) && (RegisterByte
<= 0x85)))
526 /* Compute associativity */
528 if (RegisterByte
>= 0x79) Associativity
= 8;
530 /* Mask out only the first nibble */
531 RegisterByte
&= 0x07;
533 /* Check if this cache is bigger than the last */
534 Size
= 0x10000 << RegisterByte
;
535 if ((Size
/ Associativity
) > CurrentSize
)
537 /* Set the L2 Cache Size and Associativity */
538 CurrentSize
= Size
/ Associativity
;
539 Pcr
->SecondLevelCacheSize
= Size
;
540 Pcr
->SecondLevelCacheAssociativity
= Associativity
;
543 else if ((RegisterByte
> 0x21) && (RegisterByte
<= 0x29))
545 /* Set minimum cache line size */
546 if (CacheLine
< 128) CacheLine
= 128;
548 /* Hard-code size/associativity */
550 switch (RegisterByte
)
574 /* Check if this cache is bigger than the last */
575 if ((Size
/ Associativity
) > CurrentSize
)
577 /* Set the L2 Cache Size and Associativity */
578 CurrentSize
= Size
/ Associativity
;
579 Pcr
->SecondLevelCacheSize
= Size
;
580 Pcr
->SecondLevelCacheAssociativity
= Associativity
;
583 else if (((RegisterByte
> 0x65) && (RegisterByte
< 0x69)) ||
584 (RegisterByte
== 0x2C) || (RegisterByte
== 0xF0))
586 /* Indicates L1 cache line of 64 bytes */
587 KePrefetchNTAGranularity
= 64;
589 else if (RegisterByte
== 0xF1)
591 /* Indicates L1 cache line of 128 bytes */
592 KePrefetchNTAGranularity
= 128;
594 else if (((RegisterByte
>= 0x4A) && (RegisterByte
<= 0x4C)) ||
595 (RegisterByte
== 0x78) ||
596 (RegisterByte
== 0x7D) ||
597 (RegisterByte
== 0x7F) ||
598 (RegisterByte
== 0x86) ||
599 (RegisterByte
== 0x87))
601 /* Set minimum cache line size */
602 if (CacheLine
< 64) CacheLine
= 64;
604 /* Hard-code size/associativity */
605 switch (RegisterByte
)
608 Size
= 4 * 1024 * 1024;
613 Size
= 6 * 1024 * 1024;
618 Size
= 8 * 1024 * 1024;
623 Size
= 1 * 1024 * 1024;
628 Size
= 2 * 1024 * 1024;
643 Size
= 1 * 1024 * 1024;
652 /* Check if this cache is bigger than the last */
653 if ((Size
/ Associativity
) > CurrentSize
)
655 /* Set the L2 Cache Size and Associativity */
656 CurrentSize
= Size
/ Associativity
;
657 Pcr
->SecondLevelCacheSize
= Size
;
658 Pcr
->SecondLevelCacheAssociativity
= Associativity
;
663 } while (--CacheRequests
);
669 /* Check if we support CPUID 0x80000005 */
670 CPUID(0x80000000, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
671 if (Data
[0] >= 0x80000006)
673 /* Get L1 size first */
674 CPUID(0x80000005, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
675 KePrefetchNTAGranularity
= Data
[2] & 0xFF;
677 /* Check if we support CPUID 0x80000006 */
678 CPUID(0x80000000, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
679 if (Data
[0] >= 0x80000006)
681 /* Get 2nd level cache and tlb size */
682 CPUID(0x80000006, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
684 /* Cache line size */
685 CacheLine
= Data
[2] & 0xFF;
687 /* Hardcode associativity */
688 RegisterByte
= Data
[2] >> 12;
689 switch (RegisterByte
)
714 Size
= (Data
[2] >> 16) << 10;
716 /* Hack for Model 6, Steping 300 */
717 if ((KeGetCurrentPrcb()->CpuType
== 6) &&
718 (KeGetCurrentPrcb()->CpuStep
== 0x300))
720 /* Stick 64K in there */
724 /* Set the L2 Cache Size and associativity */
725 Pcr
->SecondLevelCacheSize
= Size
;
726 Pcr
->SecondLevelCacheAssociativity
= Associativity
;
740 /* Set the cache line */
741 if (CacheLine
> KeLargestCacheLine
) KeLargestCacheLine
= CacheLine
;
742 DPRINT1("Prefetch Cache: %d bytes\tL2 Cache: %d bytes\tL2 Cache Line: %d bytes\tL2 Cache Associativity: %d\n",
743 KePrefetchNTAGranularity
,
744 Pcr
->SecondLevelCacheSize
,
746 Pcr
->SecondLevelCacheAssociativity
);
755 /* Save current CR0 */
758 /* If this is a 486, enable Write-Protection */
759 if (KeGetCurrentPrcb()->CpuType
> 3) Cr0
|= CR0_WP
;
767 KiInitializeTSS2(IN PKTSS Tss
,
768 IN PKGDTENTRY TssEntry OPTIONAL
)
772 /* Make sure the GDT Entry is valid */
776 TssEntry
->LimitLow
= sizeof(KTSS
) - 1;
777 TssEntry
->HighWord
.Bits
.LimitHi
= 0;
780 /* Now clear the I/O Map */
781 ASSERT(IOPM_COUNT
== 1);
782 RtlFillMemory(Tss
->IoMaps
[0].IoMap
, IOPM_FULL_SIZE
, 0xFF);
784 /* Initialize Interrupt Direction Maps */
785 p
= (PUCHAR
)(Tss
->IoMaps
[0].DirectionMap
);
786 RtlZeroMemory(p
, IOPM_DIRECTION_MAP_SIZE
);
788 /* Add DPMI support for interrupts */
793 /* Initialize the default Interrupt Direction Map */
794 p
= Tss
->IntDirectionMap
;
795 RtlZeroMemory(Tss
->IntDirectionMap
, IOPM_DIRECTION_MAP_SIZE
);
797 /* Add DPMI support */
805 KiInitializeTSS(IN PKTSS Tss
)
807 /* Set an invalid map base */
808 Tss
->IoMapBase
= KiComputeIopmOffset(IO_ACCESS_MAP_NONE
);
810 /* Disable traps during Task Switches */
813 /* Set LDT and Ring 0 SS */
815 Tss
->Ss0
= KGDT_R0_DATA
;
820 Ki386InitializeTss(IN PKTSS Tss
,
824 PKGDTENTRY TssEntry
, TaskGateEntry
;
826 /* Initialize the boot TSS. */
827 TssEntry
= &Gdt
[KGDT_TSS
/ sizeof(KGDTENTRY
)];
828 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
829 TssEntry
->HighWord
.Bits
.Pres
= 1;
830 TssEntry
->HighWord
.Bits
.Dpl
= 0;
831 KiInitializeTSS2(Tss
, TssEntry
);
832 KiInitializeTSS(Tss
);
834 /* Load the task register */
835 Ke386SetTr(KGDT_TSS
);
837 /* Setup the Task Gate for Double Fault Traps */
838 TaskGateEntry
= (PKGDTENTRY
)&Idt
[8];
839 TaskGateEntry
->HighWord
.Bits
.Type
= I386_TASK_GATE
;
840 TaskGateEntry
->HighWord
.Bits
.Pres
= 1;
841 TaskGateEntry
->HighWord
.Bits
.Dpl
= 0;
842 ((PKIDTENTRY
)TaskGateEntry
)->Selector
= KGDT_DF_TSS
;
844 /* Initialize the TSS used for handling double faults. */
845 Tss
= (PKTSS
)KiDoubleFaultTSS
;
846 KiInitializeTSS(Tss
);
847 Tss
->CR3
= __readcr3();
848 Tss
->Esp0
= KiDoubleFaultStack
;
849 Tss
->Esp
= KiDoubleFaultStack
;
850 Tss
->Eip
= PtrToUlong(KiTrap08
);
851 Tss
->Cs
= KGDT_R0_CODE
;
852 Tss
->Fs
= KGDT_R0_PCR
;
853 Tss
->Ss
= Ke386GetSs();
854 Tss
->Es
= KGDT_R3_DATA
| RPL_MASK
;
855 Tss
->Ds
= KGDT_R3_DATA
| RPL_MASK
;
857 /* Setup the Double Trap TSS entry in the GDT */
858 TssEntry
= &Gdt
[KGDT_DF_TSS
/ sizeof(KGDTENTRY
)];
859 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
860 TssEntry
->HighWord
.Bits
.Pres
= 1;
861 TssEntry
->HighWord
.Bits
.Dpl
= 0;
862 TssEntry
->BaseLow
= (USHORT
)((ULONG_PTR
)Tss
& 0xFFFF);
863 TssEntry
->HighWord
.Bytes
.BaseMid
= (UCHAR
)((ULONG_PTR
)Tss
>> 16);
864 TssEntry
->HighWord
.Bytes
.BaseHi
= (UCHAR
)((ULONG_PTR
)Tss
>> 24);
865 TssEntry
->LimitLow
= KTSS_IO_MAPS
;
867 /* Now setup the NMI Task Gate */
868 TaskGateEntry
= (PKGDTENTRY
)&Idt
[2];
869 TaskGateEntry
->HighWord
.Bits
.Type
= I386_TASK_GATE
;
870 TaskGateEntry
->HighWord
.Bits
.Pres
= 1;
871 TaskGateEntry
->HighWord
.Bits
.Dpl
= 0;
872 ((PKIDTENTRY
)TaskGateEntry
)->Selector
= KGDT_NMI_TSS
;
874 /* Initialize the actual TSS */
875 Tss
= (PKTSS
)KiNMITSS
;
876 KiInitializeTSS(Tss
);
877 Tss
->CR3
= __readcr3();
878 Tss
->Esp0
= KiDoubleFaultStack
;
879 Tss
->Esp
= KiDoubleFaultStack
;
880 Tss
->Eip
= PtrToUlong(KiTrap02
);
881 Tss
->Cs
= KGDT_R0_CODE
;
882 Tss
->Fs
= KGDT_R0_PCR
;
883 Tss
->Ss
= Ke386GetSs();
884 Tss
->Es
= KGDT_R3_DATA
| RPL_MASK
;
885 Tss
->Ds
= KGDT_R3_DATA
| RPL_MASK
;
887 /* And its associated TSS Entry */
888 TssEntry
= &Gdt
[KGDT_NMI_TSS
/ sizeof(KGDTENTRY
)];
889 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
890 TssEntry
->HighWord
.Bits
.Pres
= 1;
891 TssEntry
->HighWord
.Bits
.Dpl
= 0;
892 TssEntry
->BaseLow
= (USHORT
)((ULONG_PTR
)Tss
& 0xFFFF);
893 TssEntry
->HighWord
.Bytes
.BaseMid
= (UCHAR
)((ULONG_PTR
)Tss
>> 16);
894 TssEntry
->HighWord
.Bytes
.BaseHi
= (UCHAR
)((ULONG_PTR
)Tss
>> 24);
895 TssEntry
->LimitLow
= KTSS_IO_MAPS
;
900 KeFlushCurrentTb(VOID
)
902 /* Flush the TLB by resetting CR3 */
903 __writecr3(__readcr3());
908 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState
)
913 // Restore the CR registers
915 __writecr0(ProcessorState
->SpecialRegisters
.Cr0
);
916 Ke386SetCr2(ProcessorState
->SpecialRegisters
.Cr2
);
917 __writecr3(ProcessorState
->SpecialRegisters
.Cr3
);
918 if (KeFeatureBits
& KF_CR4
) __writecr4(ProcessorState
->SpecialRegisters
.Cr4
);
921 // Restore the DR registers
923 __writedr(0, ProcessorState
->SpecialRegisters
.KernelDr0
);
924 __writedr(1, ProcessorState
->SpecialRegisters
.KernelDr1
);
925 __writedr(2, ProcessorState
->SpecialRegisters
.KernelDr2
);
926 __writedr(3, ProcessorState
->SpecialRegisters
.KernelDr3
);
927 __writedr(6, ProcessorState
->SpecialRegisters
.KernelDr6
);
928 __writedr(7, ProcessorState
->SpecialRegisters
.KernelDr7
);
931 // Restore GDT and IDT
933 Ke386SetGlobalDescriptorTable(&ProcessorState
->SpecialRegisters
.Gdtr
.Limit
);
934 __lidt(&ProcessorState
->SpecialRegisters
.Idtr
.Limit
);
937 // Clear the busy flag so we don't crash if we reload the same selector
939 TssEntry
= (PKGDTENTRY
)(ProcessorState
->SpecialRegisters
.Gdtr
.Base
+
940 ProcessorState
->SpecialRegisters
.Tr
);
941 TssEntry
->HighWord
.Bytes
.Flags1
&= ~0x2;
944 // Restore TSS and LDT
946 Ke386SetTr(ProcessorState
->SpecialRegisters
.Tr
);
947 Ke386SetLocalDescriptorTable(ProcessorState
->SpecialRegisters
.Ldtr
);
952 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState
)
954 /* Save the CR registers */
955 ProcessorState
->SpecialRegisters
.Cr0
= __readcr0();
956 ProcessorState
->SpecialRegisters
.Cr2
= __readcr2();
957 ProcessorState
->SpecialRegisters
.Cr3
= __readcr3();
958 ProcessorState
->SpecialRegisters
.Cr4
= (KeFeatureBits
& KF_CR4
) ?
961 /* Save the DR registers */
962 ProcessorState
->SpecialRegisters
.KernelDr0
= __readdr(0);
963 ProcessorState
->SpecialRegisters
.KernelDr1
= __readdr(1);
964 ProcessorState
->SpecialRegisters
.KernelDr2
= __readdr(2);
965 ProcessorState
->SpecialRegisters
.KernelDr3
= __readdr(3);
966 ProcessorState
->SpecialRegisters
.KernelDr6
= __readdr(6);
967 ProcessorState
->SpecialRegisters
.KernelDr7
= __readdr(7);
970 /* Save GDT, IDT, LDT and TSS */
971 Ke386GetGlobalDescriptorTable(&ProcessorState
->SpecialRegisters
.Gdtr
.Limit
);
972 __sidt(&ProcessorState
->SpecialRegisters
.Idtr
.Limit
);
973 ProcessorState
->SpecialRegisters
.Tr
= Ke386GetTr();
974 ProcessorState
->SpecialRegisters
.Ldtr
= Ke386GetLocalDescriptorTable();
979 KiInitializeMachineType(VOID
)
981 /* Set the Machine Type we got from NTLDR */
982 KeI386MachineType
= KeLoaderBlock
->u
.I386
.MachineType
& 0x000FF;
987 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context
)
990 WRMSR(0x174, KGDT_R0_CODE
);
991 WRMSR(0x175, (ULONG_PTR
)KeGetCurrentPrcb()->DpcStack
);
994 WRMSR(0x176, (ULONG_PTR
)KiFastCallEntry
);
1000 KiRestoreFastSyscallReturnState(VOID
)
1002 /* Check if the CPU Supports fast system call */
1003 if (KeFeatureBits
& KF_FAST_SYSCALL
)
1005 /* Check if it has been disabled */
1006 if (!KiFastSystemCallDisable
)
1008 /* Do an IPI to enable it */
1009 KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters
, 0);
1011 /* It's enabled, so use the proper exit stub */
1012 KiFastCallExitHandler
= KiSystemCallSysExitReturn
;
1013 DPRINT1("Support for SYSENTER detected.\n");
1017 /* Disable fast system call */
1018 KeFeatureBits
&= ~KF_FAST_SYSCALL
;
1019 KiFastCallExitHandler
= KiSystemCallTrapReturn
;
1020 DPRINT1("Support for SYSENTER disabled.\n");
1025 /* Use the IRET handler */
1026 KiFastCallExitHandler
= KiSystemCallTrapReturn
;
1027 DPRINT1("No support for SYSENTER detected.\n");
1033 Ki386EnableDE(IN ULONG_PTR Context
)
1036 __writecr4(__readcr4() | CR4_DE
);
1042 Ki386EnableFxsr(IN ULONG_PTR Context
)
1045 __writecr4(__readcr4() | CR4_FXSR
);
1051 Ki386EnableXMMIExceptions(IN ULONG_PTR Context
)
1053 PKIDTENTRY IdtEntry
;
1055 /* Get the IDT Entry for Interrupt 0x13 */
1056 IdtEntry
= &((PKIPCR
)KeGetPcr())->IDT
[0x13];
1059 IdtEntry
->Selector
= KGDT_R0_CODE
;
1060 IdtEntry
->Offset
= ((ULONG_PTR
)KiTrap13
& 0xFFFF);
1061 IdtEntry
->ExtendedOffset
= ((ULONG_PTR
)KiTrap13
>> 16) & 0xFFFF;
1062 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->Dpl
= 0;
1063 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->Present
= 1;
1064 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->SegmentType
= I386_INTERRUPT_GATE
;
1066 /* Enable XMMI exceptions */
1067 __writecr4(__readcr4() | CR4_XMMEXCPT
);
1073 KiI386PentiumLockErrataFixup(VOID
)
1075 KDESCRIPTOR IdtDescriptor
;
1076 PKIDTENTRY NewIdt
, NewIdt2
;
1078 /* Allocate memory for a new IDT */
1079 NewIdt
= ExAllocatePool(NonPagedPool
, 2 * PAGE_SIZE
);
1081 /* Put everything after the first 7 entries on a new page */
1082 NewIdt2
= (PVOID
)((ULONG_PTR
)NewIdt
+ PAGE_SIZE
- (7 * sizeof(KIDTENTRY
)));
1084 /* Disable interrupts */
1087 /* Get the current IDT and copy it */
1088 __sidt(&IdtDescriptor
.Limit
);
1089 RtlCopyMemory(NewIdt2
,
1090 (PVOID
)IdtDescriptor
.Base
,
1091 IdtDescriptor
.Limit
+ 1);
1092 IdtDescriptor
.Base
= (ULONG
)NewIdt2
;
1094 /* Set the new IDT */
1095 __lidt(&IdtDescriptor
.Limit
);
1096 ((PKIPCR
)KeGetPcr())->IDT
= NewIdt2
;
1098 /* Restore interrupts */
1101 /* Set the first 7 entries as read-only to produce a fault */
1102 MmSetPageProtect(NULL
, NewIdt
, PAGE_READONLY
);
1107 KeDisableInterrupts(VOID
)
1112 /* Get EFLAGS and check if the interrupt bit is set */
1113 Flags
= __readeflags();
1114 Return
= (Flags
& EFLAGS_INTERRUPT_MASK
) ? TRUE
: FALSE
;
1116 /* Disable interrupts */
1123 KeInvalidateAllCaches(VOID
)
1125 /* Only supported on Pentium Pro and higher */
1126 if (KeI386CpuType
< 6) return FALSE
;
1128 /* Invalidate all caches */
1135 KeZeroPages(IN PVOID Address
,
1138 /* Not using XMMI in this routine */
1139 RtlZeroMemory(Address
, Size
);
1144 KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame
,
1145 IN PKEXCEPTION_FRAME ExceptionFrame
)
1147 PKPRCB Prcb
= KeGetCurrentPrcb();
1150 // Save full context
1152 Prcb
->ProcessorState
.ContextFrame
.ContextFlags
= CONTEXT_FULL
|
1153 CONTEXT_DEBUG_REGISTERS
;
1154 KeTrapFrameToContext(TrapFrame
, NULL
, &Prcb
->ProcessorState
.ContextFrame
);
1157 // Save control registers
1159 KiSaveProcessorControlState(&Prcb
->ProcessorState
);
1164 KiIsNpxPresent(VOID
)
1172 /* Read CR0 and mask out FPU flags */
1173 Cr0
= __readcr0() & ~(CR0_MP
| CR0_TS
| CR0_EM
| CR0_ET
);
1175 /* Store on FPU stack */
1180 asm volatile ("fninit;" "fnstsw %0" : "+m"(Magic
));
1183 /* Magic should now be cleared */
1186 /* You don't have an FPU -- enable emulation for now */
1187 __writecr0(Cr0
| CR0_EM
| CR0_TS
);
1191 /* You have an FPU, enable it */
1194 /* Enable INT 16 on 486 and higher */
1195 if (KeGetCurrentPrcb()->CpuType
>= 3) Cr0
|= CR0_NE
;
1198 __writecr0(Cr0
| CR0_EM
| CR0_TS
);
1204 KiIsNpxErrataPresent(VOID
)
1206 BOOLEAN ErrataPresent
;
1208 volatile double Value1
, Value2
;
1210 /* Disable interrupts */
1213 /* Read CR0 and remove FPU flags */
1215 __writecr0(Cr0
& ~(CR0_MP
| CR0_TS
| CR0_EM
));
1217 /* Initialize FPU state */
1220 /* Multiply the magic values and divide, we should get the result back */
1223 ErrataPresent
= (Value1
* Value2
/ 3145727.0) != 4195835.0;
1228 /* Enable interrupts */
1231 /* Return if there's an errata */
1232 return ErrataPresent
;
1237 KiFlushNPXState(IN PFLOATING_SAVE_AREA SaveArea
)
1240 PKTHREAD Thread
, NpxThread
;
1241 PFX_SAVE_AREA FxSaveArea
;
1243 /* Save volatiles and disable interrupts */
1244 EFlags
= __readeflags();
1247 /* Save the PCR and get the current thread */
1248 Thread
= KeGetCurrentThread();
1250 /* Check if we're already loaded */
1251 if (Thread
->NpxState
!= NPX_STATE_LOADED
)
1253 /* If there's nothing to load, quit */
1254 if (!SaveArea
) return;
1256 /* Need FXSR support for this */
1257 ASSERT(KeI386FxsrPresent
== TRUE
);
1259 /* Check for sane CR0 */
1261 if (Cr0
& (CR0_MP
| CR0_TS
| CR0_EM
))
1263 /* Mask out FPU flags */
1264 __writecr0(Cr0
& ~(CR0_MP
| CR0_TS
| CR0_EM
));
1267 /* Get the NPX thread and check its FPU state */
1268 NpxThread
= KeGetCurrentPrcb()->NpxThread
;
1269 if ((NpxThread
) && (NpxThread
->NpxState
== NPX_STATE_LOADED
))
1271 /* Get the FX frame and store the state there */
1272 FxSaveArea
= KiGetThreadNpxArea(NpxThread
);
1273 Ke386FxSave(FxSaveArea
);
1275 /* NPX thread has lost its state */
1276 NpxThread
->NpxState
= NPX_STATE_NOT_LOADED
;
1279 /* Now load NPX state from the NPX area */
1280 FxSaveArea
= KiGetThreadNpxArea(Thread
);
1281 Ke386FxStore(FxSaveArea
);
1285 /* Check for sane CR0 */
1287 if (Cr0
& (CR0_MP
| CR0_TS
| CR0_EM
))
1289 /* Mask out FPU flags */
1290 __writecr0(Cr0
& ~(CR0_MP
| CR0_TS
| CR0_EM
));
1294 FxSaveArea
= KiGetThreadNpxArea(Thread
);
1295 Thread
->NpxState
= NPX_STATE_NOT_LOADED
;
1297 /* Save state if supported by CPU */
1298 if (KeI386FxsrPresent
) Ke386FxSave(FxSaveArea
);
1301 /* Now save the FN state wherever it was requested */
1302 if (SaveArea
) Ke386FnSave(SaveArea
);
1304 /* Clear NPX thread */
1305 KeGetCurrentPrcb()->NpxThread
= NULL
;
1307 /* Add the CR0 from the NPX frame */
1308 Cr0
|= NPX_STATE_NOT_LOADED
;
1309 Cr0
|= FxSaveArea
->Cr0NpxState
;
1312 /* Restore interrupt state */
1313 __writeeflags(EFlags
);
1316 /* PUBLIC FUNCTIONS **********************************************************/
1323 KiCoprocessorError(VOID
)
1325 PFX_SAVE_AREA NpxArea
;
1327 /* Get the FPU area */
1328 NpxArea
= KiGetThreadNpxArea(KeGetCurrentThread());
1331 NpxArea
->Cr0NpxState
= CR0_TS
;
1332 __writecr0(__readcr0() | CR0_TS
);
1340 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save
)
1342 PFNSAVE_FORMAT FpState
;
1343 ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL
);
1344 DPRINT1("%s is not really implemented\n", __FUNCTION__
);
1346 /* check if we are doing software emulation */
1347 if (!KeI386NpxPresent
) return STATUS_ILLEGAL_FLOAT_CONTEXT
;
1349 FpState
= ExAllocatePool(NonPagedPool
, sizeof (FNSAVE_FORMAT
));
1350 if (!FpState
) return STATUS_INSUFFICIENT_RESOURCES
;
1352 *((PVOID
*) Save
) = FpState
;
1354 asm volatile("fnsave %0\n\t" : "=m" (*FpState
));
1362 KeGetCurrentThread()->DispatcherHeader
.NpxIrql
= KeGetCurrentIrql();
1363 return STATUS_SUCCESS
;
1371 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save
)
1373 PFNSAVE_FORMAT FpState
= *((PVOID
*) Save
);
1374 ASSERT(KeGetCurrentThread()->DispatcherHeader
.NpxIrql
== KeGetCurrentIrql());
1375 DPRINT1("%s is not really implemented\n", __FUNCTION__
);
1378 asm volatile("fnclex\n\t");
1379 asm volatile("frstor %0\n\t" : "=m" (*FpState
));
1388 ExFreePool(FpState
);
1389 return STATUS_SUCCESS
;
1397 KeGetRecommendedSharedDataAlignment(VOID
)
1399 /* Return the global variable */
1400 return KeLargestCacheLine
;
1405 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext
,
1410 /* Signal this packet as done */
1411 KiIpiSignalPacketDone(PacketContext
);
1413 /* Flush the TB for the Current CPU */
1422 KeFlushEntireTb(IN BOOLEAN Invalid
,
1423 IN BOOLEAN AllProcessors
)
1427 KAFFINITY TargetAffinity
;
1428 PKPRCB Prcb
= KeGetCurrentPrcb();
1431 /* Raise the IRQL for the TB Flush */
1432 OldIrql
= KeRaiseIrqlToSynchLevel();
1435 /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1437 /* Get the current processor affinity, and exclude ourselves */
1438 TargetAffinity
= KeActiveProcessors
;
1439 TargetAffinity
&= ~Prcb
->SetMember
;
1441 /* Make sure this is MP */
1444 /* Send an IPI TB flush to the other processors */
1445 KiIpiSendPacket(TargetAffinity
,
1446 KiFlushTargetEntireTb
,
1453 /* Flush the TB for the Current CPU, and update the flush stamp */
1457 /* If this is MP, wait for the other processors to finish */
1461 ASSERT(Prcb
== (volatile PKPRCB
)KeGetCurrentPrcb());
1464 ASSERTMSG("Not yet implemented\n", FALSE
);
1468 /* Update the flush stamp and return to original IRQL */
1469 InterlockedExchangeAdd(&KiTbFlushTimeStamp
, 1);
1470 KeLowerIrql(OldIrql
);
1478 KeSetDmaIoCoherency(IN ULONG Coherency
)
1480 /* Save the coherency globally */
1481 KiDmaIoCoherency
= Coherency
;
1489 KeQueryActiveProcessors(VOID
)
1493 /* Simply return the number of active processors */
1494 return KeActiveProcessors
;
1502 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State
)
1504 /* Capture the context */
1505 RtlCaptureContext(&State
->ContextFrame
);
1507 /* Capture the control state */
1508 KiSaveProcessorControlState(State
);