2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/i386/cpu.c
5 * PURPOSE: Routines for CPU-level support
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
9 /* INCLUDES *****************************************************************/
15 /* GLOBALS *******************************************************************/
17 /* The TSS to use for Double Fault Traps (INT 0x9) */
18 UCHAR KiDoubleFaultTSS
[KTSS_IO_MAPS
];
20 /* The TSS to use for NMI Fault Traps (INT 0x2) */
21 UCHAR KiNMITSS
[KTSS_IO_MAPS
];
23 /* CPU Features and Flags */
26 ULONG KeProcessorArchitecture
;
27 ULONG KeProcessorLevel
;
28 ULONG KeProcessorRevision
;
30 ULONG KiFastSystemCallDisable
;
31 ULONG KeI386NpxPresent
= 0;
32 ULONG KiMXCsrMask
= 0;
33 ULONG MxcsrFeatureMask
= 0;
34 ULONG KeI386XMMIPresent
= 0;
35 ULONG KeI386FxsrPresent
= 0;
36 ULONG KeI386MachineType
;
37 ULONG Ke386Pae
= FALSE
;
38 ULONG Ke386NoExecute
= FALSE
;
39 ULONG KeLargestCacheLine
= 0x40;
40 ULONG KeDcacheFlushCount
= 0;
41 ULONG KeIcacheFlushCount
= 0;
42 ULONG KiDmaIoCoherency
= 0;
43 ULONG KePrefetchNTAGranularity
= 32;
44 CHAR KeNumberProcessors
;
45 KAFFINITY KeActiveProcessors
= 1;
46 BOOLEAN KiI386PentiumLockErrataPresent
;
47 BOOLEAN KiSMTProcessorsPresent
;
49 /* The distance between SYSEXIT and IRETD return modes */
50 UCHAR KiSystemCallExitAdjust
;
52 /* The offset that was applied -- either 0 or the value above */
53 UCHAR KiSystemCallExitAdjusted
;
55 /* Whether the adjustment was already done once */
56 BOOLEAN KiFastCallCopyDoneOnce
;
59 volatile LONG KiTbFlushTimeStamp
;
62 static const CHAR CmpIntelID
[] = "GenuineIntel";
63 static const CHAR CmpAmdID
[] = "AuthenticAMD";
64 static const CHAR CmpCyrixID
[] = "CyrixInstead";
65 static const CHAR CmpTransmetaID
[] = "GenuineTMx86";
66 static const CHAR CmpCentaurID
[] = "CentaurHauls";
67 static const CHAR CmpRiseID
[] = "RiseRiseRise";
69 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
73 CPUID(IN ULONG InfoType
,
74 OUT PULONG CpuInfoEax
,
75 OUT PULONG CpuInfoEbx
,
76 OUT PULONG CpuInfoEcx
,
77 OUT PULONG CpuInfoEdx
)
81 /* Perform the CPUID Operation */
82 __cpuid((int*)CpuInfo
, InfoType
);
84 /* Return the results */
85 *CpuInfoEax
= CpuInfo
[0];
86 *CpuInfoEbx
= CpuInfo
[1];
87 *CpuInfoEcx
= CpuInfo
[2];
88 *CpuInfoEdx
= CpuInfo
[3];
93 WRMSR(IN ULONG Register
,
96 /* Write to the MSR */
97 __writemsr(Register
, Value
);
102 RDMSR(IN ULONG Register
)
104 /* Read from the MSR */
105 return __readmsr(Register
);
108 /* FUNCTIONS *****************************************************************/
112 KiSetProcessorType(VOID
)
114 ULONG EFlags
, NewEFlags
;
116 ULONG Stepping
, Type
;
118 /* Start by assuming no CPUID data */
119 KeGetCurrentPrcb()->CpuID
= 0;
122 EFlags
= __readeflags();
124 /* XOR out the ID bit and update EFlags */
125 NewEFlags
= EFlags
^ EFLAGS_ID
;
126 __writeeflags(NewEFlags
);
128 /* Get them back and see if they were modified */
129 NewEFlags
= __readeflags();
130 if (NewEFlags
!= EFlags
)
132 /* The modification worked, so CPUID exists. Set the ID Bit again. */
134 __writeeflags(EFlags
);
136 /* Peform CPUID 0 to see if CPUID 1 is supported */
137 CPUID(0, &Reg
, &Dummy
, &Dummy
, &Dummy
);
141 CPUID(1, &Reg
, &Dummy
, &Dummy
, &Dummy
);
144 * Get the Stepping and Type. The stepping contains both the
145 * Model and the Step, while the Type contains the returned Type.
146 * We ignore the family.
148 * For the stepping, we convert this: zzzzzzxy into this: x0y
150 Stepping
= Reg
& 0xF0;
152 Stepping
+= (Reg
& 0xFF);
157 /* Save them in the PRCB */
158 KeGetCurrentPrcb()->CpuID
= TRUE
;
159 KeGetCurrentPrcb()->CpuType
= (UCHAR
)Type
;
160 KeGetCurrentPrcb()->CpuStep
= (USHORT
)Stepping
;
164 DPRINT1("CPUID Support lacking\n");
169 DPRINT1("CPUID Support lacking\n");
173 __writeeflags(EFlags
);
180 PKPRCB Prcb
= KeGetCurrentPrcb();
184 /* Assume no Vendor ID and fail if no CPUID Support. */
185 Prcb
->VendorString
[0] = 0;
186 if (!Prcb
->CpuID
) return 0;
188 /* Get the Vendor ID and null-terminate it */
189 CPUID(0, &Vendor
[0], &Vendor
[1], &Vendor
[2], &Vendor
[3]);
192 /* Re-arrange vendor string */
194 Vendor
[2] = Vendor
[3];
197 /* Copy it to the PRCB and null-terminate it again */
198 RtlCopyMemory(Prcb
->VendorString
,
200 sizeof(Prcb
->VendorString
) - sizeof(CHAR
));
201 Prcb
->VendorString
[sizeof(Prcb
->VendorString
) - sizeof(CHAR
)] = ANSI_NULL
;
203 /* Now check the CPU Type */
204 if (!strcmp(Prcb
->VendorString
, CmpIntelID
))
208 else if (!strcmp(Prcb
->VendorString
, CmpAmdID
))
212 else if (!strcmp(Prcb
->VendorString
, CmpCyrixID
))
214 DPRINT1("Cyrix CPU support not fully tested!\n");
217 else if (!strcmp(Prcb
->VendorString
, CmpTransmetaID
))
219 DPRINT1("Transmeta CPU support not fully tested!\n");
220 return CPU_TRANSMETA
;
222 else if (!strcmp(Prcb
->VendorString
, CmpCentaurID
))
224 DPRINT1("Centaur CPU support not fully tested!\n");
227 else if (!strcmp(Prcb
->VendorString
, CmpRiseID
))
229 DPRINT1("Rise CPU support not fully tested!\n");
239 KiGetFeatureBits(VOID
)
241 PKPRCB Prcb
= KeGetCurrentPrcb();
243 ULONG FeatureBits
= KF_WORKING_PTE
;
245 BOOLEAN ExtendedCPUID
= TRUE
;
246 ULONG CpuFeatures
= 0;
248 /* Get the Vendor ID */
249 Vendor
= KiGetCpuVendor();
251 /* Make sure we got a valid vendor ID at least. */
252 if (!Vendor
) return FeatureBits
;
254 /* Get the CPUID Info. Features are in Reg[3]. */
255 CPUID(1, &Reg
[0], &Reg
[1], &Dummy
, &Reg
[3]);
257 /* Set the initial APIC ID */
258 Prcb
->InitialApicId
= (UCHAR
)(Reg
[1] >> 24);
265 /* Check if it's a P6 */
266 if (Prcb
->CpuType
== 6)
268 /* Perform the special sequence to get the MicroCode Signature */
270 CPUID(1, &Dummy
, &Dummy
, &Dummy
, &Dummy
);
271 Prcb
->UpdateSignature
.QuadPart
= RDMSR(0x8B);
273 else if (Prcb
->CpuType
== 5)
275 /* On P5, enable workaround for the LOCK errata. */
276 KiI386PentiumLockErrataPresent
= TRUE
;
279 /* Check for broken P6 with bad SMP PTE implementation */
280 if (((Reg
[0] & 0x0FF0) == 0x0610 && (Reg
[0] & 0x000F) <= 0x9) ||
281 ((Reg
[0] & 0x0FF0) == 0x0630 && (Reg
[0] & 0x000F) <= 0x4))
283 /* Remove support for correct PTE support. */
284 FeatureBits
&= ~KF_WORKING_PTE
;
287 /* Check if the CPU is too old to support SYSENTER */
288 if ((Prcb
->CpuType
< 6) ||
289 ((Prcb
->CpuType
== 6) && (Prcb
->CpuStep
< 0x0303)))
295 /* Set the current features */
296 CpuFeatures
= Reg
[3];
303 /* Check if this is a K5 or K6. (family 5) */
304 if ((Reg
[0] & 0x0F00) == 0x0500)
306 /* Get the Model Number */
307 switch (Reg
[0] & 0x00F0)
309 /* Model 1: K5 - 5k86 (initial models) */
312 /* Check if this is Step 0 or 1. They don't support PGE */
313 if ((Reg
[0] & 0x000F) > 0x03) break;
315 /* Model 0: K5 - SSA5 */
318 /* Model 0 doesn't support PGE at all. */
325 /* K6-2, Step 8 and over have support for MTRR. */
326 if ((Reg
[0] & 0x000F) >= 0x8) FeatureBits
|= KF_AMDK6MTRR
;
330 Model D: K6-2+, K6-III+ */
334 FeatureBits
|= KF_AMDK6MTRR
;
338 else if((Reg
[0] & 0x0F00) < 0x0500)
340 /* Families below 5 don't support PGE, PSE or CMOV at all */
341 Reg
[3] &= ~(0x08 | 0x2000 | 0x8000);
343 /* They also don't support advanced CPUID functions. */
344 ExtendedCPUID
= FALSE
;
347 /* Set the current features */
348 CpuFeatures
= Reg
[3];
355 /* FIXME: CMPXCGH8B */
362 /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
363 if ((Reg
[0] & 0x0FFF) >= 0x0542)
365 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
366 FeatureBits
|= KF_CMPXCHG8B
;
371 /* Centaur, IDT, Rise and VIA CPUs */
375 /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
376 However, this feature exists and operates properly without any additional steps. */
377 FeatureBits
|= KF_CMPXCHG8B
;
382 /* Convert all CPUID Feature bits into our format */
383 if (CpuFeatures
& 0x00000002) FeatureBits
|= KF_V86_VIS
| KF_CR4
;
384 if (CpuFeatures
& 0x00000008) FeatureBits
|= KF_LARGE_PAGE
| KF_CR4
;
385 if (CpuFeatures
& 0x00000010) FeatureBits
|= KF_RDTSC
;
386 if (CpuFeatures
& 0x00000100) FeatureBits
|= KF_CMPXCHG8B
;
387 if (CpuFeatures
& 0x00000800) FeatureBits
|= KF_FAST_SYSCALL
;
388 if (CpuFeatures
& 0x00001000) FeatureBits
|= KF_MTRR
;
389 if (CpuFeatures
& 0x00002000) FeatureBits
|= KF_GLOBAL_PAGE
| KF_CR4
;
390 if (CpuFeatures
& 0x00008000) FeatureBits
|= KF_CMOV
;
391 if (CpuFeatures
& 0x00010000) FeatureBits
|= KF_PAT
;
392 if (CpuFeatures
& 0x00200000) FeatureBits
|= KF_DTS
;
393 if (CpuFeatures
& 0x00800000) FeatureBits
|= KF_MMX
;
394 if (CpuFeatures
& 0x01000000) FeatureBits
|= KF_FXSR
;
395 if (CpuFeatures
& 0x02000000) FeatureBits
|= KF_XMMI
;
396 if (CpuFeatures
& 0x04000000) FeatureBits
|= KF_XMMI64
;
398 /* Check if the CPU has hyper-threading */
399 if (CpuFeatures
& 0x10000000)
401 /* Set the number of logical CPUs */
402 Prcb
->LogicalProcessorsPerPhysicalProcessor
= (UCHAR
)(Reg
[1] >> 16);
403 if (Prcb
->LogicalProcessorsPerPhysicalProcessor
> 1)
405 /* We're on dual-core */
406 KiSMTProcessorsPresent
= TRUE
;
411 /* We only have a single CPU */
412 Prcb
->LogicalProcessorsPerPhysicalProcessor
= 1;
415 /* Check if CPUID 0x80000000 is supported */
419 CPUID(0x80000000, &Reg
[0], &Dummy
, &Dummy
, &Dummy
);
420 if ((Reg
[0] & 0xffffff00) == 0x80000000)
422 /* Check if CPUID 0x80000001 is supported */
423 if (Reg
[0] >= 0x80000001)
425 /* Check which extended features are available. */
426 CPUID(0x80000001, &Dummy
, &Dummy
, &Dummy
, &Reg
[3]);
428 /* Check if NX-bit is supported */
429 if (Reg
[3] & 0x00100000) FeatureBits
|= KF_NX_BIT
;
431 /* Now handle each features for each CPU Vendor */
436 if (Reg
[3] & 0x80000000) FeatureBits
|= KF_3DNOW
;
443 /* Return the Feature Bits */
449 KiGetCacheInformation(VOID
)
451 PKIPCR Pcr
= (PKIPCR
)KeGetPcr();
453 ULONG Data
[4], Dummy
;
454 ULONG CacheRequests
= 0, i
;
455 ULONG CurrentRegister
;
457 ULONG Size
, Associativity
= 0, CacheLine
= 64, CurrentSize
= 0;
458 BOOLEAN FirstPass
= TRUE
;
460 /* Set default L2 size */
461 Pcr
->SecondLevelCacheSize
= 0;
463 /* Get the Vendor ID and make sure we support CPUID */
464 Vendor
= KiGetCpuVendor();
467 /* Check the Vendor ID */
470 /* Handle Intel case */
473 /*Check if we support CPUID 2 */
474 CPUID(0, &Data
[0], &Dummy
, &Dummy
, &Dummy
);
477 /* We need to loop for the number of times CPUID will tell us to */
480 /* Do the CPUID call */
481 CPUID(2, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
483 /* Check if it was the first call */
487 * The number of times to loop is the first byte. Read
488 * it and then destroy it so we don't get confused.
490 CacheRequests
= Data
[0] & 0xFF;
491 Data
[0] &= 0xFFFFFF00;
493 /* Don't go over this again */
497 /* Loop all 4 registers */
498 for (i
= 0; i
< 4; i
++)
500 /* Get the current register */
501 CurrentRegister
= Data
[i
];
504 * If the upper bit is set, then this register should
507 if (CurrentRegister
& 0x80000000) continue;
509 /* Keep looping for every byte inside this register */
510 while (CurrentRegister
)
512 /* Read a byte, skip a byte. */
513 RegisterByte
= (UCHAR
)(CurrentRegister
& 0xFF);
514 CurrentRegister
>>= 8;
515 if (!RegisterByte
) continue;
518 * Valid values are from 0x40 (0 bytes) to 0x49
519 * (32MB), or from 0x80 to 0x89 (same size but
522 if (((RegisterByte
> 0x40) && (RegisterByte
<= 0x47)) ||
523 ((RegisterByte
> 0x78) && (RegisterByte
<= 0x7C)) ||
524 ((RegisterByte
> 0x80) && (RegisterByte
<= 0x85)))
526 /* Compute associativity */
528 if (RegisterByte
>= 0x79) Associativity
= 8;
530 /* Mask out only the first nibble */
531 RegisterByte
&= 0x07;
533 /* Check if this cache is bigger than the last */
534 Size
= 0x10000 << RegisterByte
;
535 if ((Size
/ Associativity
) > CurrentSize
)
537 /* Set the L2 Cache Size and Associativity */
538 CurrentSize
= Size
/ Associativity
;
539 Pcr
->SecondLevelCacheSize
= Size
;
540 Pcr
->SecondLevelCacheAssociativity
= Associativity
;
543 else if ((RegisterByte
> 0x21) && (RegisterByte
<= 0x29))
545 /* Set minimum cache line size */
546 if (CacheLine
< 128) CacheLine
= 128;
548 /* Hard-code size/associativity */
550 switch (RegisterByte
)
574 /* Check if this cache is bigger than the last */
575 if ((Size
/ Associativity
) > CurrentSize
)
577 /* Set the L2 Cache Size and Associativity */
578 CurrentSize
= Size
/ Associativity
;
579 Pcr
->SecondLevelCacheSize
= Size
;
580 Pcr
->SecondLevelCacheAssociativity
= Associativity
;
583 else if (((RegisterByte
> 0x65) && (RegisterByte
< 0x69)) ||
584 (RegisterByte
== 0x2C) || (RegisterByte
== 0xF0))
586 /* Indicates L1 cache line of 64 bytes */
587 KePrefetchNTAGranularity
= 64;
589 else if (RegisterByte
== 0xF1)
591 /* Indicates L1 cache line of 128 bytes */
592 KePrefetchNTAGranularity
= 128;
594 else if (((RegisterByte
>= 0x4A) && (RegisterByte
<= 0x4C)) ||
595 (RegisterByte
== 0x78) ||
596 (RegisterByte
== 0x7D) ||
597 (RegisterByte
== 0x7F) ||
598 (RegisterByte
== 0x86) ||
599 (RegisterByte
== 0x87))
601 /* Set minimum cache line size */
602 if (CacheLine
< 64) CacheLine
= 64;
604 /* Hard-code size/associativity */
605 switch (RegisterByte
)
608 Size
= 4 * 1024 * 1024;
613 Size
= 6 * 1024 * 1024;
618 Size
= 8 * 1024 * 1024;
623 Size
= 1 * 1024 * 1024;
628 Size
= 2 * 1024 * 1024;
643 Size
= 1 * 1024 * 1024;
652 /* Check if this cache is bigger than the last */
653 if ((Size
/ Associativity
) > CurrentSize
)
655 /* Set the L2 Cache Size and Associativity */
656 CurrentSize
= Size
/ Associativity
;
657 Pcr
->SecondLevelCacheSize
= Size
;
658 Pcr
->SecondLevelCacheAssociativity
= Associativity
;
663 } while (--CacheRequests
);
669 /* Check if we support CPUID 0x80000005 */
670 CPUID(0x80000000, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
671 if (Data
[0] >= 0x80000006)
673 /* Get L1 size first */
674 CPUID(0x80000005, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
675 KePrefetchNTAGranularity
= Data
[2] & 0xFF;
677 /* Check if we support CPUID 0x80000006 */
678 CPUID(0x80000000, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
679 if (Data
[0] >= 0x80000006)
681 /* Get 2nd level cache and tlb size */
682 CPUID(0x80000006, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
684 /* Cache line size */
685 CacheLine
= Data
[2] & 0xFF;
687 /* Hardcode associativity */
688 RegisterByte
= Data
[2] >> 12;
689 switch (RegisterByte
)
714 Size
= (Data
[2] >> 16) << 10;
716 /* Hack for Model 6, Steping 300 */
717 if ((KeGetCurrentPrcb()->CpuType
== 6) &&
718 (KeGetCurrentPrcb()->CpuStep
== 0x300))
720 /* Stick 64K in there */
724 /* Set the L2 Cache Size and associativity */
725 Pcr
->SecondLevelCacheSize
= Size
;
726 Pcr
->SecondLevelCacheAssociativity
= Associativity
;
740 /* Set the cache line */
741 if (CacheLine
> KeLargestCacheLine
) KeLargestCacheLine
= CacheLine
;
742 DPRINT1("Prefetch Cache: %d bytes\tL2 Cache: %d bytes\tL2 Cache Line: %d bytes\tL2 Cache Associativity: %d\n",
743 KePrefetchNTAGranularity
,
744 Pcr
->SecondLevelCacheSize
,
746 Pcr
->SecondLevelCacheAssociativity
);
755 /* Save current CR0 */
758 /* If this is a 486, enable Write-Protection */
759 if (KeGetCurrentPrcb()->CpuType
> 3) Cr0
|= CR0_WP
;
767 KiInitializeTSS2(IN PKTSS Tss
,
768 IN PKGDTENTRY TssEntry OPTIONAL
)
772 /* Make sure the GDT Entry is valid */
776 TssEntry
->LimitLow
= sizeof(KTSS
) - 1;
777 TssEntry
->HighWord
.Bits
.LimitHi
= 0;
780 /* Now clear the I/O Map */
781 ASSERT(IOPM_COUNT
== 1);
782 RtlFillMemory(Tss
->IoMaps
[0].IoMap
, IOPM_FULL_SIZE
, 0xFF);
784 /* Initialize Interrupt Direction Maps */
785 p
= (PUCHAR
)(Tss
->IoMaps
[0].DirectionMap
);
786 RtlZeroMemory(p
, IOPM_DIRECTION_MAP_SIZE
);
788 /* Add DPMI support for interrupts */
793 /* Initialize the default Interrupt Direction Map */
794 p
= Tss
->IntDirectionMap
;
795 RtlZeroMemory(Tss
->IntDirectionMap
, IOPM_DIRECTION_MAP_SIZE
);
797 /* Add DPMI support */
805 KiInitializeTSS(IN PKTSS Tss
)
807 /* Set an invalid map base */
808 Tss
->IoMapBase
= KiComputeIopmOffset(IO_ACCESS_MAP_NONE
);
810 /* Disable traps during Task Switches */
813 /* Set LDT and Ring 0 SS */
815 Tss
->Ss0
= KGDT_R0_DATA
;
820 Ki386InitializeTss(IN PKTSS Tss
,
824 PKGDTENTRY TssEntry
, TaskGateEntry
;
826 /* Initialize the boot TSS. */
827 TssEntry
= &Gdt
[KGDT_TSS
/ sizeof(KGDTENTRY
)];
828 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
829 TssEntry
->HighWord
.Bits
.Pres
= 1;
830 TssEntry
->HighWord
.Bits
.Dpl
= 0;
831 KiInitializeTSS2(Tss
, TssEntry
);
832 KiInitializeTSS(Tss
);
834 /* Load the task register */
835 Ke386SetTr(KGDT_TSS
);
837 /* Setup the Task Gate for Double Fault Traps */
838 TaskGateEntry
= (PKGDTENTRY
)&Idt
[8];
839 TaskGateEntry
->HighWord
.Bits
.Type
= I386_TASK_GATE
;
840 TaskGateEntry
->HighWord
.Bits
.Pres
= 1;
841 TaskGateEntry
->HighWord
.Bits
.Dpl
= 0;
842 ((PKIDTENTRY
)TaskGateEntry
)->Selector
= KGDT_DF_TSS
;
844 /* Initialize the TSS used for handling double faults. */
845 Tss
= (PKTSS
)KiDoubleFaultTSS
;
846 KiInitializeTSS(Tss
);
847 Tss
->CR3
= __readcr3();
848 Tss
->Esp0
= KiDoubleFaultStack
;
849 Tss
->Esp
= KiDoubleFaultStack
;
850 Tss
->Eip
= PtrToUlong(KiTrap08
);
851 Tss
->Cs
= KGDT_R0_CODE
;
852 Tss
->Fs
= KGDT_R0_PCR
;
853 Tss
->Ss
= Ke386GetSs();
854 Tss
->Es
= KGDT_R3_DATA
| RPL_MASK
;
855 Tss
->Ds
= KGDT_R3_DATA
| RPL_MASK
;
857 /* Setup the Double Trap TSS entry in the GDT */
858 TssEntry
= &Gdt
[KGDT_DF_TSS
/ sizeof(KGDTENTRY
)];
859 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
860 TssEntry
->HighWord
.Bits
.Pres
= 1;
861 TssEntry
->HighWord
.Bits
.Dpl
= 0;
862 TssEntry
->BaseLow
= (USHORT
)((ULONG_PTR
)Tss
& 0xFFFF);
863 TssEntry
->HighWord
.Bytes
.BaseMid
= (UCHAR
)((ULONG_PTR
)Tss
>> 16);
864 TssEntry
->HighWord
.Bytes
.BaseHi
= (UCHAR
)((ULONG_PTR
)Tss
>> 24);
865 TssEntry
->LimitLow
= KTSS_IO_MAPS
;
867 /* Now setup the NMI Task Gate */
868 TaskGateEntry
= (PKGDTENTRY
)&Idt
[2];
869 TaskGateEntry
->HighWord
.Bits
.Type
= I386_TASK_GATE
;
870 TaskGateEntry
->HighWord
.Bits
.Pres
= 1;
871 TaskGateEntry
->HighWord
.Bits
.Dpl
= 0;
872 ((PKIDTENTRY
)TaskGateEntry
)->Selector
= KGDT_NMI_TSS
;
874 /* Initialize the actual TSS */
875 Tss
= (PKTSS
)KiNMITSS
;
876 KiInitializeTSS(Tss
);
877 Tss
->CR3
= __readcr3();
878 Tss
->Esp0
= KiDoubleFaultStack
;
879 Tss
->Esp
= KiDoubleFaultStack
;
880 Tss
->Eip
= PtrToUlong(KiTrap02
);
881 Tss
->Cs
= KGDT_R0_CODE
;
882 Tss
->Fs
= KGDT_R0_PCR
;
883 Tss
->Ss
= Ke386GetSs();
884 Tss
->Es
= KGDT_R3_DATA
| RPL_MASK
;
885 Tss
->Ds
= KGDT_R3_DATA
| RPL_MASK
;
887 /* And its associated TSS Entry */
888 TssEntry
= &Gdt
[KGDT_NMI_TSS
/ sizeof(KGDTENTRY
)];
889 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
890 TssEntry
->HighWord
.Bits
.Pres
= 1;
891 TssEntry
->HighWord
.Bits
.Dpl
= 0;
892 TssEntry
->BaseLow
= (USHORT
)((ULONG_PTR
)Tss
& 0xFFFF);
893 TssEntry
->HighWord
.Bytes
.BaseMid
= (UCHAR
)((ULONG_PTR
)Tss
>> 16);
894 TssEntry
->HighWord
.Bytes
.BaseHi
= (UCHAR
)((ULONG_PTR
)Tss
>> 24);
895 TssEntry
->LimitLow
= KTSS_IO_MAPS
;
900 KeFlushCurrentTb(VOID
)
902 /* Flush the TLB by resetting CR3 */
903 __writecr3(__readcr3());
908 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState
)
913 // Restore the CR registers
915 __writecr0(ProcessorState
->SpecialRegisters
.Cr0
);
916 Ke386SetCr2(ProcessorState
->SpecialRegisters
.Cr2
);
917 __writecr3(ProcessorState
->SpecialRegisters
.Cr3
);
918 if (KeFeatureBits
& KF_CR4
) __writecr4(ProcessorState
->SpecialRegisters
.Cr4
);
921 // Restore the DR registers
923 __writedr(0, ProcessorState
->SpecialRegisters
.KernelDr0
);
924 __writedr(1, ProcessorState
->SpecialRegisters
.KernelDr1
);
925 __writedr(2, ProcessorState
->SpecialRegisters
.KernelDr2
);
926 __writedr(3, ProcessorState
->SpecialRegisters
.KernelDr3
);
927 __writedr(6, ProcessorState
->SpecialRegisters
.KernelDr6
);
928 __writedr(7, ProcessorState
->SpecialRegisters
.KernelDr7
);
931 // Restore GDT and IDT
933 Ke386SetGlobalDescriptorTable(&ProcessorState
->SpecialRegisters
.Gdtr
.Limit
);
934 __lidt(&ProcessorState
->SpecialRegisters
.Idtr
.Limit
);
937 // Clear the busy flag so we don't crash if we reload the same selector
939 TssEntry
= (PKGDTENTRY
)(ProcessorState
->SpecialRegisters
.Gdtr
.Base
+
940 ProcessorState
->SpecialRegisters
.Tr
);
941 TssEntry
->HighWord
.Bytes
.Flags1
&= ~0x2;
944 // Restore TSS and LDT
946 Ke386SetTr(ProcessorState
->SpecialRegisters
.Tr
);
947 Ke386SetLocalDescriptorTable(ProcessorState
->SpecialRegisters
.Ldtr
);
952 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState
)
954 /* Save the CR registers */
955 ProcessorState
->SpecialRegisters
.Cr0
= __readcr0();
956 ProcessorState
->SpecialRegisters
.Cr2
= __readcr2();
957 ProcessorState
->SpecialRegisters
.Cr3
= __readcr3();
958 ProcessorState
->SpecialRegisters
.Cr4
= (KeFeatureBits
& KF_CR4
) ?
961 /* Save the DR registers */
962 ProcessorState
->SpecialRegisters
.KernelDr0
= __readdr(0);
963 ProcessorState
->SpecialRegisters
.KernelDr1
= __readdr(1);
964 ProcessorState
->SpecialRegisters
.KernelDr2
= __readdr(2);
965 ProcessorState
->SpecialRegisters
.KernelDr3
= __readdr(3);
966 ProcessorState
->SpecialRegisters
.KernelDr6
= __readdr(6);
967 ProcessorState
->SpecialRegisters
.KernelDr7
= __readdr(7);
970 /* Save GDT, IDT, LDT and TSS */
971 Ke386GetGlobalDescriptorTable(&ProcessorState
->SpecialRegisters
.Gdtr
.Limit
);
972 __sidt(&ProcessorState
->SpecialRegisters
.Idtr
.Limit
);
973 ProcessorState
->SpecialRegisters
.Tr
= Ke386GetTr();
974 ProcessorState
->SpecialRegisters
.Ldtr
= Ke386GetLocalDescriptorTable();
979 KiInitializeMachineType(VOID
)
981 /* Set the Machine Type we got from NTLDR */
982 KeI386MachineType
= KeLoaderBlock
->u
.I386
.MachineType
& 0x000FF;
987 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context
)
990 WRMSR(0x174, KGDT_R0_CODE
);
991 WRMSR(0x175, (ULONG_PTR
)KeGetCurrentPrcb()->DpcStack
);
994 WRMSR(0x176, (ULONG_PTR
)KiFastCallEntry
);
1000 KiRestoreFastSyscallReturnState(VOID
)
1002 /* Check if the CPU Supports fast system call */
1003 if (KeFeatureBits
& KF_FAST_SYSCALL
)
1005 /* Check if it has been disabled */
1006 if (!KiFastSystemCallDisable
)
1008 /* Do an IPI to enable it */
1009 KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters
, 0);
1011 /* It's enabled, so use the proper exit stub */
1012 KiFastCallExitHandler
= KiSystemCallSysExitReturn
;
1013 DPRINT1("Support for SYSENTER detected.\n");
1017 /* Disable fast system call */
1018 KeFeatureBits
&= ~KF_FAST_SYSCALL
;
1019 KiFastCallExitHandler
= KiSystemCallTrapReturn
;
1020 DPRINT1("Support for SYSENTER disabled.\n");
1025 /* Use the IRET handler */
1026 KiFastCallExitHandler
= KiSystemCallTrapReturn
;
1027 DPRINT1("No support for SYSENTER detected.\n");
1033 Ki386EnableDE(IN ULONG_PTR Context
)
1036 __writecr4(__readcr4() | CR4_DE
);
1042 Ki386EnableFxsr(IN ULONG_PTR Context
)
1045 __writecr4(__readcr4() | CR4_FXSR
);
1051 Ki386EnableXMMIExceptions(IN ULONG_PTR Context
)
1053 PKIDTENTRY IdtEntry
;
1055 /* Get the IDT Entry for Interrupt 0x13 */
1056 IdtEntry
= &((PKIPCR
)KeGetPcr())->IDT
[0x13];
1059 IdtEntry
->Selector
= KGDT_R0_CODE
;
1060 IdtEntry
->Offset
= ((ULONG_PTR
)KiTrap13
& 0xFFFF);
1061 IdtEntry
->ExtendedOffset
= ((ULONG_PTR
)KiTrap13
>> 16) & 0xFFFF;
1062 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->Dpl
= 0;
1063 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->Present
= 1;
1064 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->SegmentType
= I386_INTERRUPT_GATE
;
1066 /* Enable XMMI exceptions */
1067 __writecr4(__readcr4() | CR4_XMMEXCPT
);
1073 KiI386PentiumLockErrataFixup(VOID
)
1075 KDESCRIPTOR IdtDescriptor
;
1076 PKIDTENTRY NewIdt
, NewIdt2
;
1078 /* Allocate memory for a new IDT */
1079 NewIdt
= ExAllocatePool(NonPagedPool
, 2 * PAGE_SIZE
);
1081 /* Put everything after the first 7 entries on a new page */
1082 NewIdt2
= (PVOID
)((ULONG_PTR
)NewIdt
+ PAGE_SIZE
- (7 * sizeof(KIDTENTRY
)));
1084 /* Disable interrupts */
1087 /* Get the current IDT and copy it */
1088 __sidt(&IdtDescriptor
.Limit
);
1089 RtlCopyMemory(NewIdt2
,
1090 (PVOID
)IdtDescriptor
.Base
,
1091 IdtDescriptor
.Limit
+ 1);
1092 IdtDescriptor
.Base
= (ULONG
)NewIdt2
;
1094 /* Set the new IDT */
1095 __lidt(&IdtDescriptor
.Limit
);
1096 ((PKIPCR
)KeGetPcr())->IDT
= NewIdt2
;
1098 /* Restore interrupts */
1101 /* Set the first 7 entries as read-only to produce a fault */
1102 MmSetPageProtect(NULL
, NewIdt
, PAGE_READONLY
);
1107 KeDisableInterrupts(VOID
)
1112 /* Get EFLAGS and check if the interrupt bit is set */
1113 Flags
= __readeflags();
1114 Return
= (Flags
& EFLAGS_INTERRUPT_MASK
) ? TRUE
: FALSE
;
1116 /* Disable interrupts */
1123 KeInvalidateAllCaches(VOID
)
1125 /* Only supported on Pentium Pro and higher */
1126 if (KeI386CpuType
< 6) return FALSE
;
1128 /* Invalidate all caches */
1135 KeZeroPages(IN PVOID Address
,
1138 /* Not using XMMI in this routine */
1139 RtlZeroMemory(Address
, Size
);
1144 KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame
,
1145 IN PKEXCEPTION_FRAME ExceptionFrame
)
1147 PKPRCB Prcb
= KeGetCurrentPrcb();
1150 // Save full context
1152 Prcb
->ProcessorState
.ContextFrame
.ContextFlags
= CONTEXT_FULL
|
1153 CONTEXT_DEBUG_REGISTERS
;
1154 KeTrapFrameToContext(TrapFrame
, NULL
, &Prcb
->ProcessorState
.ContextFrame
);
1157 // Save control registers
1159 KiSaveProcessorControlState(&Prcb
->ProcessorState
);
1164 KiIsNpxPresent(VOID
)
1172 /* Read CR0 and mask out FPU flags */
1173 Cr0
= __readcr0() & ~(CR0_MP
| CR0_TS
| CR0_EM
| CR0_ET
);
1175 /* Store on FPU stack */
1176 asm volatile ("fninit;" "fnstsw %0" : "+m"(Magic
));
1178 /* Magic should now be cleared */
1181 /* You don't have an FPU -- enable emulation for now */
1182 __writecr0(Cr0
| CR0_EM
| CR0_TS
);
1186 /* You have an FPU, enable it */
1189 /* Enable INT 16 on 486 and higher */
1190 if (KeGetCurrentPrcb()->CpuType
>= 3) Cr0
|= CR0_NE
;
1193 __writecr0(Cr0
| CR0_EM
| CR0_TS
);
1199 KiIsNpxErrataPresent(VOID
)
1201 BOOLEAN ErrataPresent
;
1203 volatile double Value1
, Value2
;
1205 /* Disable interrupts */
1208 /* Read CR0 and remove FPU flags */
1210 __writecr0(Cr0
& ~(CR0_MP
| CR0_TS
| CR0_EM
));
1212 /* Initialize FPU state */
1213 asm volatile ("fninit");
1215 /* Multiply the magic values and divide, we should get the result back */
1218 ErrataPresent
= (Value1
* Value2
/ 3145727.0) != 4195835.0;
1223 /* Enable interrupts */
1226 /* Return if there's an errata */
1227 return ErrataPresent
;
1232 KiFlushNPXState(IN PFLOATING_SAVE_AREA SaveArea
)
1235 PKTHREAD Thread
, NpxThread
;
1236 PFX_SAVE_AREA FxSaveArea
;
1238 /* Save volatiles and disable interrupts */
1239 EFlags
= __readeflags();
1242 /* Save the PCR and get the current thread */
1243 Thread
= KeGetCurrentThread();
1245 /* Check if we're already loaded */
1246 if (Thread
->NpxState
!= NPX_STATE_LOADED
)
1248 /* If there's nothing to load, quit */
1249 if (!SaveArea
) return;
1251 /* Need FXSR support for this */
1252 ASSERT(KeI386FxsrPresent
== TRUE
);
1254 /* Check for sane CR0 */
1256 if (Cr0
& (CR0_MP
| CR0_TS
| CR0_EM
))
1258 /* Mask out FPU flags */
1259 __writecr0(Cr0
& ~(CR0_MP
| CR0_TS
| CR0_EM
));
1262 /* Get the NPX thread and check its FPU state */
1263 NpxThread
= KeGetCurrentPrcb()->NpxThread
;
1264 if ((NpxThread
) && (NpxThread
->NpxState
== NPX_STATE_LOADED
))
1266 /* Get the FX frame and store the state there */
1267 FxSaveArea
= KiGetThreadNpxArea(NpxThread
);
1268 Ke386FxSave(FxSaveArea
);
1270 /* NPX thread has lost its state */
1271 NpxThread
->NpxState
= NPX_STATE_NOT_LOADED
;
1274 /* Now load NPX state from the NPX area */
1275 FxSaveArea
= KiGetThreadNpxArea(Thread
);
1276 Ke386FxStore(FxSaveArea
);
1280 /* Check for sane CR0 */
1282 if (Cr0
& (CR0_MP
| CR0_TS
| CR0_EM
))
1284 /* Mask out FPU flags */
1285 __writecr0(Cr0
& ~(CR0_MP
| CR0_TS
| CR0_EM
));
1289 FxSaveArea
= KiGetThreadNpxArea(Thread
);
1290 Thread
->NpxState
= NPX_STATE_NOT_LOADED
;
1292 /* Save state if supported by CPU */
1293 if (KeI386FxsrPresent
) Ke386FxSave(FxSaveArea
);
1296 /* Now save the FN state wherever it was requested */
1297 if (SaveArea
) Ke386FnSave(SaveArea
);
1299 /* Clear NPX thread */
1300 KeGetCurrentPrcb()->NpxThread
= NULL
;
1302 /* Add the CR0 from the NPX frame */
1303 Cr0
|= NPX_STATE_NOT_LOADED
;
1304 Cr0
|= FxSaveArea
->Cr0NpxState
;
1307 /* Restore interrupt state */
1308 __writeeflags(EFlags
);
1311 /* PUBLIC FUNCTIONS **********************************************************/
1318 KiCoprocessorError(VOID
)
1320 PFX_SAVE_AREA NpxArea
;
1322 /* Get the FPU area */
1323 NpxArea
= KiGetThreadNpxArea(KeGetCurrentThread());
1326 NpxArea
->Cr0NpxState
= CR0_TS
;
1327 __writecr0(__readcr0() | CR0_TS
);
1335 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save
)
1337 PFNSAVE_FORMAT FpState
;
1338 ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL
);
1339 DPRINT1("%s is not really implemented\n", __FUNCTION__
);
1341 /* check if we are doing software emulation */
1342 if (!KeI386NpxPresent
) return STATUS_ILLEGAL_FLOAT_CONTEXT
;
1344 FpState
= ExAllocatePool(NonPagedPool
, sizeof (FNSAVE_FORMAT
));
1345 if (!FpState
) return STATUS_INSUFFICIENT_RESOURCES
;
1347 *((PVOID
*) Save
) = FpState
;
1349 asm volatile("fnsave %0\n\t" : "=m" (*FpState
));
1357 KeGetCurrentThread()->DispatcherHeader
.NpxIrql
= KeGetCurrentIrql();
1358 return STATUS_SUCCESS
;
1366 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save
)
1368 PFNSAVE_FORMAT FpState
= *((PVOID
*) Save
);
1369 ASSERT(KeGetCurrentThread()->DispatcherHeader
.NpxIrql
== KeGetCurrentIrql());
1370 DPRINT1("%s is not really implemented\n", __FUNCTION__
);
1373 asm volatile("fnclex\n\t");
1374 asm volatile("frstor %0\n\t" : "=m" (*FpState
));
1383 ExFreePool(FpState
);
1384 return STATUS_SUCCESS
;
1392 KeGetRecommendedSharedDataAlignment(VOID
)
1394 /* Return the global variable */
1395 return KeLargestCacheLine
;
1400 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext
,
1405 /* Signal this packet as done */
1406 KiIpiSignalPacketDone(PacketContext
);
1408 /* Flush the TB for the Current CPU */
1417 KeFlushEntireTb(IN BOOLEAN Invalid
,
1418 IN BOOLEAN AllProcessors
)
1422 KAFFINITY TargetAffinity
;
1423 PKPRCB Prcb
= KeGetCurrentPrcb();
1426 /* Raise the IRQL for the TB Flush */
1427 OldIrql
= KeRaiseIrqlToSynchLevel();
1430 /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1432 /* Get the current processor affinity, and exclude ourselves */
1433 TargetAffinity
= KeActiveProcessors
;
1434 TargetAffinity
&= ~Prcb
->SetMember
;
1436 /* Make sure this is MP */
1439 /* Send an IPI TB flush to the other processors */
1440 KiIpiSendPacket(TargetAffinity
,
1441 KiFlushTargetEntireTb
,
1448 /* Flush the TB for the Current CPU, and update the flush stamp */
1452 /* If this is MP, wait for the other processors to finish */
1456 ASSERT(Prcb
== (volatile PKPRCB
)KeGetCurrentPrcb());
1459 ASSERTMSG("Not yet implemented\n", FALSE
);
1463 /* Update the flush stamp and return to original IRQL */
1464 InterlockedExchangeAdd(&KiTbFlushTimeStamp
, 1);
1465 KeLowerIrql(OldIrql
);
1473 KeSetDmaIoCoherency(IN ULONG Coherency
)
1475 /* Save the coherency globally */
1476 KiDmaIoCoherency
= Coherency
;
1484 KeQueryActiveProcessors(VOID
)
1488 /* Simply return the number of active processors */
1489 return KeActiveProcessors
;
1497 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State
)
1499 /* Capture the context */
1500 RtlCaptureContext(&State
->ContextFrame
);
1502 /* Capture the control state */
1503 KiSaveProcessorControlState(State
);