2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/i386/cpu.c
5 * PURPOSE: Routines for CPU-level support
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
9 /* INCLUDES *****************************************************************/
15 /* GLOBALS *******************************************************************/
20 /* The TSS to use for Double Fault Traps (INT 0x9) */
21 UCHAR KiDoubleFaultTSS
[KTSS_IO_MAPS
];
23 /* The TSS to use for NMI Fault Traps (INT 0x2) */
24 UCHAR KiNMITSS
[KTSS_IO_MAPS
];
27 KGDTENTRY KiBootGdt
[256] =
29 {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}}, /* KGDT_NULL */
30 {0xffff, 0x0000, {{0x00, 0x9b, 0xcf, 0x00}}}, /* KGDT_R0_CODE */
31 {0xffff, 0x0000, {{0x00, 0x93, 0xcf, 0x00}}}, /* KGDT_R0_DATA */
32 {0xffff, 0x0000, {{0x00, 0xfb, 0xcf, 0x00}}}, /* KGDT_R3_CODE */
33 {0xffff, 0x0000, {{0x00, 0xf3, 0xcf, 0x00}}}, /* KGDT_R3_DATA*/
34 {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}}, /* KGDT_TSS */
35 {0x0001, 0xf000, {{0xdf, 0x93, 0xc0, 0xff}}}, /* KGDT_R0_PCR */
36 {0x0fff, 0x0000, {{0x00, 0xf3, 0x40, 0x00}}}, /* KGDT_R3_TEB */
37 {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}}, /* KGDT_UNUSED */
38 {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}}, /* KGDT_LDT */
39 {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}}, /* KGDT_DF_TSS */
40 {0x0000, 0x0000, {{0x00, 0x00, 0x00, 0x00}}} /* KGDT_NMI_TSS */
44 KDESCRIPTOR KiGdtDescriptor
= {0, sizeof(KiBootGdt
) - 1, (ULONG
)KiBootGdt
};
46 /* CPU Features and Flags */
49 ULONG KeProcessorArchitecture
;
50 ULONG KeProcessorLevel
;
51 ULONG KeProcessorRevision
;
53 ULONG KiFastSystemCallDisable
= 1;
54 ULONG KeI386NpxPresent
= 0;
55 ULONG KiMXCsrMask
= 0;
56 ULONG MxcsrFeatureMask
= 0;
57 ULONG KeI386XMMIPresent
= 0;
58 ULONG KeI386FxsrPresent
= 0;
59 ULONG KeI386MachineType
;
60 ULONG Ke386Pae
= FALSE
;
61 ULONG Ke386NoExecute
= FALSE
;
62 ULONG KeLargestCacheLine
= 0x40;
63 ULONG KeDcacheFlushCount
= 0;
64 ULONG KeIcacheFlushCount
= 0;
65 ULONG KiDmaIoCoherency
= 0;
66 CHAR KeNumberProcessors
;
67 KAFFINITY KeActiveProcessors
= 1;
68 BOOLEAN KiI386PentiumLockErrataPresent
;
69 BOOLEAN KiSMTProcessorsPresent
;
76 volatile LONG KiTbFlushTimeStamp
;
79 static const CHAR CmpIntelID
[] = "GenuineIntel";
80 static const CHAR CmpAmdID
[] = "AuthenticAMD";
81 static const CHAR CmpCyrixID
[] = "CyrixInstead";
82 static const CHAR CmpTransmetaID
[] = "GenuineTMx86";
83 static const CHAR CmpCentaurID
[] = "CentaurHauls";
84 static const CHAR CmpRiseID
[] = "RiseRiseRise";
86 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
90 CPUID(IN ULONG InfoType
,
91 OUT PULONG CpuInfoEax
,
92 OUT PULONG CpuInfoEbx
,
93 OUT PULONG CpuInfoEcx
,
94 OUT PULONG CpuInfoEdx
)
98 /* Perform the CPUID Operation */
99 __cpuid((int*)CpuInfo
, InfoType
);
101 /* Return the results */
102 *CpuInfoEax
= CpuInfo
[0];
103 *CpuInfoEbx
= CpuInfo
[1];
104 *CpuInfoEcx
= CpuInfo
[2];
105 *CpuInfoEdx
= CpuInfo
[3];
110 WRMSR(IN ULONG Register
,
113 /* Write to the MSR */
114 __writemsr(Register
, Value
);
119 RDMSR(IN ULONG Register
)
121 /* Read from the MSR */
122 return __readmsr(Register
);
125 /* FUNCTIONS *****************************************************************/
129 KiSetProcessorType(VOID
)
131 ULONG EFlags
, NewEFlags
;
133 ULONG Stepping
, Type
;
135 /* Start by assuming no CPUID data */
136 KeGetCurrentPrcb()->CpuID
= 0;
139 EFlags
= __readeflags();
141 /* XOR out the ID bit and update EFlags */
142 NewEFlags
= EFlags
^ EFLAGS_ID
;
143 __writeeflags(NewEFlags
);
145 /* Get them back and see if they were modified */
146 NewEFlags
= __readeflags();
147 if (NewEFlags
!= EFlags
)
149 /* The modification worked, so CPUID exists. Set the ID Bit again. */
151 __writeeflags(EFlags
);
153 /* Peform CPUID 0 to see if CPUID 1 is supported */
154 CPUID(0, &Reg
, &Dummy
, &Dummy
, &Dummy
);
158 CPUID(1, &Reg
, &Dummy
, &Dummy
, &Dummy
);
161 * Get the Stepping and Type. The stepping contains both the
162 * Model and the Step, while the Type contains the returned Type.
163 * We ignore the family.
165 * For the stepping, we convert this: zzzzzzxy into this: x0y
167 Stepping
= Reg
& 0xF0;
169 Stepping
+= (Reg
& 0xFF);
174 /* Save them in the PRCB */
175 KeGetCurrentPrcb()->CpuID
= TRUE
;
176 KeGetCurrentPrcb()->CpuType
= (UCHAR
)Type
;
177 KeGetCurrentPrcb()->CpuStep
= (USHORT
)Stepping
;
181 DPRINT1("CPUID Support lacking\n");
186 DPRINT1("CPUID Support lacking\n");
190 __writeeflags(EFlags
);
197 PKPRCB Prcb
= KeGetCurrentPrcb();
201 /* Assume no Vendor ID and fail if no CPUID Support. */
202 Prcb
->VendorString
[0] = 0;
203 if (!Prcb
->CpuID
) return 0;
205 /* Get the Vendor ID and null-terminate it */
206 CPUID(0, &Vendor
[0], &Vendor
[1], &Vendor
[2], &Vendor
[3]);
209 /* Re-arrange vendor string */
211 Vendor
[2] = Vendor
[3];
214 /* Copy it to the PRCB and null-terminate it again */
215 RtlCopyMemory(Prcb
->VendorString
,
217 sizeof(Prcb
->VendorString
) - sizeof(CHAR
));
218 Prcb
->VendorString
[sizeof(Prcb
->VendorString
) - sizeof(CHAR
)] = ANSI_NULL
;
220 /* Now check the CPU Type */
221 if (!strcmp(Prcb
->VendorString
, CmpIntelID
))
225 else if (!strcmp(Prcb
->VendorString
, CmpAmdID
))
229 else if (!strcmp(Prcb
->VendorString
, CmpCyrixID
))
231 DPRINT1("Cyrix CPU support not fully tested!\n");
234 else if (!strcmp(Prcb
->VendorString
, CmpTransmetaID
))
236 DPRINT1("Transmeta CPU support not fully tested!\n");
237 return CPU_TRANSMETA
;
239 else if (!strcmp(Prcb
->VendorString
, CmpCentaurID
))
241 DPRINT1("Centaur CPU support not fully tested!\n");
244 else if (!strcmp(Prcb
->VendorString
, CmpRiseID
))
246 DPRINT1("Rise CPU support not fully tested!\n");
256 KiGetFeatureBits(VOID
)
258 PKPRCB Prcb
= KeGetCurrentPrcb();
260 ULONG FeatureBits
= KF_WORKING_PTE
;
262 BOOLEAN ExtendedCPUID
= TRUE
;
263 ULONG CpuFeatures
= 0;
265 /* Get the Vendor ID */
266 Vendor
= KiGetCpuVendor();
268 /* Make sure we got a valid vendor ID at least. */
269 if (!Vendor
) return FeatureBits
;
271 /* Get the CPUID Info. Features are in Reg[3]. */
272 CPUID(1, &Reg
[0], &Reg
[1], &Dummy
, &Reg
[3]);
274 /* Set the initial APIC ID */
275 Prcb
->InitialApicId
= (UCHAR
)(Reg
[1] >> 24);
282 /* Check if it's a P6 */
283 if (Prcb
->CpuType
== 6)
285 /* Perform the special sequence to get the MicroCode Signature */
287 CPUID(1, &Dummy
, &Dummy
, &Dummy
, &Dummy
);
288 Prcb
->UpdateSignature
.QuadPart
= RDMSR(0x8B);
290 else if (Prcb
->CpuType
== 5)
292 /* On P5, enable workaround for the LOCK errata. */
293 KiI386PentiumLockErrataPresent
= TRUE
;
296 /* Check for broken P6 with bad SMP PTE implementation */
297 if (((Reg
[0] & 0x0FF0) == 0x0610 && (Reg
[0] & 0x000F) <= 0x9) ||
298 ((Reg
[0] & 0x0FF0) == 0x0630 && (Reg
[0] & 0x000F) <= 0x4))
300 /* Remove support for correct PTE support. */
301 FeatureBits
&= ~KF_WORKING_PTE
;
304 /* Check if the CPU is too old to support SYSENTER */
305 if ((Prcb
->CpuType
< 6) ||
306 ((Prcb
->CpuType
== 6) && (Prcb
->CpuStep
< 0x0303)))
312 /* Set the current features */
313 CpuFeatures
= Reg
[3];
320 /* Check if this is a K5 or K6. (family 5) */
321 if ((Reg
[0] & 0x0F00) == 0x0500)
323 /* Get the Model Number */
324 switch (Reg
[0] & 0x00F0)
326 /* Model 1: K5 - 5k86 (initial models) */
329 /* Check if this is Step 0 or 1. They don't support PGE */
330 if ((Reg
[0] & 0x000F) > 0x03) break;
332 /* Model 0: K5 - SSA5 */
335 /* Model 0 doesn't support PGE at all. */
342 /* K6-2, Step 8 and over have support for MTRR. */
343 if ((Reg
[0] & 0x000F) >= 0x8) FeatureBits
|= KF_AMDK6MTRR
;
347 Model D: K6-2+, K6-III+ */
351 FeatureBits
|= KF_AMDK6MTRR
;
355 else if((Reg
[0] & 0x0F00) < 0x0500)
357 /* Families below 5 don't support PGE, PSE or CMOV at all */
358 Reg
[3] &= ~(0x08 | 0x2000 | 0x8000);
360 /* They also don't support advanced CPUID functions. */
361 ExtendedCPUID
= FALSE
;
364 /* Set the current features */
365 CpuFeatures
= Reg
[3];
372 /* FIXME: CMPXCGH8B */
379 /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
380 if ((Reg
[0] & 0x0FFF) >= 0x0542)
382 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
383 FeatureBits
|= KF_CMPXCHG8B
;
388 /* Centaur, IDT, Rise and VIA CPUs */
392 /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
393 However, this feature exists and operates properly without any additional steps. */
394 FeatureBits
|= KF_CMPXCHG8B
;
399 /* Convert all CPUID Feature bits into our format */
400 if (CpuFeatures
& 0x00000002) FeatureBits
|= KF_V86_VIS
| KF_CR4
;
401 if (CpuFeatures
& 0x00000008) FeatureBits
|= KF_LARGE_PAGE
| KF_CR4
;
402 if (CpuFeatures
& 0x00000010) FeatureBits
|= KF_RDTSC
;
403 if (CpuFeatures
& 0x00000100) FeatureBits
|= KF_CMPXCHG8B
;
404 if (CpuFeatures
& 0x00000800) FeatureBits
|= KF_FAST_SYSCALL
;
405 if (CpuFeatures
& 0x00001000) FeatureBits
|= KF_MTRR
;
406 if (CpuFeatures
& 0x00002000) FeatureBits
|= KF_GLOBAL_PAGE
| KF_CR4
;
407 if (CpuFeatures
& 0x00008000) FeatureBits
|= KF_CMOV
;
408 if (CpuFeatures
& 0x00010000) FeatureBits
|= KF_PAT
;
409 if (CpuFeatures
& 0x00200000) FeatureBits
|= KF_DTS
;
410 if (CpuFeatures
& 0x00800000) FeatureBits
|= KF_MMX
;
411 if (CpuFeatures
& 0x01000000) FeatureBits
|= KF_FXSR
;
412 if (CpuFeatures
& 0x02000000) FeatureBits
|= KF_XMMI
;
413 if (CpuFeatures
& 0x04000000) FeatureBits
|= KF_XMMI64
;
415 /* Check if the CPU has hyper-threading */
416 if (CpuFeatures
& 0x10000000)
418 /* Set the number of logical CPUs */
419 Prcb
->LogicalProcessorsPerPhysicalProcessor
= (UCHAR
)(Reg
[1] >> 16);
420 if (Prcb
->LogicalProcessorsPerPhysicalProcessor
> 1)
422 /* We're on dual-core */
423 KiSMTProcessorsPresent
= TRUE
;
428 /* We only have a single CPU */
429 Prcb
->LogicalProcessorsPerPhysicalProcessor
= 1;
432 /* Check if CPUID 0x80000000 is supported */
436 CPUID(0x80000000, &Reg
[0], &Dummy
, &Dummy
, &Dummy
);
437 if ((Reg
[0] & 0xffffff00) == 0x80000000)
439 /* Check if CPUID 0x80000001 is supported */
440 if (Reg
[0] >= 0x80000001)
442 /* Check which extended features are available. */
443 CPUID(0x80000001, &Dummy
, &Dummy
, &Dummy
, &Reg
[3]);
445 /* Check if NX-bit is supported */
446 if (Reg
[3] & 0x00100000) FeatureBits
|= KF_NX_BIT
;
448 /* Now handle each features for each CPU Vendor */
453 if (Reg
[3] & 0x80000000) FeatureBits
|= KF_3DNOW
;
460 /* Return the Feature Bits */
466 KiGetCacheInformation(VOID
)
468 PKIPCR Pcr
= (PKIPCR
)KeGetPcr();
470 ULONG Data
[4], Dummy
;
471 ULONG CacheRequests
= 0, i
;
472 ULONG CurrentRegister
;
474 BOOLEAN FirstPass
= TRUE
;
476 /* Set default L2 size */
477 Pcr
->SecondLevelCacheSize
= 0;
479 /* Get the Vendor ID and make sure we support CPUID */
480 Vendor
= KiGetCpuVendor();
483 /* Check the Vendor ID */
486 /* Handle Intel case */
489 /*Check if we support CPUID 2 */
490 CPUID(0, &Data
[0], &Dummy
, &Dummy
, &Dummy
);
493 /* We need to loop for the number of times CPUID will tell us to */
496 /* Do the CPUID call */
497 CPUID(2, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
499 /* Check if it was the first call */
503 * The number of times to loop is the first byte. Read
504 * it and then destroy it so we don't get confused.
506 CacheRequests
= Data
[0] & 0xFF;
507 Data
[0] &= 0xFFFFFF00;
509 /* Don't go over this again */
513 /* Loop all 4 registers */
514 for (i
= 0; i
< 4; i
++)
516 /* Get the current register */
517 CurrentRegister
= Data
[i
];
520 * If the upper bit is set, then this register should
523 if (CurrentRegister
& 0x80000000) continue;
525 /* Keep looping for every byte inside this register */
526 while (CurrentRegister
)
528 /* Read a byte, skip a byte. */
529 RegisterByte
= (UCHAR
)(CurrentRegister
& 0xFF);
530 CurrentRegister
>>= 8;
531 if (!RegisterByte
) continue;
534 * Valid values are from 0x40 (0 bytes) to 0x49
535 * (32MB), or from 0x80 to 0x89 (same size but
538 if (((RegisterByte
> 0x40) &&
539 (RegisterByte
<= 0x49)) ||
540 ((RegisterByte
> 0x80) &&
541 (RegisterByte
<= 0x89)))
543 /* Mask out only the first nibble */
544 RegisterByte
&= 0x0F;
546 /* Set the L2 Cache Size */
547 Pcr
->SecondLevelCacheSize
= 0x10000 <<
552 } while (--CacheRequests
);
558 /* Check if we support CPUID 0x80000006 */
559 CPUID(0x80000000, &Data
[0], &Dummy
, &Dummy
, &Dummy
);
562 /* Get 2nd level cache and tlb size */
563 CPUID(0x80000006, &Dummy
, &Dummy
, &Data
[2], &Dummy
);
565 /* Set the L2 Cache Size */
566 Pcr
->SecondLevelCacheSize
= (Data
[2] & 0xFFFF0000) >> 6;
586 /* Save current CR0 */
589 /* If this is a 486, enable Write-Protection */
590 if (KeGetCurrentPrcb()->CpuType
> 3) Cr0
|= CR0_WP
;
598 KiInitializeTSS2(IN PKTSS Tss
,
599 IN PKGDTENTRY TssEntry OPTIONAL
)
603 /* Make sure the GDT Entry is valid */
607 TssEntry
->LimitLow
= sizeof(KTSS
) - 1;
608 TssEntry
->HighWord
.Bits
.LimitHi
= 0;
611 /* Now clear the I/O Map */
612 RtlFillMemory(Tss
->IoMaps
[0].IoMap
, 8096, -1);
614 /* Initialize Interrupt Direction Maps */
615 p
= (PUCHAR
)(Tss
->IoMaps
[0].DirectionMap
);
616 RtlZeroMemory(p
, 32);
618 /* Add DPMI support for interrupts */
623 /* Initialize the default Interrupt Direction Map */
624 p
= Tss
->IntDirectionMap
;
625 RtlZeroMemory(Tss
->IntDirectionMap
, 32);
627 /* Add DPMI support */
635 KiInitializeTSS(IN PKTSS Tss
)
637 /* Set an invalid map base */
638 Tss
->IoMapBase
= KiComputeIopmOffset(IO_ACCESS_MAP_NONE
);
640 /* Disable traps during Task Switches */
643 /* Set LDT and Ring 0 SS */
645 Tss
->Ss0
= KGDT_R0_DATA
;
650 Ki386InitializeTss(IN PKTSS Tss
,
654 PKGDTENTRY TssEntry
, TaskGateEntry
;
656 /* Initialize the boot TSS. */
657 TssEntry
= &Gdt
[KGDT_TSS
/ sizeof(KGDTENTRY
)];
658 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
659 TssEntry
->HighWord
.Bits
.Pres
= 1;
660 TssEntry
->HighWord
.Bits
.Dpl
= 0;
661 KiInitializeTSS2(Tss
, TssEntry
);
662 KiInitializeTSS(Tss
);
664 /* Load the task register */
665 Ke386SetTr(KGDT_TSS
);
667 /* Setup the Task Gate for Double Fault Traps */
668 TaskGateEntry
= (PKGDTENTRY
)&Idt
[8];
669 TaskGateEntry
->HighWord
.Bits
.Type
= I386_TASK_GATE
;
670 TaskGateEntry
->HighWord
.Bits
.Pres
= 1;
671 TaskGateEntry
->HighWord
.Bits
.Dpl
= 0;
672 ((PKIDTENTRY
)TaskGateEntry
)->Selector
= KGDT_DF_TSS
;
674 /* Initialize the TSS used for handling double faults. */
675 Tss
= (PKTSS
)KiDoubleFaultTSS
;
676 KiInitializeTSS(Tss
);
677 Tss
->CR3
= __readcr3();
678 Tss
->Esp0
= PtrToUlong(KiDoubleFaultStack
);
679 Tss
->Esp
= PtrToUlong(KiDoubleFaultStack
);
680 Tss
->Eip
= PtrToUlong(KiTrap8
);
681 Tss
->Cs
= KGDT_R0_CODE
;
682 Tss
->Fs
= KGDT_R0_PCR
;
683 Tss
->Ss
= Ke386GetSs();
684 Tss
->Es
= KGDT_R3_DATA
| RPL_MASK
;
685 Tss
->Ds
= KGDT_R3_DATA
| RPL_MASK
;
687 /* Setup the Double Trap TSS entry in the GDT */
688 TssEntry
= &Gdt
[KGDT_DF_TSS
/ sizeof(KGDTENTRY
)];
689 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
690 TssEntry
->HighWord
.Bits
.Pres
= 1;
691 TssEntry
->HighWord
.Bits
.Dpl
= 0;
692 TssEntry
->BaseLow
= (USHORT
)((ULONG_PTR
)Tss
& 0xFFFF);
693 TssEntry
->HighWord
.Bytes
.BaseMid
= (UCHAR
)((ULONG_PTR
)Tss
>> 16);
694 TssEntry
->HighWord
.Bytes
.BaseHi
= (UCHAR
)((ULONG_PTR
)Tss
>> 24);
695 TssEntry
->LimitLow
= KTSS_IO_MAPS
;
697 /* Now setup the NMI Task Gate */
698 TaskGateEntry
= (PKGDTENTRY
)&Idt
[2];
699 TaskGateEntry
->HighWord
.Bits
.Type
= I386_TASK_GATE
;
700 TaskGateEntry
->HighWord
.Bits
.Pres
= 1;
701 TaskGateEntry
->HighWord
.Bits
.Dpl
= 0;
702 ((PKIDTENTRY
)TaskGateEntry
)->Selector
= KGDT_NMI_TSS
;
704 /* Initialize the actual TSS */
705 Tss
= (PKTSS
)KiNMITSS
;
706 KiInitializeTSS(Tss
);
707 Tss
->CR3
= __readcr3();
708 Tss
->Esp0
= PtrToUlong(KiDoubleFaultStack
);
709 Tss
->Esp
= PtrToUlong(KiDoubleFaultStack
);
710 Tss
->Eip
= PtrToUlong(KiTrap2
);
711 Tss
->Cs
= KGDT_R0_CODE
;
712 Tss
->Fs
= KGDT_R0_PCR
;
713 Tss
->Ss
= Ke386GetSs();
714 Tss
->Es
= KGDT_R3_DATA
| RPL_MASK
;
715 Tss
->Ds
= KGDT_R3_DATA
| RPL_MASK
;
717 /* And its associated TSS Entry */
718 TssEntry
= &Gdt
[KGDT_NMI_TSS
/ sizeof(KGDTENTRY
)];
719 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
720 TssEntry
->HighWord
.Bits
.Pres
= 1;
721 TssEntry
->HighWord
.Bits
.Dpl
= 0;
722 TssEntry
->BaseLow
= (USHORT
)((ULONG_PTR
)Tss
& 0xFFFF);
723 TssEntry
->HighWord
.Bytes
.BaseMid
= (UCHAR
)((ULONG_PTR
)Tss
>> 16);
724 TssEntry
->HighWord
.Bytes
.BaseHi
= (UCHAR
)((ULONG_PTR
)Tss
>> 24);
725 TssEntry
->LimitLow
= KTSS_IO_MAPS
;
730 KeFlushCurrentTb(VOID
)
732 /* Flush the TLB by resetting CR3 */
733 __writecr3(__readcr3());
738 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState
)
743 // Restore the CR registers
745 __writecr0(ProcessorState
->SpecialRegisters
.Cr0
);
746 Ke386SetCr2(ProcessorState
->SpecialRegisters
.Cr2
);
747 __writecr3(ProcessorState
->SpecialRegisters
.Cr3
);
748 if (KeFeatureBits
& KF_CR4
) __writecr4(ProcessorState
->SpecialRegisters
.Cr4
);
751 // Restore the DR registers
753 __writedr(0, ProcessorState
->SpecialRegisters
.KernelDr0
);
754 __writedr(1, ProcessorState
->SpecialRegisters
.KernelDr1
);
755 __writedr(2, ProcessorState
->SpecialRegisters
.KernelDr2
);
756 __writedr(3, ProcessorState
->SpecialRegisters
.KernelDr3
);
757 __writedr(6, ProcessorState
->SpecialRegisters
.KernelDr6
);
758 __writedr(7, ProcessorState
->SpecialRegisters
.KernelDr7
);
761 // Restore GDT and IDT
763 Ke386SetGlobalDescriptorTable(&ProcessorState
->SpecialRegisters
.Gdtr
.Limit
);
764 __lidt(&ProcessorState
->SpecialRegisters
.Idtr
.Limit
);
767 // Clear the busy flag so we don't crash if we reload the same selector
769 TssEntry
= (PKGDTENTRY
)(ProcessorState
->SpecialRegisters
.Gdtr
.Base
+
770 ProcessorState
->SpecialRegisters
.Tr
);
771 TssEntry
->HighWord
.Bytes
.Flags1
&= ~0x2;
774 // Restore TSS and LDT
776 Ke386SetTr(ProcessorState
->SpecialRegisters
.Tr
);
777 Ke386SetLocalDescriptorTable(ProcessorState
->SpecialRegisters
.Ldtr
);
782 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState
)
784 /* Save the CR registers */
785 ProcessorState
->SpecialRegisters
.Cr0
= __readcr0();
786 ProcessorState
->SpecialRegisters
.Cr2
= __readcr2();
787 ProcessorState
->SpecialRegisters
.Cr3
= __readcr3();
788 ProcessorState
->SpecialRegisters
.Cr4
= (KeFeatureBits
& KF_CR4
) ?
791 /* Save the DR registers */
792 ProcessorState
->SpecialRegisters
.KernelDr0
= __readdr(0);
793 ProcessorState
->SpecialRegisters
.KernelDr1
= __readdr(1);
794 ProcessorState
->SpecialRegisters
.KernelDr2
= __readdr(2);
795 ProcessorState
->SpecialRegisters
.KernelDr3
= __readdr(3);
796 ProcessorState
->SpecialRegisters
.KernelDr6
= __readdr(6);
797 ProcessorState
->SpecialRegisters
.KernelDr7
= __readdr(7);
800 /* Save GDT, IDT, LDT and TSS */
801 Ke386GetGlobalDescriptorTable(&ProcessorState
->SpecialRegisters
.Gdtr
.Limit
);
802 __sidt(&ProcessorState
->SpecialRegisters
.Idtr
.Limit
);
803 ProcessorState
->SpecialRegisters
.Tr
= Ke386GetTr();
804 ProcessorState
->SpecialRegisters
.Ldtr
= Ke386GetLocalDescriptorTable();
809 KiInitializeMachineType(VOID
)
811 /* Set the Machine Type we got from NTLDR */
812 KeI386MachineType
= KeLoaderBlock
->u
.I386
.MachineType
& 0x000FF;
817 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context
)
820 WRMSR(0x174, KGDT_R0_CODE
);
821 WRMSR(0x175, (ULONG_PTR
)KeGetCurrentPrcb()->DpcStack
);
824 WRMSR(0x176, (ULONG_PTR
)KiFastCallEntry
);
830 KiRestoreFastSyscallReturnState(VOID
)
832 /* FIXME: NT has support for SYSCALL, IA64-SYSENTER, etc. */
834 /* Check if the CPU Supports fast system call */
835 if (KeFeatureBits
& KF_FAST_SYSCALL
)
837 /* Do an IPI to enable it */
838 KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters
, 0);
844 Ki386EnableDE(IN ULONG_PTR Context
)
847 __writecr4(__readcr4() | CR4_DE
);
853 Ki386EnableFxsr(IN ULONG_PTR Context
)
856 __writecr4(__readcr4() | CR4_FXSR
);
862 Ki386EnableXMMIExceptions(IN ULONG_PTR Context
)
866 /* Get the IDT Entry for Interrupt 19 */
867 IdtEntry
= &((PKIPCR
)KeGetPcr())->IDT
[19];
870 IdtEntry
->Selector
= KGDT_R0_CODE
;
871 IdtEntry
->Offset
= ((ULONG_PTR
)KiTrap19
& 0xFFFF);
872 IdtEntry
->ExtendedOffset
= ((ULONG_PTR
)KiTrap19
>> 16) & 0xFFFF;
873 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->Dpl
= 0;
874 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->Present
= 1;
875 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->SegmentType
= I386_INTERRUPT_GATE
;
877 /* Enable XMMI exceptions */
878 __writecr4(__readcr4() | CR4_XMMEXCPT
);
884 KiI386PentiumLockErrataFixup(VOID
)
886 KDESCRIPTOR IdtDescriptor
;
887 PKIDTENTRY NewIdt
, NewIdt2
;
889 /* Allocate memory for a new IDT */
890 NewIdt
= ExAllocatePool(NonPagedPool
, 2 * PAGE_SIZE
);
892 /* Put everything after the first 7 entries on a new page */
893 NewIdt2
= (PVOID
)((ULONG_PTR
)NewIdt
+ PAGE_SIZE
- (7 * sizeof(KIDTENTRY
)));
895 /* Disable interrupts */
898 /* Get the current IDT and copy it */
899 __sidt(&IdtDescriptor
.Limit
);
900 RtlCopyMemory(NewIdt2
,
901 (PVOID
)IdtDescriptor
.Base
,
902 IdtDescriptor
.Limit
+ 1);
903 IdtDescriptor
.Base
= (ULONG
)NewIdt2
;
905 /* Set the new IDT */
906 __lidt(&IdtDescriptor
.Limit
);
907 ((PKIPCR
)KeGetPcr())->IDT
= NewIdt2
;
909 /* Restore interrupts */
912 /* Set the first 7 entries as read-only to produce a fault */
913 MmSetPageProtect(NULL
, NewIdt
, PAGE_READONLY
);
918 KeFreezeExecution(IN PKTRAP_FRAME TrapFrame
,
919 IN PKEXCEPTION_FRAME ExceptionFrame
)
923 /* Disable interrupts and get previous state */
924 Flags
= __readeflags();
925 //Flags = __getcallerseflags();
928 /* Save freeze flag */
931 /* Save the old IRQL */
932 KiOldIrql
= KeGetCurrentIrql();
934 /* Return whether interrupts were enabled */
935 return (Flags
& EFLAGS_INTERRUPT_MASK
) ? TRUE
: FALSE
;
940 KeThawExecution(IN BOOLEAN Enable
)
942 /* Cleanup CPU caches */
945 /* Re-enable interrupts */
946 if (Enable
) _enable();
951 KeInvalidateAllCaches(VOID
)
953 /* Only supported on Pentium Pro and higher */
954 if (KeI386CpuType
< 6) return FALSE
;
956 /* Invalidate all caches */
963 KeZeroPages(IN PVOID Address
,
966 /* Not using XMMI in this routine */
967 RtlZeroMemory(Address
, Size
);
970 /* PUBLIC FUNCTIONS **********************************************************/
977 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save
)
979 PFNSAVE_FORMAT FpState
;
980 ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL
);
981 DPRINT1("%s is not really implemented\n", __FUNCTION__
);
983 /* check if we are doing software emulation */
984 if (!KeI386NpxPresent
) return STATUS_ILLEGAL_FLOAT_CONTEXT
;
986 FpState
= ExAllocatePool(NonPagedPool
, sizeof (FNSAVE_FORMAT
));
987 if (!FpState
) return STATUS_INSUFFICIENT_RESOURCES
;
989 *((PVOID
*) Save
) = FpState
;
991 asm volatile("fnsave %0\n\t" : "=m" (*FpState
));
999 KeGetCurrentThread()->DispatcherHeader
.NpxIrql
= KeGetCurrentIrql();
1000 return STATUS_SUCCESS
;
1008 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save
)
1010 PFNSAVE_FORMAT FpState
= *((PVOID
*) Save
);
1011 ASSERT(KeGetCurrentThread()->DispatcherHeader
.NpxIrql
== KeGetCurrentIrql());
1012 DPRINT1("%s is not really implemented\n", __FUNCTION__
);
1015 asm volatile("fnclex\n\t");
1016 asm volatile("frstor %0\n\t" : "=m" (*FpState
));
1025 ExFreePool(FpState
);
1026 return STATUS_SUCCESS
;
1034 KeGetRecommendedSharedDataAlignment(VOID
)
1036 /* Return the global variable */
1037 return KeLargestCacheLine
;
1042 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext
,
1047 /* Signal this packet as done */
1048 KiIpiSignalPacketDone(PacketContext
);
1050 /* Flush the TB for the Current CPU */
1059 KeFlushEntireTb(IN BOOLEAN Invalid
,
1060 IN BOOLEAN AllProcessors
)
1064 KAFFINITY TargetAffinity
;
1065 PKPRCB Prcb
= KeGetCurrentPrcb();
1068 /* Raise the IRQL for the TB Flush */
1069 OldIrql
= KeRaiseIrqlToSynchLevel();
1072 /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1074 /* Get the current processor affinity, and exclude ourselves */
1075 TargetAffinity
= KeActiveProcessors
;
1076 TargetAffinity
&= ~Prcb
->SetMember
;
1078 /* Make sure this is MP */
1081 /* Send an IPI TB flush to the other processors */
1082 KiIpiSendPacket(TargetAffinity
,
1083 KiFlushTargetEntireTb
,
1090 /* Flush the TB for the Current CPU, and update the flush stamp */
1094 /* If this is MP, wait for the other processors to finish */
1098 ASSERT(Prcb
== (volatile PKPRCB
)KeGetCurrentPrcb());
1101 ASSERTMSG("Not yet implemented\n", FALSE
);
1105 /* Update the flush stamp and return to original IRQL */
1106 InterlockedExchangeAdd(&KiTbFlushTimeStamp
, 1);
1107 KeLowerIrql(OldIrql
);
1115 KeSetDmaIoCoherency(IN ULONG Coherency
)
1117 /* Save the coherency globally */
1118 KiDmaIoCoherency
= Coherency
;
1126 KeQueryActiveProcessors(VOID
)
1130 /* Simply return the number of active processors */
1131 return KeActiveProcessors
;
1139 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State
)
1141 /* Capture the context */
1142 RtlCaptureContext(&State
->ContextFrame
);
1144 /* Capture the control state */
1145 KiSaveProcessorControlState(State
);