2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/i386/cpu.c
5 * PURPOSE: Routines for CPU-level support
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
9 /* INCLUDES *****************************************************************/
15 /* GLOBALS *******************************************************************/
17 /* The TSS to use for Double Fault Traps (INT 0x9) */
18 UCHAR KiDoubleFaultTSS
[KTSS_IO_MAPS
];
20 /* The TSS to use for NMI Fault Traps (INT 0x2) */
21 UCHAR KiNMITSS
[KTSS_IO_MAPS
];
23 /* CPU Features and Flags */
26 ULONG KeProcessorArchitecture
;
27 ULONG KeProcessorLevel
;
28 ULONG KeProcessorRevision
;
30 ULONG KiFastSystemCallDisable
= 1;
31 ULONG KeI386NpxPresent
= 0;
32 ULONG KiMXCsrMask
= 0;
33 ULONG MxcsrFeatureMask
= 0;
34 ULONG KeI386XMMIPresent
= 0;
35 ULONG KeI386FxsrPresent
= 0;
36 ULONG KeI386MachineType
;
37 ULONG Ke386Pae
= FALSE
;
38 ULONG Ke386NoExecute
= FALSE
;
39 ULONG KeLargestCacheLine
= 0x40;
40 ULONG KeDcacheFlushCount
= 0;
41 ULONG KeIcacheFlushCount
= 0;
42 ULONG KiDmaIoCoherency
= 0;
43 CHAR KeNumberProcessors
;
44 KAFFINITY KeActiveProcessors
= 1;
45 BOOLEAN KiI386PentiumLockErrataPresent
;
46 BOOLEAN KiSMTProcessorsPresent
;
49 volatile LONG KiTbFlushTimeStamp
;
52 static const CHAR CmpIntelID
[] = "GenuineIntel";
53 static const CHAR CmpAmdID
[] = "AuthenticAMD";
54 static const CHAR CmpCyrixID
[] = "CyrixInstead";
55 static const CHAR CmpTransmetaID
[] = "GenuineTMx86";
56 static const CHAR CmpCentaurID
[] = "CentaurHauls";
57 static const CHAR CmpRiseID
[] = "RiseRiseRise";
59 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
63 CPUID(IN ULONG InfoType
,
64 OUT PULONG CpuInfoEax
,
65 OUT PULONG CpuInfoEbx
,
66 OUT PULONG CpuInfoEcx
,
67 OUT PULONG CpuInfoEdx
)
71 /* Perform the CPUID Operation */
72 __cpuid((int*)CpuInfo
, InfoType
);
74 /* Return the results */
75 *CpuInfoEax
= CpuInfo
[0];
76 *CpuInfoEbx
= CpuInfo
[1];
77 *CpuInfoEcx
= CpuInfo
[2];
78 *CpuInfoEdx
= CpuInfo
[3];
83 WRMSR(IN ULONG Register
,
86 /* Write to the MSR */
87 __writemsr(Register
, Value
);
92 RDMSR(IN ULONG Register
)
94 /* Read from the MSR */
95 return __readmsr(Register
);
98 /* FUNCTIONS *****************************************************************/
102 KiSetProcessorType(VOID
)
104 ULONG EFlags
, NewEFlags
;
106 ULONG Stepping
, Type
;
108 /* Start by assuming no CPUID data */
109 KeGetCurrentPrcb()->CpuID
= 0;
112 EFlags
= __readeflags();
114 /* XOR out the ID bit and update EFlags */
115 NewEFlags
= EFlags
^ EFLAGS_ID
;
116 __writeeflags(NewEFlags
);
118 /* Get them back and see if they were modified */
119 NewEFlags
= __readeflags();
120 if (NewEFlags
!= EFlags
)
122 /* The modification worked, so CPUID exists. Set the ID Bit again. */
124 __writeeflags(EFlags
);
126 /* Peform CPUID 0 to see if CPUID 1 is supported */
127 CPUID(0, &Reg
, &Dummy
, &Dummy
, &Dummy
);
131 CPUID(1, &Reg
, &Dummy
, &Dummy
, &Dummy
);
134 * Get the Stepping and Type. The stepping contains both the
135 * Model and the Step, while the Type contains the returned Type.
136 * We ignore the family.
138 * For the stepping, we convert this: zzzzzzxy into this: x0y
140 Stepping
= Reg
& 0xF0;
142 Stepping
+= (Reg
& 0xFF);
147 /* Save them in the PRCB */
148 KeGetCurrentPrcb()->CpuID
= TRUE
;
149 KeGetCurrentPrcb()->CpuType
= (UCHAR
)Type
;
150 KeGetCurrentPrcb()->CpuStep
= (USHORT
)Stepping
;
154 DPRINT1("CPUID Support lacking\n");
159 DPRINT1("CPUID Support lacking\n");
163 __writeeflags(EFlags
);
170 PKPRCB Prcb
= KeGetCurrentPrcb();
174 /* Assume no Vendor ID and fail if no CPUID Support. */
175 Prcb
->VendorString
[0] = 0;
176 if (!Prcb
->CpuID
) return 0;
178 /* Get the Vendor ID and null-terminate it */
179 CPUID(0, &Vendor
[0], &Vendor
[1], &Vendor
[2], &Vendor
[3]);
182 /* Re-arrange vendor string */
184 Vendor
[2] = Vendor
[3];
187 /* Copy it to the PRCB and null-terminate it again */
188 RtlCopyMemory(Prcb
->VendorString
,
190 sizeof(Prcb
->VendorString
) - sizeof(CHAR
));
191 Prcb
->VendorString
[sizeof(Prcb
->VendorString
) - sizeof(CHAR
)] = ANSI_NULL
;
193 /* Now check the CPU Type */
194 if (!strcmp(Prcb
->VendorString
, CmpIntelID
))
198 else if (!strcmp(Prcb
->VendorString
, CmpAmdID
))
202 else if (!strcmp(Prcb
->VendorString
, CmpCyrixID
))
204 DPRINT1("Cyrix CPU support not fully tested!\n");
207 else if (!strcmp(Prcb
->VendorString
, CmpTransmetaID
))
209 DPRINT1("Transmeta CPU support not fully tested!\n");
210 return CPU_TRANSMETA
;
212 else if (!strcmp(Prcb
->VendorString
, CmpCentaurID
))
214 DPRINT1("Centaur CPU support not fully tested!\n");
217 else if (!strcmp(Prcb
->VendorString
, CmpRiseID
))
219 DPRINT1("Rise CPU support not fully tested!\n");
229 KiGetFeatureBits(VOID
)
231 PKPRCB Prcb
= KeGetCurrentPrcb();
233 ULONG FeatureBits
= KF_WORKING_PTE
;
235 BOOLEAN ExtendedCPUID
= TRUE
;
236 ULONG CpuFeatures
= 0;
238 /* Get the Vendor ID */
239 Vendor
= KiGetCpuVendor();
241 /* Make sure we got a valid vendor ID at least. */
242 if (!Vendor
) return FeatureBits
;
244 /* Get the CPUID Info. Features are in Reg[3]. */
245 CPUID(1, &Reg
[0], &Reg
[1], &Dummy
, &Reg
[3]);
247 /* Set the initial APIC ID */
248 Prcb
->InitialApicId
= (UCHAR
)(Reg
[1] >> 24);
255 /* Check if it's a P6 */
256 if (Prcb
->CpuType
== 6)
258 /* Perform the special sequence to get the MicroCode Signature */
260 CPUID(1, &Dummy
, &Dummy
, &Dummy
, &Dummy
);
261 Prcb
->UpdateSignature
.QuadPart
= RDMSR(0x8B);
263 else if (Prcb
->CpuType
== 5)
265 /* On P5, enable workaround for the LOCK errata. */
266 KiI386PentiumLockErrataPresent
= TRUE
;
269 /* Check for broken P6 with bad SMP PTE implementation */
270 if (((Reg
[0] & 0x0FF0) == 0x0610 && (Reg
[0] & 0x000F) <= 0x9) ||
271 ((Reg
[0] & 0x0FF0) == 0x0630 && (Reg
[0] & 0x000F) <= 0x4))
273 /* Remove support for correct PTE support. */
274 FeatureBits
&= ~KF_WORKING_PTE
;
277 /* Check if the CPU is too old to support SYSENTER */
278 if ((Prcb
->CpuType
< 6) ||
279 ((Prcb
->CpuType
== 6) && (Prcb
->CpuStep
< 0x0303)))
285 /* Set the current features */
286 CpuFeatures
= Reg
[3];
293 /* Check if this is a K5 or K6. (family 5) */
294 if ((Reg
[0] & 0x0F00) == 0x0500)
296 /* Get the Model Number */
297 switch (Reg
[0] & 0x00F0)
299 /* Model 1: K5 - 5k86 (initial models) */
302 /* Check if this is Step 0 or 1. They don't support PGE */
303 if ((Reg
[0] & 0x000F) > 0x03) break;
305 /* Model 0: K5 - SSA5 */
308 /* Model 0 doesn't support PGE at all. */
315 /* K6-2, Step 8 and over have support for MTRR. */
316 if ((Reg
[0] & 0x000F) >= 0x8) FeatureBits
|= KF_AMDK6MTRR
;
320 Model D: K6-2+, K6-III+ */
324 FeatureBits
|= KF_AMDK6MTRR
;
328 else if((Reg
[0] & 0x0F00) < 0x0500)
330 /* Families below 5 don't support PGE, PSE or CMOV at all */
331 Reg
[3] &= ~(0x08 | 0x2000 | 0x8000);
333 /* They also don't support advanced CPUID functions. */
334 ExtendedCPUID
= FALSE
;
337 /* Set the current features */
338 CpuFeatures
= Reg
[3];
345 /* FIXME: CMPXCGH8B */
352 /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
353 if ((Reg
[0] & 0x0FFF) >= 0x0542)
355 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
356 FeatureBits
|= KF_CMPXCHG8B
;
361 /* Centaur, IDT, Rise and VIA CPUs */
365 /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
366 However, this feature exists and operates properly without any additional steps. */
367 FeatureBits
|= KF_CMPXCHG8B
;
372 /* Convert all CPUID Feature bits into our format */
373 if (CpuFeatures
& 0x00000002) FeatureBits
|= KF_V86_VIS
| KF_CR4
;
374 if (CpuFeatures
& 0x00000008) FeatureBits
|= KF_LARGE_PAGE
| KF_CR4
;
375 if (CpuFeatures
& 0x00000010) FeatureBits
|= KF_RDTSC
;
376 if (CpuFeatures
& 0x00000100) FeatureBits
|= KF_CMPXCHG8B
;
377 if (CpuFeatures
& 0x00000800) FeatureBits
|= KF_FAST_SYSCALL
;
378 if (CpuFeatures
& 0x00001000) FeatureBits
|= KF_MTRR
;
379 if (CpuFeatures
& 0x00002000) FeatureBits
|= KF_GLOBAL_PAGE
| KF_CR4
;
380 if (CpuFeatures
& 0x00008000) FeatureBits
|= KF_CMOV
;
381 if (CpuFeatures
& 0x00010000) FeatureBits
|= KF_PAT
;
382 if (CpuFeatures
& 0x00200000) FeatureBits
|= KF_DTS
;
383 if (CpuFeatures
& 0x00800000) FeatureBits
|= KF_MMX
;
384 if (CpuFeatures
& 0x01000000) FeatureBits
|= KF_FXSR
;
385 if (CpuFeatures
& 0x02000000) FeatureBits
|= KF_XMMI
;
386 if (CpuFeatures
& 0x04000000) FeatureBits
|= KF_XMMI64
;
388 /* Check if the CPU has hyper-threading */
389 if (CpuFeatures
& 0x10000000)
391 /* Set the number of logical CPUs */
392 Prcb
->LogicalProcessorsPerPhysicalProcessor
= (UCHAR
)(Reg
[1] >> 16);
393 if (Prcb
->LogicalProcessorsPerPhysicalProcessor
> 1)
395 /* We're on dual-core */
396 KiSMTProcessorsPresent
= TRUE
;
401 /* We only have a single CPU */
402 Prcb
->LogicalProcessorsPerPhysicalProcessor
= 1;
405 /* Check if CPUID 0x80000000 is supported */
409 CPUID(0x80000000, &Reg
[0], &Dummy
, &Dummy
, &Dummy
);
410 if ((Reg
[0] & 0xffffff00) == 0x80000000)
412 /* Check if CPUID 0x80000001 is supported */
413 if (Reg
[0] >= 0x80000001)
415 /* Check which extended features are available. */
416 CPUID(0x80000001, &Dummy
, &Dummy
, &Dummy
, &Reg
[3]);
418 /* Check if NX-bit is supported */
419 if (Reg
[3] & 0x00100000) FeatureBits
|= KF_NX_BIT
;
421 /* Now handle each features for each CPU Vendor */
426 if (Reg
[3] & 0x80000000) FeatureBits
|= KF_3DNOW
;
433 /* Return the Feature Bits */
439 KiGetCacheInformation(VOID
)
441 PKIPCR Pcr
= (PKIPCR
)KeGetPcr();
443 ULONG Data
[4], Dummy
;
444 ULONG CacheRequests
= 0, i
;
445 ULONG CurrentRegister
;
447 BOOLEAN FirstPass
= TRUE
;
449 /* Set default L2 size */
450 Pcr
->SecondLevelCacheSize
= 0;
452 /* Get the Vendor ID and make sure we support CPUID */
453 Vendor
= KiGetCpuVendor();
456 /* Check the Vendor ID */
459 /* Handle Intel case */
462 /*Check if we support CPUID 2 */
463 CPUID(0, &Data
[0], &Dummy
, &Dummy
, &Dummy
);
466 /* We need to loop for the number of times CPUID will tell us to */
469 /* Do the CPUID call */
470 CPUID(2, &Data
[0], &Data
[1], &Data
[2], &Data
[3]);
472 /* Check if it was the first call */
476 * The number of times to loop is the first byte. Read
477 * it and then destroy it so we don't get confused.
479 CacheRequests
= Data
[0] & 0xFF;
480 Data
[0] &= 0xFFFFFF00;
482 /* Don't go over this again */
486 /* Loop all 4 registers */
487 for (i
= 0; i
< 4; i
++)
489 /* Get the current register */
490 CurrentRegister
= Data
[i
];
493 * If the upper bit is set, then this register should
496 if (CurrentRegister
& 0x80000000) continue;
498 /* Keep looping for every byte inside this register */
499 while (CurrentRegister
)
501 /* Read a byte, skip a byte. */
502 RegisterByte
= (UCHAR
)(CurrentRegister
& 0xFF);
503 CurrentRegister
>>= 8;
504 if (!RegisterByte
) continue;
507 * Valid values are from 0x40 (0 bytes) to 0x49
508 * (32MB), or from 0x80 to 0x89 (same size but
511 if (((RegisterByte
> 0x40) &&
512 (RegisterByte
<= 0x49)) ||
513 ((RegisterByte
> 0x80) &&
514 (RegisterByte
<= 0x89)))
516 /* Mask out only the first nibble */
517 RegisterByte
&= 0x0F;
519 /* Set the L2 Cache Size */
520 Pcr
->SecondLevelCacheSize
= 0x10000 <<
525 } while (--CacheRequests
);
531 /* Check if we support CPUID 0x80000006 */
532 CPUID(0x80000000, &Data
[0], &Dummy
, &Dummy
, &Dummy
);
535 /* Get 2nd level cache and tlb size */
536 CPUID(0x80000006, &Dummy
, &Dummy
, &Data
[2], &Dummy
);
538 /* Set the L2 Cache Size */
539 Pcr
->SecondLevelCacheSize
= (Data
[2] & 0xFFFF0000) >> 6;
559 /* Save current CR0 */
562 /* If this is a 486, enable Write-Protection */
563 if (KeGetCurrentPrcb()->CpuType
> 3) Cr0
|= CR0_WP
;
571 KiInitializeTSS2(IN PKTSS Tss
,
572 IN PKGDTENTRY TssEntry OPTIONAL
)
576 /* Make sure the GDT Entry is valid */
580 TssEntry
->LimitLow
= sizeof(KTSS
) - 1;
581 TssEntry
->HighWord
.Bits
.LimitHi
= 0;
584 /* Now clear the I/O Map */
585 RtlFillMemory(Tss
->IoMaps
[0].IoMap
, 8096, -1);
587 /* Initialize Interrupt Direction Maps */
588 p
= (PUCHAR
)(Tss
->IoMaps
[0].DirectionMap
);
589 RtlZeroMemory(p
, 32);
591 /* Add DPMI support for interrupts */
596 /* Initialize the default Interrupt Direction Map */
597 p
= Tss
->IntDirectionMap
;
598 RtlZeroMemory(Tss
->IntDirectionMap
, 32);
600 /* Add DPMI support */
608 KiInitializeTSS(IN PKTSS Tss
)
610 /* Set an invalid map base */
611 Tss
->IoMapBase
= KiComputeIopmOffset(IO_ACCESS_MAP_NONE
);
613 /* Disable traps during Task Switches */
616 /* Set LDT and Ring 0 SS */
618 Tss
->Ss0
= KGDT_R0_DATA
;
623 Ki386InitializeTss(IN PKTSS Tss
,
627 PKGDTENTRY TssEntry
, TaskGateEntry
;
629 /* Initialize the boot TSS. */
630 TssEntry
= &Gdt
[KGDT_TSS
/ sizeof(KGDTENTRY
)];
631 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
632 TssEntry
->HighWord
.Bits
.Pres
= 1;
633 TssEntry
->HighWord
.Bits
.Dpl
= 0;
634 KiInitializeTSS2(Tss
, TssEntry
);
635 KiInitializeTSS(Tss
);
637 /* Load the task register */
638 Ke386SetTr(KGDT_TSS
);
640 /* Setup the Task Gate for Double Fault Traps */
641 TaskGateEntry
= (PKGDTENTRY
)&Idt
[8];
642 TaskGateEntry
->HighWord
.Bits
.Type
= I386_TASK_GATE
;
643 TaskGateEntry
->HighWord
.Bits
.Pres
= 1;
644 TaskGateEntry
->HighWord
.Bits
.Dpl
= 0;
645 ((PKIDTENTRY
)TaskGateEntry
)->Selector
= KGDT_DF_TSS
;
647 /* Initialize the TSS used for handling double faults. */
648 Tss
= (PKTSS
)KiDoubleFaultTSS
;
649 KiInitializeTSS(Tss
);
650 Tss
->CR3
= __readcr3();
651 Tss
->Esp0
= PtrToUlong(KiDoubleFaultStack
);
652 Tss
->Esp
= PtrToUlong(KiDoubleFaultStack
);
653 Tss
->Eip
= PtrToUlong(KiTrap8
);
654 Tss
->Cs
= KGDT_R0_CODE
;
655 Tss
->Fs
= KGDT_R0_PCR
;
656 Tss
->Ss
= Ke386GetSs();
657 Tss
->Es
= KGDT_R3_DATA
| RPL_MASK
;
658 Tss
->Ds
= KGDT_R3_DATA
| RPL_MASK
;
660 /* Setup the Double Trap TSS entry in the GDT */
661 TssEntry
= &Gdt
[KGDT_DF_TSS
/ sizeof(KGDTENTRY
)];
662 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
663 TssEntry
->HighWord
.Bits
.Pres
= 1;
664 TssEntry
->HighWord
.Bits
.Dpl
= 0;
665 TssEntry
->BaseLow
= (USHORT
)((ULONG_PTR
)Tss
& 0xFFFF);
666 TssEntry
->HighWord
.Bytes
.BaseMid
= (UCHAR
)((ULONG_PTR
)Tss
>> 16);
667 TssEntry
->HighWord
.Bytes
.BaseHi
= (UCHAR
)((ULONG_PTR
)Tss
>> 24);
668 TssEntry
->LimitLow
= KTSS_IO_MAPS
;
670 /* Now setup the NMI Task Gate */
671 TaskGateEntry
= (PKGDTENTRY
)&Idt
[2];
672 TaskGateEntry
->HighWord
.Bits
.Type
= I386_TASK_GATE
;
673 TaskGateEntry
->HighWord
.Bits
.Pres
= 1;
674 TaskGateEntry
->HighWord
.Bits
.Dpl
= 0;
675 ((PKIDTENTRY
)TaskGateEntry
)->Selector
= KGDT_NMI_TSS
;
677 /* Initialize the actual TSS */
678 Tss
= (PKTSS
)KiNMITSS
;
679 KiInitializeTSS(Tss
);
680 Tss
->CR3
= __readcr3();
681 Tss
->Esp0
= PtrToUlong(KiDoubleFaultStack
);
682 Tss
->Esp
= PtrToUlong(KiDoubleFaultStack
);
683 Tss
->Eip
= PtrToUlong(KiTrap2
);
684 Tss
->Cs
= KGDT_R0_CODE
;
685 Tss
->Fs
= KGDT_R0_PCR
;
686 Tss
->Ss
= Ke386GetSs();
687 Tss
->Es
= KGDT_R3_DATA
| RPL_MASK
;
688 Tss
->Ds
= KGDT_R3_DATA
| RPL_MASK
;
690 /* And its associated TSS Entry */
691 TssEntry
= &Gdt
[KGDT_NMI_TSS
/ sizeof(KGDTENTRY
)];
692 TssEntry
->HighWord
.Bits
.Type
= I386_TSS
;
693 TssEntry
->HighWord
.Bits
.Pres
= 1;
694 TssEntry
->HighWord
.Bits
.Dpl
= 0;
695 TssEntry
->BaseLow
= (USHORT
)((ULONG_PTR
)Tss
& 0xFFFF);
696 TssEntry
->HighWord
.Bytes
.BaseMid
= (UCHAR
)((ULONG_PTR
)Tss
>> 16);
697 TssEntry
->HighWord
.Bytes
.BaseHi
= (UCHAR
)((ULONG_PTR
)Tss
>> 24);
698 TssEntry
->LimitLow
= KTSS_IO_MAPS
;
703 KeFlushCurrentTb(VOID
)
705 /* Flush the TLB by resetting CR3 */
706 __writecr3(__readcr3());
711 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState
)
716 // Restore the CR registers
718 __writecr0(ProcessorState
->SpecialRegisters
.Cr0
);
719 Ke386SetCr2(ProcessorState
->SpecialRegisters
.Cr2
);
720 __writecr3(ProcessorState
->SpecialRegisters
.Cr3
);
721 if (KeFeatureBits
& KF_CR4
) __writecr4(ProcessorState
->SpecialRegisters
.Cr4
);
724 // Restore the DR registers
726 __writedr(0, ProcessorState
->SpecialRegisters
.KernelDr0
);
727 __writedr(1, ProcessorState
->SpecialRegisters
.KernelDr1
);
728 __writedr(2, ProcessorState
->SpecialRegisters
.KernelDr2
);
729 __writedr(3, ProcessorState
->SpecialRegisters
.KernelDr3
);
730 __writedr(6, ProcessorState
->SpecialRegisters
.KernelDr6
);
731 __writedr(7, ProcessorState
->SpecialRegisters
.KernelDr7
);
734 // Restore GDT and IDT
736 Ke386SetGlobalDescriptorTable(&ProcessorState
->SpecialRegisters
.Gdtr
.Limit
);
737 __lidt(&ProcessorState
->SpecialRegisters
.Idtr
.Limit
);
740 // Clear the busy flag so we don't crash if we reload the same selector
742 TssEntry
= (PKGDTENTRY
)(ProcessorState
->SpecialRegisters
.Gdtr
.Base
+
743 ProcessorState
->SpecialRegisters
.Tr
);
744 TssEntry
->HighWord
.Bytes
.Flags1
&= ~0x2;
747 // Restore TSS and LDT
749 Ke386SetTr(ProcessorState
->SpecialRegisters
.Tr
);
750 Ke386SetLocalDescriptorTable(ProcessorState
->SpecialRegisters
.Ldtr
);
755 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState
)
757 /* Save the CR registers */
758 ProcessorState
->SpecialRegisters
.Cr0
= __readcr0();
759 ProcessorState
->SpecialRegisters
.Cr2
= __readcr2();
760 ProcessorState
->SpecialRegisters
.Cr3
= __readcr3();
761 ProcessorState
->SpecialRegisters
.Cr4
= (KeFeatureBits
& KF_CR4
) ?
764 /* Save the DR registers */
765 ProcessorState
->SpecialRegisters
.KernelDr0
= __readdr(0);
766 ProcessorState
->SpecialRegisters
.KernelDr1
= __readdr(1);
767 ProcessorState
->SpecialRegisters
.KernelDr2
= __readdr(2);
768 ProcessorState
->SpecialRegisters
.KernelDr3
= __readdr(3);
769 ProcessorState
->SpecialRegisters
.KernelDr6
= __readdr(6);
770 ProcessorState
->SpecialRegisters
.KernelDr7
= __readdr(7);
773 /* Save GDT, IDT, LDT and TSS */
774 Ke386GetGlobalDescriptorTable(&ProcessorState
->SpecialRegisters
.Gdtr
.Limit
);
775 __sidt(&ProcessorState
->SpecialRegisters
.Idtr
.Limit
);
776 ProcessorState
->SpecialRegisters
.Tr
= Ke386GetTr();
777 ProcessorState
->SpecialRegisters
.Ldtr
= Ke386GetLocalDescriptorTable();
782 KiInitializeMachineType(VOID
)
784 /* Set the Machine Type we got from NTLDR */
785 KeI386MachineType
= KeLoaderBlock
->u
.I386
.MachineType
& 0x000FF;
790 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context
)
793 WRMSR(0x174, KGDT_R0_CODE
);
794 WRMSR(0x175, (ULONG_PTR
)KeGetCurrentPrcb()->DpcStack
);
797 WRMSR(0x176, (ULONG_PTR
)KiFastCallEntry
);
803 KiRestoreFastSyscallReturnState(VOID
)
805 /* FIXME: NT has support for SYSCALL, IA64-SYSENTER, etc. */
807 /* Check if the CPU Supports fast system call */
808 if (KeFeatureBits
& KF_FAST_SYSCALL
)
810 /* Do an IPI to enable it */
811 KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters
, 0);
817 Ki386EnableDE(IN ULONG_PTR Context
)
820 __writecr4(__readcr4() | CR4_DE
);
826 Ki386EnableFxsr(IN ULONG_PTR Context
)
829 __writecr4(__readcr4() | CR4_FXSR
);
835 Ki386EnableXMMIExceptions(IN ULONG_PTR Context
)
839 /* Get the IDT Entry for Interrupt 19 */
840 IdtEntry
= &((PKIPCR
)KeGetPcr())->IDT
[19];
843 IdtEntry
->Selector
= KGDT_R0_CODE
;
844 IdtEntry
->Offset
= ((ULONG_PTR
)KiTrap19
& 0xFFFF);
845 IdtEntry
->ExtendedOffset
= ((ULONG_PTR
)KiTrap19
>> 16) & 0xFFFF;
846 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->Dpl
= 0;
847 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->Present
= 1;
848 ((PKIDT_ACCESS
)&IdtEntry
->Access
)->SegmentType
= I386_INTERRUPT_GATE
;
850 /* Enable XMMI exceptions */
851 __writecr4(__readcr4() | CR4_XMMEXCPT
);
857 KiI386PentiumLockErrataFixup(VOID
)
859 KDESCRIPTOR IdtDescriptor
;
860 PKIDTENTRY NewIdt
, NewIdt2
;
862 /* Allocate memory for a new IDT */
863 NewIdt
= ExAllocatePool(NonPagedPool
, 2 * PAGE_SIZE
);
865 /* Put everything after the first 7 entries on a new page */
866 NewIdt2
= (PVOID
)((ULONG_PTR
)NewIdt
+ PAGE_SIZE
- (7 * sizeof(KIDTENTRY
)));
868 /* Disable interrupts */
871 /* Get the current IDT and copy it */
872 __sidt(&IdtDescriptor
.Limit
);
873 RtlCopyMemory(NewIdt2
,
874 (PVOID
)IdtDescriptor
.Base
,
875 IdtDescriptor
.Limit
+ 1);
876 IdtDescriptor
.Base
= (ULONG
)NewIdt2
;
878 /* Set the new IDT */
879 __lidt(&IdtDescriptor
.Limit
);
880 ((PKIPCR
)KeGetPcr())->IDT
= NewIdt2
;
882 /* Restore interrupts */
885 /* Set the first 7 entries as read-only to produce a fault */
886 MmSetPageProtect(NULL
, NewIdt
, PAGE_READONLY
);
891 KeDisableInterrupts(VOID
)
896 /* Get EFLAGS and check if the interrupt bit is set */
897 Flags
= __readeflags();
898 Return
= (Flags
& EFLAGS_INTERRUPT_MASK
) ? TRUE
: FALSE
;
900 /* Disable interrupts */
907 KeInvalidateAllCaches(VOID
)
909 /* Only supported on Pentium Pro and higher */
910 if (KeI386CpuType
< 6) return FALSE
;
912 /* Invalidate all caches */
919 KeZeroPages(IN PVOID Address
,
922 /* Not using XMMI in this routine */
923 RtlZeroMemory(Address
, Size
);
926 /* PUBLIC FUNCTIONS **********************************************************/
933 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save
)
935 PFNSAVE_FORMAT FpState
;
936 ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL
);
937 DPRINT1("%s is not really implemented\n", __FUNCTION__
);
939 /* check if we are doing software emulation */
940 if (!KeI386NpxPresent
) return STATUS_ILLEGAL_FLOAT_CONTEXT
;
942 FpState
= ExAllocatePool(NonPagedPool
, sizeof (FNSAVE_FORMAT
));
943 if (!FpState
) return STATUS_INSUFFICIENT_RESOURCES
;
945 *((PVOID
*) Save
) = FpState
;
947 asm volatile("fnsave %0\n\t" : "=m" (*FpState
));
955 KeGetCurrentThread()->DispatcherHeader
.NpxIrql
= KeGetCurrentIrql();
956 return STATUS_SUCCESS
;
964 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save
)
966 PFNSAVE_FORMAT FpState
= *((PVOID
*) Save
);
967 ASSERT(KeGetCurrentThread()->DispatcherHeader
.NpxIrql
== KeGetCurrentIrql());
968 DPRINT1("%s is not really implemented\n", __FUNCTION__
);
971 asm volatile("fnclex\n\t");
972 asm volatile("frstor %0\n\t" : "=m" (*FpState
));
982 return STATUS_SUCCESS
;
990 KeGetRecommendedSharedDataAlignment(VOID
)
992 /* Return the global variable */
993 return KeLargestCacheLine
;
998 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext
,
1003 /* Signal this packet as done */
1004 KiIpiSignalPacketDone(PacketContext
);
1006 /* Flush the TB for the Current CPU */
1015 KeFlushEntireTb(IN BOOLEAN Invalid
,
1016 IN BOOLEAN AllProcessors
)
1020 KAFFINITY TargetAffinity
;
1021 PKPRCB Prcb
= KeGetCurrentPrcb();
1024 /* Raise the IRQL for the TB Flush */
1025 OldIrql
= KeRaiseIrqlToSynchLevel();
1028 /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1030 /* Get the current processor affinity, and exclude ourselves */
1031 TargetAffinity
= KeActiveProcessors
;
1032 TargetAffinity
&= ~Prcb
->SetMember
;
1034 /* Make sure this is MP */
1037 /* Send an IPI TB flush to the other processors */
1038 KiIpiSendPacket(TargetAffinity
,
1039 KiFlushTargetEntireTb
,
1046 /* Flush the TB for the Current CPU, and update the flush stamp */
1050 /* If this is MP, wait for the other processors to finish */
1054 ASSERT(Prcb
== (volatile PKPRCB
)KeGetCurrentPrcb());
1057 ASSERTMSG("Not yet implemented\n", FALSE
);
1061 /* Update the flush stamp and return to original IRQL */
1062 InterlockedExchangeAdd(&KiTbFlushTimeStamp
, 1);
1063 KeLowerIrql(OldIrql
);
1071 KeSetDmaIoCoherency(IN ULONG Coherency
)
1073 /* Save the coherency globally */
1074 KiDmaIoCoherency
= Coherency
;
1082 KeQueryActiveProcessors(VOID
)
1086 /* Simply return the number of active processors */
1087 return KeActiveProcessors
;
1095 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State
)
1097 /* Capture the context */
1098 RtlCaptureContext(&State
->ContextFrame
);
1100 /* Capture the control state */
1101 KiSaveProcessorControlState(State
);