Sync with trunk (48237)
[reactos.git] / ntoskrnl / ke / amd64 / cpu.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/amd64/cpu.c
5 * PURPOSE: Routines for CPU-level support
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 * Timo Kreuzer (timo.kreuzer@reactos.org)
8 */
9
10 /* INCLUDES *****************************************************************/
11
12 #include <ntoskrnl.h>
13 #define NDEBUG
14 #include <debug.h>
15
16 /* FIXME: Local EFLAGS defines not used anywhere else */
17 #define EFLAGS_IOPL 0x3000
18 #define EFLAGS_NF 0x4000
19 #define EFLAGS_RF 0x10000
20 #define EFLAGS_ID 0x200000
21
22 /* GLOBALS *******************************************************************/
23
24 /* The Boot TSS */
25 KTSS64 KiBootTss;
26
27 /* CPU Features and Flags */
28 ULONG KeI386CpuType;
29 ULONG KeI386CpuStep;
30 ULONG KeProcessorArchitecture;
31 ULONG KeProcessorLevel;
32 ULONG KeProcessorRevision;
33 ULONG KeFeatureBits;
34 ULONG KeI386MachineType;
35 ULONG KeI386NpxPresent = 1;
36 ULONG KeLargestCacheLine = 0x40;
37 ULONG KiDmaIoCoherency = 0;
38 CHAR KeNumberProcessors = 0;
39 KAFFINITY KeActiveProcessors = 1;
40 BOOLEAN KiI386PentiumLockErrataPresent;
41 BOOLEAN KiSMTProcessorsPresent;
42
43 /* Freeze data */
44 KIRQL KiOldIrql;
45 ULONG KiFreezeFlag;
46
47 /* Flush data */
48 volatile LONG KiTbFlushTimeStamp;
49
50 /* CPU Signatures */
51 static const CHAR CmpIntelID[] = "GenuineIntel";
52 static const CHAR CmpAmdID[] = "AuthenticAMD";
53 static const CHAR CmpCyrixID[] = "CyrixInstead";
54 static const CHAR CmpTransmetaID[] = "GenuineTMx86";
55 static const CHAR CmpCentaurID[] = "CentaurHauls";
56 static const CHAR CmpRiseID[] = "RiseRiseRise";
57
58 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
59
60 VOID
61 NTAPI
62 CPUID(IN ULONG InfoType,
63 OUT PULONG CpuInfoEax,
64 OUT PULONG CpuInfoEbx,
65 OUT PULONG CpuInfoEcx,
66 OUT PULONG CpuInfoEdx)
67 {
68 ULONG CpuInfo[4];
69
70 /* Perform the CPUID Operation */
71 __cpuid((int*)CpuInfo, InfoType);
72
73 /* Return the results */
74 *CpuInfoEax = CpuInfo[0];
75 *CpuInfoEbx = CpuInfo[1];
76 *CpuInfoEcx = CpuInfo[2];
77 *CpuInfoEdx = CpuInfo[3];
78 }
79
80 /* FUNCTIONS *****************************************************************/
81
82 VOID
83 NTAPI
84 KiSetProcessorType(VOID)
85 {
86 ULONG64 EFlags;
87 INT Reg[4];
88 ULONG Stepping, Type;
89
90 /* Start by assuming no CPUID data */
91 KeGetCurrentPrcb()->CpuID = 0;
92
93 /* Save EFlags */
94 EFlags = __readeflags();
95
96 /* Do CPUID 1 now */
97 __cpuid(Reg, 1);
98
99 /*
100 * Get the Stepping and Type. The stepping contains both the
101 * Model and the Step, while the Type contains the returned Type.
102 * We ignore the family.
103 *
104 * For the stepping, we convert this: zzzzzzxy into this: x0y
105 */
106 Stepping = Reg[0] & 0xF0;
107 Stepping <<= 4;
108 Stepping += (Reg[0] & 0xFF);
109 Stepping &= 0xF0F;
110 Type = Reg[0] & 0xF00;
111 Type >>= 8;
112
113 /* Save them in the PRCB */
114 KeGetCurrentPrcb()->CpuID = TRUE;
115 KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
116 KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
117
118 /* Restore EFLAGS */
119 __writeeflags(EFlags);
120 }
121
122 ULONG
123 NTAPI
124 KiGetCpuVendor(VOID)
125 {
126 PKPRCB Prcb = KeGetCurrentPrcb();
127 INT Vendor[5];
128
129 /* Get the Vendor ID and null-terminate it */
130 __cpuid(Vendor, 0);
131
132 /* Copy it to the PRCB and null-terminate it */
133 *(ULONG*)&Prcb->VendorString[0] = Vendor[1]; // ebx
134 *(ULONG*)&Prcb->VendorString[4] = Vendor[3]; // edx
135 *(ULONG*)&Prcb->VendorString[8] = Vendor[2]; // ecx
136 *(ULONG*)&Prcb->VendorString[12] = 0;
137
138 /* Now check the CPU Type */
139 if (!strcmp((PCHAR)Prcb->VendorString, CmpIntelID))
140 {
141 return CPU_INTEL;
142 }
143 else if (!strcmp((PCHAR)Prcb->VendorString, CmpAmdID))
144 {
145 return CPU_AMD;
146 }
147 else if (!strcmp((PCHAR)Prcb->VendorString, CmpCyrixID))
148 {
149 DPRINT1("Cyrix CPUs not fully supported\n");
150 return 0;
151 }
152 else if (!strcmp((PCHAR)Prcb->VendorString, CmpTransmetaID))
153 {
154 DPRINT1("Transmeta CPUs not fully supported\n");
155 return 0;
156 }
157 else if (!strcmp((PCHAR)Prcb->VendorString, CmpCentaurID))
158 {
159 DPRINT1("VIA CPUs not fully supported\n");
160 return 0;
161 }
162 else if (!strcmp((PCHAR)Prcb->VendorString, CmpRiseID))
163 {
164 DPRINT1("Rise CPUs not fully supported\n");
165 return 0;
166 }
167
168 /* Invalid CPU */
169 return 0;
170 }
171
172 ULONG
173 NTAPI
174 KiGetFeatureBits(VOID)
175 {
176 PKPRCB Prcb = KeGetCurrentPrcb();
177 ULONG Vendor;
178 ULONG FeatureBits = KF_WORKING_PTE;
179 INT Reg[4];
180 ULONG CpuFeatures = 0;
181
182 /* Get the Vendor ID */
183 Vendor = KiGetCpuVendor();
184
185 /* Make sure we got a valid vendor ID at least. */
186 if (!Vendor) return FeatureBits;
187
188 /* Get the CPUID Info. Features are in Reg[3]. */
189 __cpuid(Reg, 1);
190
191 /* Set the initial APIC ID */
192 Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
193
194 /* Set the current features */
195 CpuFeatures = Reg[3];
196
197 /* Convert all CPUID Feature bits into our format */
198 if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
199 if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
200 if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
201 if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
202 if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
203 if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
204 if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
205 if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
206 if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
207 if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
208 if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
209 if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
210 if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
211 if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
212
213 #if 0
214 if (Reg[2] & 0x00000001) FeatureBits |= KF_SSE3NEW;
215 if (Reg[2] & 0x00000008) FeatureBits |= KF_MONITOR;
216 if (Reg[2] & 0x00000200) FeatureBits |= KF_SSE3SUP;
217 if (Reg[2] & 0x00002000) FeatureBits |= KF_CMPXCHG16B;
218 if (Reg[2] & 0x00080000) FeatureBits |= KF_SSE41;
219 if (Reg[2] & 0x00800000) FeatureBits |= KF_POPCNT;
220 #endif
221
222 /* Check if the CPU has hyper-threading */
223 if (CpuFeatures & 0x10000000)
224 {
225 /* Set the number of logical CPUs */
226 Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
227 if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
228 {
229 /* We're on dual-core */
230 KiSMTProcessorsPresent = TRUE;
231 }
232 }
233 else
234 {
235 /* We only have a single CPU */
236 Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
237 }
238
239 /* Check extended cpuid features */
240 __cpuid(Reg, 0x80000000);
241 if ((Reg[0] & 0xffffff00) == 0x80000000)
242 {
243 /* Check if CPUID 0x80000001 is supported */
244 if (Reg[0] >= 0x80000001)
245 {
246 /* Check which extended features are available. */
247 __cpuid(Reg, 0x80000001);
248
249 /* Check if NX-bit is supported */
250 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
251
252 /* Now handle each features for each CPU Vendor */
253 switch (Vendor)
254 {
255 case CPU_AMD:
256 if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
257 break;
258 }
259 }
260 }
261
262 /* Return the Feature Bits */
263 return FeatureBits;
264 }
265
266 VOID
267 NTAPI
268 KiGetCacheInformation(VOID)
269 {
270 PKIPCR Pcr = (PKIPCR)KeGetPcr();
271 ULONG Vendor;
272 INT Data[4];
273 ULONG CacheRequests = 0, i;
274 ULONG CurrentRegister;
275 UCHAR RegisterByte;
276 BOOLEAN FirstPass = TRUE;
277
278 /* Set default L2 size */
279 Pcr->SecondLevelCacheSize = 0;
280
281 /* Get the Vendor ID and make sure we support CPUID */
282 Vendor = KiGetCpuVendor();
283 if (!Vendor) return;
284
285 /* Check the Vendor ID */
286 switch (Vendor)
287 {
288 /* Handle Intel case */
289 case CPU_INTEL:
290
291 /*Check if we support CPUID 2 */
292 __cpuid(Data, 0);
293 if (Data[0] >= 2)
294 {
295 /* We need to loop for the number of times CPUID will tell us to */
296 do
297 {
298 /* Do the CPUID call */
299 __cpuid(Data, 2);
300
301 /* Check if it was the first call */
302 if (FirstPass)
303 {
304 /*
305 * The number of times to loop is the first byte. Read
306 * it and then destroy it so we don't get confused.
307 */
308 CacheRequests = Data[0] & 0xFF;
309 Data[0] &= 0xFFFFFF00;
310
311 /* Don't go over this again */
312 FirstPass = FALSE;
313 }
314
315 /* Loop all 4 registers */
316 for (i = 0; i < 4; i++)
317 {
318 /* Get the current register */
319 CurrentRegister = Data[i];
320
321 /*
322 * If the upper bit is set, then this register should
323 * be skipped.
324 */
325 if (CurrentRegister & 0x80000000) continue;
326
327 /* Keep looping for every byte inside this register */
328 while (CurrentRegister)
329 {
330 /* Read a byte, skip a byte. */
331 RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
332 CurrentRegister >>= 8;
333 if (!RegisterByte) continue;
334
335 /*
336 * Valid values are from 0x40 (0 bytes) to 0x49
337 * (32MB), or from 0x80 to 0x89 (same size but
338 * 8-way associative.
339 */
340 if (((RegisterByte > 0x40) &&
341 (RegisterByte <= 0x49)) ||
342 ((RegisterByte > 0x80) &&
343 (RegisterByte <= 0x89)))
344 {
345 /* Mask out only the first nibble */
346 RegisterByte &= 0x0F;
347
348 /* Set the L2 Cache Size */
349 Pcr->SecondLevelCacheSize = 0x10000 <<
350 RegisterByte;
351 }
352 }
353 }
354 } while (--CacheRequests);
355 }
356 break;
357
358 case CPU_AMD:
359
360 /* Check if we support CPUID 0x80000006 */
361 __cpuid(Data, 0x80000000);
362 if (Data[0] >= 6)
363 {
364 /* Get 2nd level cache and tlb size */
365 __cpuid(Data, 0x80000006);
366
367 /* Set the L2 Cache Size */
368 Pcr->SecondLevelCacheSize = (Data[2] & 0xFFFF0000) >> 6;
369 }
370 break;
371 }
372 }
373
374 VOID
375 FASTCALL
376 KiInitializeTss(IN PKTSS64 Tss,
377 IN UINT64 Stack)
378 {
379 PKGDTENTRY64 TssEntry;
380
381 /* Get pointer to the GDT entry */
382 TssEntry = KiGetGdtEntry(KeGetPcr()->GdtBase, KGDT64_SYS_TSS);
383
384 /* Initialize the GDT entry */
385 KiInitGdtEntry(TssEntry, (ULONG64)Tss, sizeof(KTSS64), AMD64_TSS, 0);
386
387 /* Zero out the TSS */
388 RtlZeroMemory(Tss, sizeof(KTSS64));
389
390 /* FIXME: I/O Map? */
391 Tss->IoMapBase = 0x68;
392
393 /* Setup ring 0 stack pointer */
394 Tss->Rsp0 = Stack;
395
396 /* Setup a stack for Double Fault Traps */
397 Tss->Ist[1] = (ULONG64)KiDoubleFaultStack;
398
399 /* Setup a stack for CheckAbort Traps */
400 Tss->Ist[2] = (ULONG64)KiDoubleFaultStack;
401
402 /* Setup a stack for NMI Traps */
403 Tss->Ist[3] = (ULONG64)KiDoubleFaultStack;
404
405 /* Load the task register */
406 __ltr(KGDT64_SYS_TSS);
407 }
408
409 VOID
410 NTAPI
411 KeFlushCurrentTb(VOID)
412 {
413 /* Flush the TLB by resetting CR3 */
414 __writecr3(__readcr3());
415 }
416
417 VOID
418 NTAPI
419 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
420 {
421 /* Restore the CR registers */
422 __writecr0(ProcessorState->SpecialRegisters.Cr0);
423 // __writecr2(ProcessorState->SpecialRegisters.Cr2);
424 __writecr3(ProcessorState->SpecialRegisters.Cr3);
425 __writecr4(ProcessorState->SpecialRegisters.Cr4);
426 __writecr8(ProcessorState->SpecialRegisters.Cr8);
427
428 /* Restore the DR registers */
429 __writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
430 __writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
431 __writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
432 __writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
433 __writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
434 __writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
435
436 /* Restore GDT, IDT, LDT and TSS */
437 __lgdt(&ProcessorState->SpecialRegisters.Gdtr.Limit);
438 // __lldt(&ProcessorState->SpecialRegisters.Ldtr);
439 // __ltr(&ProcessorState->SpecialRegisters.Tr);
440 __lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
441
442 // __ldmxcsr(&ProcessorState->SpecialRegisters.MxCsr); // FIXME
443 // ProcessorState->SpecialRegisters.DebugControl
444 // ProcessorState->SpecialRegisters.LastBranchToRip
445 // ProcessorState->SpecialRegisters.LastBranchFromRip
446 // ProcessorState->SpecialRegisters.LastExceptionToRip
447 // ProcessorState->SpecialRegisters.LastExceptionFromRip
448
449 /* Restore MSRs */
450 __writemsr(X86_MSR_GSBASE, ProcessorState->SpecialRegisters.MsrGsBase);
451 __writemsr(X86_MSR_KERNEL_GSBASE, ProcessorState->SpecialRegisters.MsrGsSwap);
452 __writemsr(X86_MSR_STAR, ProcessorState->SpecialRegisters.MsrStar);
453 __writemsr(X86_MSR_LSTAR, ProcessorState->SpecialRegisters.MsrLStar);
454 __writemsr(X86_MSR_CSTAR, ProcessorState->SpecialRegisters.MsrCStar);
455 __writemsr(X86_MSR_SFMASK, ProcessorState->SpecialRegisters.MsrSyscallMask);
456
457 }
458
459 VOID
460 NTAPI
461 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
462 {
463 /* Save the CR registers */
464 ProcessorState->SpecialRegisters.Cr0 = __readcr0();
465 ProcessorState->SpecialRegisters.Cr2 = __readcr2();
466 ProcessorState->SpecialRegisters.Cr3 = __readcr3();
467 ProcessorState->SpecialRegisters.Cr4 = __readcr4();
468 ProcessorState->SpecialRegisters.Cr8 = __readcr8();
469
470 /* Save the DR registers */
471 ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
472 ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
473 ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
474 ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
475 ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
476 ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
477
478 /* Save GDT, IDT, LDT and TSS */
479 __sgdt(&ProcessorState->SpecialRegisters.Gdtr.Limit);
480 __sldt(&ProcessorState->SpecialRegisters.Ldtr);
481 __str(&ProcessorState->SpecialRegisters.Tr);
482 __sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
483
484 // __stmxcsr(&ProcessorState->SpecialRegisters.MxCsr);
485 // ProcessorState->SpecialRegisters.DebugControl =
486 // ProcessorState->SpecialRegisters.LastBranchToRip =
487 // ProcessorState->SpecialRegisters.LastBranchFromRip =
488 // ProcessorState->SpecialRegisters.LastExceptionToRip =
489 // ProcessorState->SpecialRegisters.LastExceptionFromRip =
490
491 /* Save MSRs */
492 ProcessorState->SpecialRegisters.MsrGsBase = __readmsr(X86_MSR_GSBASE);
493 ProcessorState->SpecialRegisters.MsrGsSwap = __readmsr(X86_MSR_KERNEL_GSBASE);
494 ProcessorState->SpecialRegisters.MsrStar = __readmsr(X86_MSR_STAR);
495 ProcessorState->SpecialRegisters.MsrLStar = __readmsr(X86_MSR_LSTAR);
496 ProcessorState->SpecialRegisters.MsrCStar = __readmsr(X86_MSR_CSTAR);
497 ProcessorState->SpecialRegisters.MsrSyscallMask = __readmsr(X86_MSR_SFMASK);
498 }
499
500 VOID
501 NTAPI
502 KeFlushEntireTb(IN BOOLEAN Invalid,
503 IN BOOLEAN AllProcessors)
504 {
505 KIRQL OldIrql;
506
507 // FIXME: halfplemented
508 /* Raise the IRQL for the TB Flush */
509 OldIrql = KeRaiseIrqlToSynchLevel();
510
511 /* Flush the TB for the Current CPU, and update the flush stamp */
512 KeFlushCurrentTb();
513
514 /* Update the flush stamp and return to original IRQL */
515 InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
516 KeLowerIrql(OldIrql);
517
518 }
519
520 KAFFINITY
521 NTAPI
522 KeQueryActiveProcessors(VOID)
523 {
524 PAGED_CODE();
525
526 /* Simply return the number of active processors */
527 return KeActiveProcessors;
528 }
529
530 NTSTATUS
531 NTAPI
532 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
533 {
534 UNIMPLEMENTED;
535 return STATUS_UNSUCCESSFUL;
536 }
537
538 NTSTATUS
539 NTAPI
540 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
541 {
542 UNIMPLEMENTED;
543 return STATUS_UNSUCCESSFUL;
544 }
545
546 BOOLEAN
547 NTAPI
548 KeInvalidateAllCaches(VOID)
549 {
550 /* Invalidate all caches */
551 __wbinvd();
552 return TRUE;
553 }
554
555 /*
556 * @implemented
557 */
558 ULONG
559 NTAPI
560 KeGetRecommendedSharedDataAlignment(VOID)
561 {
562 /* Return the global variable */
563 return KeLargestCacheLine;
564 }
565
566 /*
567 * @implemented
568 */
569 VOID
570 __cdecl
571 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
572 {
573 /* Capture the context */
574 RtlCaptureContext(&State->ContextFrame);
575
576 /* Capture the control state */
577 KiSaveProcessorControlState(State);
578 }
579
580 /*
581 * @implemented
582 */
583 VOID
584 NTAPI
585 KeSetDmaIoCoherency(IN ULONG Coherency)
586 {
587 /* Save the coherency globally */
588 KiDmaIoCoherency = Coherency;
589 }