693701b13afb5723e796220be41b56cfc7c93ec3
[reactos.git] / ntoskrnl / ke / i386 / cpu.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/i386/cpu.c
5 * PURPOSE: Routines for CPU-level support
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8
9 /* INCLUDES *****************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14
15 /* GLOBALS *******************************************************************/
16
17 /* The TSS to use for Double Fault Traps (INT 0x9) */
18 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
19
20 /* The TSS to use for NMI Fault Traps (INT 0x2) */
21 UCHAR KiNMITSS[KTSS_IO_MAPS];
22
23 /* CPU Features and Flags */
24 ULONG KeI386CpuType;
25 ULONG KeI386CpuStep;
26 ULONG KeProcessorArchitecture;
27 ULONG KeProcessorLevel;
28 ULONG KeProcessorRevision;
29 ULONG KeFeatureBits;
30 ULONG KiFastSystemCallDisable;
31 ULONG KeI386NpxPresent = 0;
32 ULONG KiMXCsrMask = 0;
33 ULONG MxcsrFeatureMask = 0;
34 ULONG KeI386XMMIPresent = 0;
35 ULONG KeI386FxsrPresent = 0;
36 ULONG KeI386MachineType;
37 ULONG Ke386Pae = FALSE;
38 ULONG Ke386NoExecute = FALSE;
39 ULONG KeLargestCacheLine = 0x40;
40 ULONG KeDcacheFlushCount = 0;
41 ULONG KeIcacheFlushCount = 0;
42 ULONG KiDmaIoCoherency = 0;
43 ULONG KePrefetchNTAGranularity = 32;
44 CHAR KeNumberProcessors;
45 KAFFINITY KeActiveProcessors = 1;
46 BOOLEAN KiI386PentiumLockErrataPresent;
47 BOOLEAN KiSMTProcessorsPresent;
48
49 /* The distance between SYSEXIT and IRETD return modes */
50 UCHAR KiSystemCallExitAdjust;
51
52 /* The offset that was applied -- either 0 or the value above */
53 UCHAR KiSystemCallExitAdjusted;
54
55 /* Whether the adjustment was already done once */
56 BOOLEAN KiFastCallCopyDoneOnce;
57
58 /* Flush data */
59 volatile LONG KiTbFlushTimeStamp;
60
61 /* CPU Signatures */
62 static const CHAR CmpIntelID[] = "GenuineIntel";
63 static const CHAR CmpAmdID[] = "AuthenticAMD";
64 static const CHAR CmpCyrixID[] = "CyrixInstead";
65 static const CHAR CmpTransmetaID[] = "GenuineTMx86";
66 static const CHAR CmpCentaurID[] = "CentaurHauls";
67 static const CHAR CmpRiseID[] = "RiseRiseRise";
68
69 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
70
71 VOID
72 NTAPI
73 CPUID(IN ULONG InfoType,
74 OUT PULONG CpuInfoEax,
75 OUT PULONG CpuInfoEbx,
76 OUT PULONG CpuInfoEcx,
77 OUT PULONG CpuInfoEdx)
78 {
79 ULONG CpuInfo[4];
80
81 /* Perform the CPUID Operation */
82 __cpuid((int*)CpuInfo, InfoType);
83
84 /* Return the results */
85 *CpuInfoEax = CpuInfo[0];
86 *CpuInfoEbx = CpuInfo[1];
87 *CpuInfoEcx = CpuInfo[2];
88 *CpuInfoEdx = CpuInfo[3];
89 }
90
91 VOID
92 NTAPI
93 WRMSR(IN ULONG Register,
94 IN LONGLONG Value)
95 {
96 /* Write to the MSR */
97 __writemsr(Register, Value);
98 }
99
100 LONGLONG
101 FASTCALL
102 RDMSR(IN ULONG Register)
103 {
104 /* Read from the MSR */
105 return __readmsr(Register);
106 }
107
108 /* NSC/Cyrix CPU configuration register index */
109 #define CX86_CCR1 0xc1
110
111 /* NSC/Cyrix CPU indexed register access macros */
112 static __inline
113 ULONG
114 getCx86(UCHAR reg)
115 {
116 WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22, reg);
117 return READ_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23);
118 }
119
120 #define setCx86(reg, data) do { \
121 WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22,(reg)); \
122 WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23,(data)); \
123 } while (0)
124
125 /* FUNCTIONS *****************************************************************/
126
127 VOID
128 NTAPI
129 INIT_FUNCTION
130 KiSetProcessorType(VOID)
131 {
132 ULONG EFlags, NewEFlags;
133 ULONG Reg, Dummy;
134 ULONG Stepping, Type;
135
136 /* Start by assuming no CPUID data */
137 KeGetCurrentPrcb()->CpuID = 0;
138
139 /* Save EFlags */
140 EFlags = __readeflags();
141
142 /* XOR out the ID bit and update EFlags */
143 NewEFlags = EFlags ^ EFLAGS_ID;
144 __writeeflags(NewEFlags);
145
146 /* Get them back and see if they were modified */
147 NewEFlags = __readeflags();
148 if (NewEFlags != EFlags)
149 {
150 /* The modification worked, so CPUID exists. Set the ID Bit again. */
151 EFlags |= EFLAGS_ID;
152 __writeeflags(EFlags);
153
154 /* Peform CPUID 0 to see if CPUID 1 is supported */
155 CPUID(0, &Reg, &Dummy, &Dummy, &Dummy);
156 if (Reg > 0)
157 {
158 /* Do CPUID 1 now */
159 CPUID(1, &Reg, &Dummy, &Dummy, &Dummy);
160
161 /*
162 * Get the Stepping and Type. The stepping contains both the
163 * Model and the Step, while the Type contains the returned Type.
164 * We ignore the family.
165 *
166 * For the stepping, we convert this: zzzzzzxy into this: x0y
167 */
168 Stepping = Reg & 0xF0;
169 Stepping <<= 4;
170 Stepping += (Reg & 0xFF);
171 Stepping &= 0xF0F;
172 Type = Reg & 0xF00;
173 Type >>= 8;
174
175 /* Save them in the PRCB */
176 KeGetCurrentPrcb()->CpuID = TRUE;
177 KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
178 KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
179 }
180 else
181 {
182 DPRINT1("CPUID Support lacking\n");
183 }
184 }
185 else
186 {
187 DPRINT1("CPUID Support lacking\n");
188 }
189
190 /* Restore EFLAGS */
191 __writeeflags(EFlags);
192 }
193
194 ULONG
195 NTAPI
196 INIT_FUNCTION
197 KiGetCpuVendor(VOID)
198 {
199 PKPRCB Prcb = KeGetCurrentPrcb();
200 ULONG Vendor[5];
201 ULONG Temp;
202
203 /* Assume no Vendor ID and fail if no CPUID Support. */
204 Prcb->VendorString[0] = 0;
205 if (!Prcb->CpuID) return 0;
206
207 /* Get the Vendor ID and null-terminate it */
208 CPUID(0, &Vendor[0], &Vendor[1], &Vendor[2], &Vendor[3]);
209 Vendor[4] = 0;
210
211 /* Re-arrange vendor string */
212 Temp = Vendor[2];
213 Vendor[2] = Vendor[3];
214 Vendor[3] = Temp;
215
216 /* Copy it to the PRCB and null-terminate it again */
217 RtlCopyMemory(Prcb->VendorString,
218 &Vendor[1],
219 sizeof(Prcb->VendorString) - sizeof(CHAR));
220 Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
221
222 /* Now check the CPU Type */
223 if (!strcmp(Prcb->VendorString, CmpIntelID))
224 {
225 return CPU_INTEL;
226 }
227 else if (!strcmp(Prcb->VendorString, CmpAmdID))
228 {
229 return CPU_AMD;
230 }
231 else if (!strcmp(Prcb->VendorString, CmpCyrixID))
232 {
233 DPRINT1("Cyrix CPU support not fully tested!\n");
234 return CPU_CYRIX;
235 }
236 else if (!strcmp(Prcb->VendorString, CmpTransmetaID))
237 {
238 DPRINT1("Transmeta CPU support not fully tested!\n");
239 return CPU_TRANSMETA;
240 }
241 else if (!strcmp(Prcb->VendorString, CmpCentaurID))
242 {
243 DPRINT1("Centaur CPU support not fully tested!\n");
244 return CPU_CENTAUR;
245 }
246 else if (!strcmp(Prcb->VendorString, CmpRiseID))
247 {
248 DPRINT1("Rise CPU support not fully tested!\n");
249 return CPU_RISE;
250 }
251
252 /* Invalid CPU */
253 return 0;
254 }
255
256 ULONG
257 NTAPI
258 INIT_FUNCTION
259 KiGetFeatureBits(VOID)
260 {
261 PKPRCB Prcb = KeGetCurrentPrcb();
262 ULONG Vendor;
263 ULONG FeatureBits = KF_WORKING_PTE;
264 ULONG Reg[4], Dummy, Ccr1;
265 BOOLEAN ExtendedCPUID = TRUE;
266 ULONG CpuFeatures = 0;
267
268 /* Get the Vendor ID */
269 Vendor = KiGetCpuVendor();
270
271 /* Make sure we got a valid vendor ID at least. */
272 if (!Vendor) return FeatureBits;
273
274 /* Get the CPUID Info. Features are in Reg[3]. */
275 CPUID(1, &Reg[0], &Reg[1], &Dummy, &Reg[3]);
276
277 /* Set the initial APIC ID */
278 Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
279
280 switch (Vendor)
281 {
282 /* Intel CPUs */
283 case CPU_INTEL:
284
285 /* Check if it's a P6 */
286 if (Prcb->CpuType == 6)
287 {
288 /* Perform the special sequence to get the MicroCode Signature */
289 WRMSR(0x8B, 0);
290 CPUID(1, &Dummy, &Dummy, &Dummy, &Dummy);
291 Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
292 }
293 else if (Prcb->CpuType == 5)
294 {
295 /* On P5, enable workaround for the LOCK errata. */
296 KiI386PentiumLockErrataPresent = TRUE;
297 }
298
299 /* Check for broken P6 with bad SMP PTE implementation */
300 if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
301 ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
302 {
303 /* Remove support for correct PTE support. */
304 FeatureBits &= ~KF_WORKING_PTE;
305 }
306
307 /* Check if the CPU is too old to support SYSENTER */
308 if ((Prcb->CpuType < 6) ||
309 ((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
310 {
311 /* Disable it */
312 Reg[3] &= ~0x800;
313 }
314
315 /* Set the current features */
316 CpuFeatures = Reg[3];
317
318 break;
319
320 /* AMD CPUs */
321 case CPU_AMD:
322
323 /* Check if this is a K5 or K6. (family 5) */
324 if ((Reg[0] & 0x0F00) == 0x0500)
325 {
326 /* Get the Model Number */
327 switch (Reg[0] & 0x00F0)
328 {
329 /* Model 1: K5 - 5k86 (initial models) */
330 case 0x0010:
331
332 /* Check if this is Step 0 or 1. They don't support PGE */
333 if ((Reg[0] & 0x000F) > 0x03) break;
334
335 /* Model 0: K5 - SSA5 */
336 case 0x0000:
337
338 /* Model 0 doesn't support PGE at all. */
339 Reg[3] &= ~0x2000;
340 break;
341
342 /* Model 8: K6-2 */
343 case 0x0080:
344
345 /* K6-2, Step 8 and over have support for MTRR. */
346 if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
347 break;
348
349 /* Model 9: K6-III
350 Model D: K6-2+, K6-III+ */
351 case 0x0090:
352 case 0x00D0:
353
354 FeatureBits |= KF_AMDK6MTRR;
355 break;
356 }
357 }
358 else if((Reg[0] & 0x0F00) < 0x0500)
359 {
360 /* Families below 5 don't support PGE, PSE or CMOV at all */
361 Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
362
363 /* They also don't support advanced CPUID functions. */
364 ExtendedCPUID = FALSE;
365 }
366
367 /* Set the current features */
368 CpuFeatures = Reg[3];
369
370 break;
371
372 /* Cyrix CPUs */
373 case CPU_CYRIX:
374
375 /* Workaround the "COMA" bug on 6x family of Cyrix CPUs */
376 if (Prcb->CpuType == 6 &&
377 Prcb->CpuStep <= 1)
378 {
379 /* Get CCR1 value */
380 Ccr1 = getCx86(CX86_CCR1);
381
382 /* Enable the NO_LOCK bit */
383 Ccr1 |= 0x10;
384
385 /* Set the new CCR1 value */
386 setCx86(CX86_CCR1, Ccr1);
387 }
388
389 /* Set the current features */
390 CpuFeatures = Reg[3];
391
392 break;
393
394 /* Transmeta CPUs */
395 case CPU_TRANSMETA:
396
397 /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
398 if ((Reg[0] & 0x0FFF) >= 0x0542)
399 {
400 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
401 FeatureBits |= KF_CMPXCHG8B;
402 }
403
404 break;
405
406 /* Centaur, IDT, Rise and VIA CPUs */
407 case CPU_CENTAUR:
408 case CPU_RISE:
409
410 /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
411 However, this feature exists and operates properly without any additional steps. */
412 FeatureBits |= KF_CMPXCHG8B;
413
414 break;
415 }
416
417 /* Convert all CPUID Feature bits into our format */
418 if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
419 if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
420 if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
421 if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
422 if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
423 if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
424 if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
425 if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
426 if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
427 if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
428 if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
429 if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
430 if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
431 if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
432
433 /* Check if the CPU has hyper-threading */
434 if (CpuFeatures & 0x10000000)
435 {
436 /* Set the number of logical CPUs */
437 Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
438 if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
439 {
440 /* We're on dual-core */
441 KiSMTProcessorsPresent = TRUE;
442 }
443 }
444 else
445 {
446 /* We only have a single CPU */
447 Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
448 }
449
450 /* Check if CPUID 0x80000000 is supported */
451 if (ExtendedCPUID)
452 {
453 /* Do the call */
454 CPUID(0x80000000, &Reg[0], &Dummy, &Dummy, &Dummy);
455 if ((Reg[0] & 0xffffff00) == 0x80000000)
456 {
457 /* Check if CPUID 0x80000001 is supported */
458 if (Reg[0] >= 0x80000001)
459 {
460 /* Check which extended features are available. */
461 CPUID(0x80000001, &Dummy, &Dummy, &Dummy, &Reg[3]);
462
463 /* Check if NX-bit is supported */
464 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
465
466 /* Now handle each features for each CPU Vendor */
467 switch (Vendor)
468 {
469 case CPU_AMD:
470 case CPU_CENTAUR:
471 if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
472 break;
473 }
474 }
475 }
476 }
477
478 DPRINT1("Supported CPU features : %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n",
479 #define print_supported(kf_value) \
480 FeatureBits & kf_value ? #kf_value : ""
481 print_supported(KF_V86_VIS),
482 print_supported(KF_RDTSC),
483 print_supported(KF_CR4),
484 print_supported(KF_CMOV),
485 print_supported(KF_GLOBAL_PAGE),
486 print_supported(KF_LARGE_PAGE),
487 print_supported(KF_MTRR),
488 print_supported(KF_CMPXCHG8B),
489 print_supported(KF_MMX),
490 print_supported(KF_WORKING_PTE),
491 print_supported(KF_PAT),
492 print_supported(KF_FXSR),
493 print_supported(KF_FAST_SYSCALL),
494 print_supported(KF_XMMI),
495 print_supported(KF_3DNOW),
496 print_supported(KF_AMDK6MTRR),
497 print_supported(KF_XMMI64),
498 print_supported(KF_DTS),
499 print_supported(KF_NX_BIT),
500 print_supported(KF_NX_DISABLED),
501 print_supported(KF_NX_ENABLED));
502 #undef print_supported
503
504 /* Return the Feature Bits */
505 return FeatureBits;
506 }
507
508 VOID
509 NTAPI
510 INIT_FUNCTION
511 KiGetCacheInformation(VOID)
512 {
513 PKIPCR Pcr = (PKIPCR)KeGetPcr();
514 ULONG Vendor;
515 ULONG Data[4], Dummy;
516 ULONG CacheRequests = 0, i;
517 ULONG CurrentRegister;
518 UCHAR RegisterByte;
519 ULONG Size, Associativity = 0, CacheLine = 64, CurrentSize = 0;
520 BOOLEAN FirstPass = TRUE;
521
522 /* Set default L2 size */
523 Pcr->SecondLevelCacheSize = 0;
524
525 /* Get the Vendor ID and make sure we support CPUID */
526 Vendor = KiGetCpuVendor();
527 if (!Vendor) return;
528
529 /* Check the Vendor ID */
530 switch (Vendor)
531 {
532 /* Handle Intel case */
533 case CPU_INTEL:
534
535 /*Check if we support CPUID 2 */
536 CPUID(0, &Data[0], &Dummy, &Dummy, &Dummy);
537 if (Data[0] >= 2)
538 {
539 /* We need to loop for the number of times CPUID will tell us to */
540 do
541 {
542 /* Do the CPUID call */
543 CPUID(2, &Data[0], &Data[1], &Data[2], &Data[3]);
544
545 /* Check if it was the first call */
546 if (FirstPass)
547 {
548 /*
549 * The number of times to loop is the first byte. Read
550 * it and then destroy it so we don't get confused.
551 */
552 CacheRequests = Data[0] & 0xFF;
553 Data[0] &= 0xFFFFFF00;
554
555 /* Don't go over this again */
556 FirstPass = FALSE;
557 }
558
559 /* Loop all 4 registers */
560 for (i = 0; i < 4; i++)
561 {
562 /* Get the current register */
563 CurrentRegister = Data[i];
564
565 /*
566 * If the upper bit is set, then this register should
567 * be skipped.
568 */
569 if (CurrentRegister & 0x80000000) continue;
570
571 /* Keep looping for every byte inside this register */
572 while (CurrentRegister)
573 {
574 /* Read a byte, skip a byte. */
575 RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
576 CurrentRegister >>= 8;
577 if (!RegisterByte) continue;
578
579 /*
580 * Valid values are from 0x40 (0 bytes) to 0x49
581 * (32MB), or from 0x80 to 0x89 (same size but
582 * 8-way associative.
583 */
584 if (((RegisterByte > 0x40) && (RegisterByte <= 0x47)) ||
585 ((RegisterByte > 0x78) && (RegisterByte <= 0x7C)) ||
586 ((RegisterByte > 0x80) && (RegisterByte <= 0x85)))
587 {
588 /* Compute associativity */
589 Associativity = 4;
590 if (RegisterByte >= 0x79) Associativity = 8;
591
592 /* Mask out only the first nibble */
593 RegisterByte &= 0x07;
594
595 /* Check if this cache is bigger than the last */
596 Size = 0x10000 << RegisterByte;
597 if ((Size / Associativity) > CurrentSize)
598 {
599 /* Set the L2 Cache Size and Associativity */
600 CurrentSize = Size / Associativity;
601 Pcr->SecondLevelCacheSize = Size;
602 Pcr->SecondLevelCacheAssociativity = Associativity;
603 }
604 }
605 else if ((RegisterByte > 0x21) && (RegisterByte <= 0x29))
606 {
607 /* Set minimum cache line size */
608 if (CacheLine < 128) CacheLine = 128;
609
610 /* Hard-code size/associativity */
611 Associativity = 8;
612 switch (RegisterByte)
613 {
614 case 0x22:
615 Size = 512 * 1024;
616 Associativity = 4;
617 break;
618
619 case 0x23:
620 Size = 1024 * 1024;
621 break;
622
623 case 0x25:
624 Size = 2048 * 1024;
625 break;
626
627 case 0x29:
628 Size = 4096 * 1024;
629 break;
630
631 default:
632 Size = 0;
633 break;
634 }
635
636 /* Check if this cache is bigger than the last */
637 if ((Size / Associativity) > CurrentSize)
638 {
639 /* Set the L2 Cache Size and Associativity */
640 CurrentSize = Size / Associativity;
641 Pcr->SecondLevelCacheSize = Size;
642 Pcr->SecondLevelCacheAssociativity = Associativity;
643 }
644 }
645 else if (((RegisterByte > 0x65) && (RegisterByte < 0x69)) ||
646 (RegisterByte == 0x2C) || (RegisterByte == 0xF0))
647 {
648 /* Indicates L1 cache line of 64 bytes */
649 KePrefetchNTAGranularity = 64;
650 }
651 else if (RegisterByte == 0xF1)
652 {
653 /* Indicates L1 cache line of 128 bytes */
654 KePrefetchNTAGranularity = 128;
655 }
656 else if (((RegisterByte >= 0x4A) && (RegisterByte <= 0x4C)) ||
657 (RegisterByte == 0x78) ||
658 (RegisterByte == 0x7D) ||
659 (RegisterByte == 0x7F) ||
660 (RegisterByte == 0x86) ||
661 (RegisterByte == 0x87))
662 {
663 /* Set minimum cache line size */
664 if (CacheLine < 64) CacheLine = 64;
665
666 /* Hard-code size/associativity */
667 switch (RegisterByte)
668 {
669 case 0x4A:
670 Size = 4 * 1024 * 1024;
671 Associativity = 8;
672 break;
673
674 case 0x4B:
675 Size = 6 * 1024 * 1024;
676 Associativity = 12;
677 break;
678
679 case 0x4C:
680 Size = 8 * 1024 * 1024;
681 Associativity = 16;
682 break;
683
684 case 0x78:
685 Size = 1 * 1024 * 1024;
686 Associativity = 4;
687 break;
688
689 case 0x7D:
690 Size = 2 * 1024 * 1024;
691 Associativity = 8;
692 break;
693
694 case 0x7F:
695 Size = 512 * 1024;
696 Associativity = 2;
697 break;
698
699 case 0x86:
700 Size = 512 * 1024;
701 Associativity = 4;
702 break;
703
704 case 0x87:
705 Size = 1 * 1024 * 1024;
706 Associativity = 8;
707 break;
708
709 default:
710 Size = 0;
711 break;
712 }
713
714 /* Check if this cache is bigger than the last */
715 if ((Size / Associativity) > CurrentSize)
716 {
717 /* Set the L2 Cache Size and Associativity */
718 CurrentSize = Size / Associativity;
719 Pcr->SecondLevelCacheSize = Size;
720 Pcr->SecondLevelCacheAssociativity = Associativity;
721 }
722 }
723 }
724 }
725 } while (--CacheRequests);
726 }
727 break;
728
729 case CPU_AMD:
730
731 /* Check if we support CPUID 0x80000005 */
732 CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
733 if (Data[0] >= 0x80000006)
734 {
735 /* Get L1 size first */
736 CPUID(0x80000005, &Data[0], &Data[1], &Data[2], &Data[3]);
737 KePrefetchNTAGranularity = Data[2] & 0xFF;
738
739 /* Check if we support CPUID 0x80000006 */
740 CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
741 if (Data[0] >= 0x80000006)
742 {
743 /* Get 2nd level cache and tlb size */
744 CPUID(0x80000006, &Data[0], &Data[1], &Data[2], &Data[3]);
745
746 /* Cache line size */
747 CacheLine = Data[2] & 0xFF;
748
749 /* Hardcode associativity */
750 RegisterByte = Data[2] >> 12;
751 switch (RegisterByte)
752 {
753 case 2:
754 Associativity = 2;
755 break;
756
757 case 4:
758 Associativity = 4;
759 break;
760
761 case 6:
762 Associativity = 8;
763 break;
764
765 case 8:
766 case 15:
767 Associativity = 16;
768 break;
769
770 default:
771 Associativity = 1;
772 break;
773 }
774
775 /* Compute size */
776 Size = (Data[2] >> 16) << 10;
777
778 /* Hack for Model 6, Steping 300 */
779 if ((KeGetCurrentPrcb()->CpuType == 6) &&
780 (KeGetCurrentPrcb()->CpuStep == 0x300))
781 {
782 /* Stick 64K in there */
783 Size = 64 * 1024;
784 }
785
786 /* Set the L2 Cache Size and associativity */
787 Pcr->SecondLevelCacheSize = Size;
788 Pcr->SecondLevelCacheAssociativity = Associativity;
789 }
790 }
791 break;
792
793 case CPU_CYRIX:
794 case CPU_TRANSMETA:
795 case CPU_CENTAUR:
796 case CPU_RISE:
797
798 /* FIXME */
799 break;
800 }
801
802 /* Set the cache line */
803 if (CacheLine > KeLargestCacheLine) KeLargestCacheLine = CacheLine;
804 DPRINT1("Prefetch Cache: %d bytes\tL2 Cache: %d bytes\tL2 Cache Line: %d bytes\tL2 Cache Associativity: %d\n",
805 KePrefetchNTAGranularity,
806 Pcr->SecondLevelCacheSize,
807 KeLargestCacheLine,
808 Pcr->SecondLevelCacheAssociativity);
809 }
810
811 VOID
812 NTAPI
813 INIT_FUNCTION
814 KiSetCR0Bits(VOID)
815 {
816 ULONG Cr0;
817
818 /* Save current CR0 */
819 Cr0 = __readcr0();
820
821 /* If this is a 486, enable Write-Protection */
822 if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
823
824 /* Set new Cr0 */
825 __writecr0(Cr0);
826 }
827
828 VOID
829 NTAPI
830 INIT_FUNCTION
831 KiInitializeTSS2(IN PKTSS Tss,
832 IN PKGDTENTRY TssEntry OPTIONAL)
833 {
834 PUCHAR p;
835
836 /* Make sure the GDT Entry is valid */
837 if (TssEntry)
838 {
839 /* Set the Limit */
840 TssEntry->LimitLow = sizeof(KTSS) - 1;
841 TssEntry->HighWord.Bits.LimitHi = 0;
842 }
843
844 /* Now clear the I/O Map */
845 ASSERT(IOPM_COUNT == 1);
846 RtlFillMemory(Tss->IoMaps[0].IoMap, IOPM_FULL_SIZE, 0xFF);
847
848 /* Initialize Interrupt Direction Maps */
849 p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
850 RtlZeroMemory(p, IOPM_DIRECTION_MAP_SIZE);
851
852 /* Add DPMI support for interrupts */
853 p[0] = 4;
854 p[3] = 0x18;
855 p[4] = 0x18;
856
857 /* Initialize the default Interrupt Direction Map */
858 p = Tss->IntDirectionMap;
859 RtlZeroMemory(Tss->IntDirectionMap, IOPM_DIRECTION_MAP_SIZE);
860
861 /* Add DPMI support */
862 p[0] = 4;
863 p[3] = 0x18;
864 p[4] = 0x18;
865 }
866
867 VOID
868 NTAPI
869 KiInitializeTSS(IN PKTSS Tss)
870 {
871 /* Set an invalid map base */
872 Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
873
874 /* Disable traps during Task Switches */
875 Tss->Flags = 0;
876
877 /* Set LDT and Ring 0 SS */
878 Tss->LDT = 0;
879 Tss->Ss0 = KGDT_R0_DATA;
880 }
881
882 VOID
883 FASTCALL
884 INIT_FUNCTION
885 Ki386InitializeTss(IN PKTSS Tss,
886 IN PKIDTENTRY Idt,
887 IN PKGDTENTRY Gdt)
888 {
889 PKGDTENTRY TssEntry, TaskGateEntry;
890
891 /* Initialize the boot TSS. */
892 TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
893 TssEntry->HighWord.Bits.Type = I386_TSS;
894 TssEntry->HighWord.Bits.Pres = 1;
895 TssEntry->HighWord.Bits.Dpl = 0;
896 KiInitializeTSS2(Tss, TssEntry);
897 KiInitializeTSS(Tss);
898
899 /* Load the task register */
900 Ke386SetTr(KGDT_TSS);
901
902 /* Setup the Task Gate for Double Fault Traps */
903 TaskGateEntry = (PKGDTENTRY)&Idt[8];
904 TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
905 TaskGateEntry->HighWord.Bits.Pres = 1;
906 TaskGateEntry->HighWord.Bits.Dpl = 0;
907 ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
908
909 /* Initialize the TSS used for handling double faults. */
910 Tss = (PKTSS)KiDoubleFaultTSS;
911 KiInitializeTSS(Tss);
912 Tss->CR3 = __readcr3();
913 Tss->Esp0 = KiDoubleFaultStack;
914 Tss->Esp = KiDoubleFaultStack;
915 Tss->Eip = PtrToUlong(KiTrap08);
916 Tss->Cs = KGDT_R0_CODE;
917 Tss->Fs = KGDT_R0_PCR;
918 Tss->Ss = Ke386GetSs();
919 Tss->Es = KGDT_R3_DATA | RPL_MASK;
920 Tss->Ds = KGDT_R3_DATA | RPL_MASK;
921
922 /* Setup the Double Trap TSS entry in the GDT */
923 TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
924 TssEntry->HighWord.Bits.Type = I386_TSS;
925 TssEntry->HighWord.Bits.Pres = 1;
926 TssEntry->HighWord.Bits.Dpl = 0;
927 TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
928 TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
929 TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
930 TssEntry->LimitLow = KTSS_IO_MAPS;
931
932 /* Now setup the NMI Task Gate */
933 TaskGateEntry = (PKGDTENTRY)&Idt[2];
934 TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
935 TaskGateEntry->HighWord.Bits.Pres = 1;
936 TaskGateEntry->HighWord.Bits.Dpl = 0;
937 ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
938
939 /* Initialize the actual TSS */
940 Tss = (PKTSS)KiNMITSS;
941 KiInitializeTSS(Tss);
942 Tss->CR3 = __readcr3();
943 Tss->Esp0 = KiDoubleFaultStack;
944 Tss->Esp = KiDoubleFaultStack;
945 Tss->Eip = PtrToUlong(KiTrap02);
946 Tss->Cs = KGDT_R0_CODE;
947 Tss->Fs = KGDT_R0_PCR;
948 Tss->Ss = Ke386GetSs();
949 Tss->Es = KGDT_R3_DATA | RPL_MASK;
950 Tss->Ds = KGDT_R3_DATA | RPL_MASK;
951
952 /* And its associated TSS Entry */
953 TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
954 TssEntry->HighWord.Bits.Type = I386_TSS;
955 TssEntry->HighWord.Bits.Pres = 1;
956 TssEntry->HighWord.Bits.Dpl = 0;
957 TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
958 TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
959 TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
960 TssEntry->LimitLow = KTSS_IO_MAPS;
961 }
962
963 VOID
964 NTAPI
965 KeFlushCurrentTb(VOID)
966 {
967 /* Flush the TLB by resetting CR3 */
968 __writecr3(__readcr3());
969 }
970
971 VOID
972 NTAPI
973 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
974 {
975 PKGDTENTRY TssEntry;
976
977 //
978 // Restore the CR registers
979 //
980 __writecr0(ProcessorState->SpecialRegisters.Cr0);
981 Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
982 __writecr3(ProcessorState->SpecialRegisters.Cr3);
983 if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
984
985 //
986 // Restore the DR registers
987 //
988 __writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
989 __writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
990 __writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
991 __writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
992 __writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
993 __writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
994
995 //
996 // Restore GDT and IDT
997 //
998 Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
999 __lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
1000
1001 //
1002 // Clear the busy flag so we don't crash if we reload the same selector
1003 //
1004 TssEntry = (PKGDTENTRY)(ProcessorState->SpecialRegisters.Gdtr.Base +
1005 ProcessorState->SpecialRegisters.Tr);
1006 TssEntry->HighWord.Bytes.Flags1 &= ~0x2;
1007
1008 //
1009 // Restore TSS and LDT
1010 //
1011 Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
1012 Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
1013 }
1014
1015 VOID
1016 NTAPI
1017 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
1018 {
1019 /* Save the CR registers */
1020 ProcessorState->SpecialRegisters.Cr0 = __readcr0();
1021 ProcessorState->SpecialRegisters.Cr2 = __readcr2();
1022 ProcessorState->SpecialRegisters.Cr3 = __readcr3();
1023 ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
1024 __readcr4() : 0;
1025
1026 /* Save the DR registers */
1027 ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
1028 ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
1029 ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
1030 ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
1031 ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
1032 ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
1033 __writedr(7, 0);
1034
1035 /* Save GDT, IDT, LDT and TSS */
1036 Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
1037 __sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
1038 ProcessorState->SpecialRegisters.Tr = Ke386GetTr();
1039 ProcessorState->SpecialRegisters.Ldtr = Ke386GetLocalDescriptorTable();
1040 }
1041
1042 VOID
1043 NTAPI
1044 INIT_FUNCTION
1045 KiInitializeMachineType(VOID)
1046 {
1047 /* Set the Machine Type we got from NTLDR */
1048 KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
1049 }
1050
1051 ULONG_PTR
1052 NTAPI
1053 INIT_FUNCTION
1054 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
1055 {
1056 /* Set CS and ESP */
1057 WRMSR(0x174, KGDT_R0_CODE);
1058 WRMSR(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
1059
1060 /* Set LSTAR */
1061 WRMSR(0x176, (ULONG_PTR)KiFastCallEntry);
1062 return 0;
1063 }
1064
1065 VOID
1066 NTAPI
1067 INIT_FUNCTION
1068 KiRestoreFastSyscallReturnState(VOID)
1069 {
1070 /* Check if the CPU Supports fast system call */
1071 if (KeFeatureBits & KF_FAST_SYSCALL)
1072 {
1073 /* Check if it has been disabled */
1074 if (!KiFastSystemCallDisable)
1075 {
1076 /* Do an IPI to enable it */
1077 KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
1078
1079 /* It's enabled, so use the proper exit stub */
1080 KiFastCallExitHandler = KiSystemCallSysExitReturn;
1081 DPRINT1("Support for SYSENTER detected.\n");
1082 }
1083 else
1084 {
1085 /* Disable fast system call */
1086 KeFeatureBits &= ~KF_FAST_SYSCALL;
1087 KiFastCallExitHandler = KiSystemCallTrapReturn;
1088 DPRINT1("Support for SYSENTER disabled.\n");
1089 }
1090 }
1091 else
1092 {
1093 /* Use the IRET handler */
1094 KiFastCallExitHandler = KiSystemCallTrapReturn;
1095 DPRINT1("No support for SYSENTER detected.\n");
1096 }
1097 }
1098
1099 ULONG_PTR
1100 NTAPI
1101 INIT_FUNCTION
1102 Ki386EnableDE(IN ULONG_PTR Context)
1103 {
1104 /* Enable DE */
1105 __writecr4(__readcr4() | CR4_DE);
1106 return 0;
1107 }
1108
1109 ULONG_PTR
1110 NTAPI
1111 INIT_FUNCTION
1112 Ki386EnableFxsr(IN ULONG_PTR Context)
1113 {
1114 /* Enable FXSR */
1115 __writecr4(__readcr4() | CR4_FXSR);
1116 return 0;
1117 }
1118
1119 ULONG_PTR
1120 NTAPI
1121 INIT_FUNCTION
1122 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
1123 {
1124 PKIDTENTRY IdtEntry;
1125
1126 /* Get the IDT Entry for Interrupt 0x13 */
1127 IdtEntry = &((PKIPCR)KeGetPcr())->IDT[0x13];
1128
1129 /* Set it up */
1130 IdtEntry->Selector = KGDT_R0_CODE;
1131 IdtEntry->Offset = ((ULONG_PTR)KiTrap13 & 0xFFFF);
1132 IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap13 >> 16) & 0xFFFF;
1133 ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
1134 ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
1135 ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
1136
1137 /* Enable XMMI exceptions */
1138 __writecr4(__readcr4() | CR4_XMMEXCPT);
1139 return 0;
1140 }
1141
1142 VOID
1143 NTAPI
1144 INIT_FUNCTION
1145 KiI386PentiumLockErrataFixup(VOID)
1146 {
1147 KDESCRIPTOR IdtDescriptor;
1148 PKIDTENTRY NewIdt, NewIdt2;
1149
1150 /* Allocate memory for a new IDT */
1151 NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
1152
1153 /* Put everything after the first 7 entries on a new page */
1154 NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
1155
1156 /* Disable interrupts */
1157 _disable();
1158
1159 /* Get the current IDT and copy it */
1160 __sidt(&IdtDescriptor.Limit);
1161 RtlCopyMemory(NewIdt2,
1162 (PVOID)IdtDescriptor.Base,
1163 IdtDescriptor.Limit + 1);
1164 IdtDescriptor.Base = (ULONG)NewIdt2;
1165
1166 /* Set the new IDT */
1167 __lidt(&IdtDescriptor.Limit);
1168 ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
1169
1170 /* Restore interrupts */
1171 _enable();
1172
1173 /* Set the first 7 entries as read-only to produce a fault */
1174 MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
1175 }
1176
1177 BOOLEAN
1178 NTAPI
1179 KeDisableInterrupts(VOID)
1180 {
1181 ULONG Flags;
1182 BOOLEAN Return;
1183
1184 /* Get EFLAGS and check if the interrupt bit is set */
1185 Flags = __readeflags();
1186 Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
1187
1188 /* Disable interrupts */
1189 _disable();
1190 return Return;
1191 }
1192
1193 BOOLEAN
1194 NTAPI
1195 KeInvalidateAllCaches(VOID)
1196 {
1197 /* Only supported on Pentium Pro and higher */
1198 if (KeI386CpuType < 6) return FALSE;
1199
1200 /* Invalidate all caches */
1201 __wbinvd();
1202 return TRUE;
1203 }
1204
1205 VOID
1206 FASTCALL
1207 KeZeroPages(IN PVOID Address,
1208 IN ULONG Size)
1209 {
1210 /* Not using XMMI in this routine */
1211 RtlZeroMemory(Address, Size);
1212 }
1213
1214 VOID
1215 NTAPI
1216 KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame,
1217 IN PKEXCEPTION_FRAME ExceptionFrame)
1218 {
1219 PKPRCB Prcb = KeGetCurrentPrcb();
1220
1221 //
1222 // Save full context
1223 //
1224 Prcb->ProcessorState.ContextFrame.ContextFlags = CONTEXT_FULL |
1225 CONTEXT_DEBUG_REGISTERS;
1226 KeTrapFrameToContext(TrapFrame, NULL, &Prcb->ProcessorState.ContextFrame);
1227
1228 //
1229 // Save control registers
1230 //
1231 KiSaveProcessorControlState(&Prcb->ProcessorState);
1232 }
1233
1234 BOOLEAN
1235 NTAPI
1236 INIT_FUNCTION
1237 KiIsNpxPresent(VOID)
1238 {
1239 ULONG Cr0;
1240 USHORT Magic;
1241
1242 /* Set magic */
1243 Magic = 0xFFFF;
1244
1245 /* Read CR0 and mask out FPU flags */
1246 Cr0 = __readcr0() & ~(CR0_MP | CR0_TS | CR0_EM | CR0_ET);
1247
1248 /* Store on FPU stack */
1249 #ifdef _MSC_VER
1250 __asm fninit;
1251 __asm fnstsw Magic;
1252 #else
1253 asm volatile ("fninit;" "fnstsw %0" : "+m"(Magic));
1254 #endif
1255
1256 /* Magic should now be cleared */
1257 if (Magic & 0xFF)
1258 {
1259 /* You don't have an FPU -- enable emulation for now */
1260 __writecr0(Cr0 | CR0_EM | CR0_TS);
1261 return FALSE;
1262 }
1263
1264 /* You have an FPU, enable it */
1265 Cr0 |= CR0_ET;
1266
1267 /* Enable INT 16 on 486 and higher */
1268 if (KeGetCurrentPrcb()->CpuType >= 3) Cr0 |= CR0_NE;
1269
1270 /* Set FPU state */
1271 __writecr0(Cr0 | CR0_EM | CR0_TS);
1272 return TRUE;
1273 }
1274
1275 BOOLEAN
1276 NTAPI
1277 INIT_FUNCTION
1278 KiIsNpxErrataPresent(VOID)
1279 {
1280 BOOLEAN ErrataPresent;
1281 ULONG Cr0;
1282 volatile double Value1, Value2;
1283
1284 /* Disable interrupts */
1285 _disable();
1286
1287 /* Read CR0 and remove FPU flags */
1288 Cr0 = __readcr0();
1289 __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1290
1291 /* Initialize FPU state */
1292 Ke386FnInit();
1293
1294 /* Multiply the magic values and divide, we should get the result back */
1295 Value1 = 4195835.0;
1296 Value2 = 3145727.0;
1297 ErrataPresent = (Value1 * Value2 / 3145727.0) != 4195835.0;
1298
1299 /* Restore CR0 */
1300 __writecr0(Cr0);
1301
1302 /* Enable interrupts */
1303 _enable();
1304
1305 /* Return if there's an errata */
1306 return ErrataPresent;
1307 }
1308
1309 VOID
1310 NTAPI
1311 KiFlushNPXState(IN PFLOATING_SAVE_AREA SaveArea)
1312 {
1313 ULONG EFlags, Cr0;
1314 PKTHREAD Thread, NpxThread;
1315 PFX_SAVE_AREA FxSaveArea;
1316
1317 /* Save volatiles and disable interrupts */
1318 EFlags = __readeflags();
1319 _disable();
1320
1321 /* Save the PCR and get the current thread */
1322 Thread = KeGetCurrentThread();
1323
1324 /* Check if we're already loaded */
1325 if (Thread->NpxState != NPX_STATE_LOADED)
1326 {
1327 /* If there's nothing to load, quit */
1328 if (!SaveArea) return;
1329
1330 /* Need FXSR support for this */
1331 ASSERT(KeI386FxsrPresent == TRUE);
1332
1333 /* Check for sane CR0 */
1334 Cr0 = __readcr0();
1335 if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1336 {
1337 /* Mask out FPU flags */
1338 __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1339 }
1340
1341 /* Get the NPX thread and check its FPU state */
1342 NpxThread = KeGetCurrentPrcb()->NpxThread;
1343 if ((NpxThread) && (NpxThread->NpxState == NPX_STATE_LOADED))
1344 {
1345 /* Get the FX frame and store the state there */
1346 FxSaveArea = KiGetThreadNpxArea(NpxThread);
1347 Ke386FxSave(FxSaveArea);
1348
1349 /* NPX thread has lost its state */
1350 NpxThread->NpxState = NPX_STATE_NOT_LOADED;
1351 }
1352
1353 /* Now load NPX state from the NPX area */
1354 FxSaveArea = KiGetThreadNpxArea(Thread);
1355 Ke386FxStore(FxSaveArea);
1356 }
1357 else
1358 {
1359 /* Check for sane CR0 */
1360 Cr0 = __readcr0();
1361 if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1362 {
1363 /* Mask out FPU flags */
1364 __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1365 }
1366
1367 /* Get FX frame */
1368 FxSaveArea = KiGetThreadNpxArea(Thread);
1369 Thread->NpxState = NPX_STATE_NOT_LOADED;
1370
1371 /* Save state if supported by CPU */
1372 if (KeI386FxsrPresent) Ke386FxSave(FxSaveArea);
1373 }
1374
1375 /* Now save the FN state wherever it was requested */
1376 if (SaveArea) Ke386FnSave(SaveArea);
1377
1378 /* Clear NPX thread */
1379 KeGetCurrentPrcb()->NpxThread = NULL;
1380
1381 /* Add the CR0 from the NPX frame */
1382 Cr0 |= NPX_STATE_NOT_LOADED;
1383 Cr0 |= FxSaveArea->Cr0NpxState;
1384 __writecr0(Cr0);
1385
1386 /* Restore interrupt state */
1387 __writeeflags(EFlags);
1388 }
1389
1390 /* PUBLIC FUNCTIONS **********************************************************/
1391
1392 /*
1393 * @implemented
1394 */
1395 VOID
1396 NTAPI
1397 KiCoprocessorError(VOID)
1398 {
1399 PFX_SAVE_AREA NpxArea;
1400
1401 /* Get the FPU area */
1402 NpxArea = KiGetThreadNpxArea(KeGetCurrentThread());
1403
1404 /* Set CR0_TS */
1405 NpxArea->Cr0NpxState = CR0_TS;
1406 __writecr0(__readcr0() | CR0_TS);
1407 }
1408
1409 /*
1410 * @implemented
1411 */
1412 NTSTATUS
1413 NTAPI
1414 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
1415 {
1416 PFNSAVE_FORMAT FpState;
1417 ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
1418 DPRINT1("%s is not really implemented\n", __FUNCTION__);
1419
1420 /* check if we are doing software emulation */
1421 if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
1422
1423 FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
1424 if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
1425
1426 *((PVOID *) Save) = FpState;
1427 #ifdef __GNUC__
1428 asm volatile("fnsave %0\n\t" : "=m" (*FpState));
1429 #else
1430 __asm
1431 {
1432 fnsave [FpState]
1433 };
1434 #endif
1435
1436 KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
1437 return STATUS_SUCCESS;
1438 }
1439
1440 /*
1441 * @implemented
1442 */
1443 NTSTATUS
1444 NTAPI
1445 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
1446 {
1447 PFNSAVE_FORMAT FpState = *((PVOID *) Save);
1448 ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
1449 DPRINT1("%s is not really implemented\n", __FUNCTION__);
1450
1451 #ifdef __GNUC__
1452 asm volatile("fnclex\n\t");
1453 asm volatile("frstor %0\n\t" : "=m" (*FpState));
1454 #else
1455 __asm
1456 {
1457 fnclex
1458 frstor [FpState]
1459 };
1460 #endif
1461
1462 ExFreePool(FpState);
1463 return STATUS_SUCCESS;
1464 }
1465
1466 /*
1467 * @implemented
1468 */
1469 ULONG
1470 NTAPI
1471 KeGetRecommendedSharedDataAlignment(VOID)
1472 {
1473 /* Return the global variable */
1474 return KeLargestCacheLine;
1475 }
1476
1477 VOID
1478 NTAPI
1479 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
1480 IN PVOID Ignored1,
1481 IN PVOID Ignored2,
1482 IN PVOID Ignored3)
1483 {
1484 /* Signal this packet as done */
1485 KiIpiSignalPacketDone(PacketContext);
1486
1487 /* Flush the TB for the Current CPU */
1488 KeFlushCurrentTb();
1489 }
1490
1491 /*
1492 * @implemented
1493 */
1494 VOID
1495 NTAPI
1496 KeFlushEntireTb(IN BOOLEAN Invalid,
1497 IN BOOLEAN AllProcessors)
1498 {
1499 KIRQL OldIrql;
1500 #ifdef CONFIG_SMP
1501 KAFFINITY TargetAffinity;
1502 PKPRCB Prcb = KeGetCurrentPrcb();
1503 #endif
1504
1505 /* Raise the IRQL for the TB Flush */
1506 OldIrql = KeRaiseIrqlToSynchLevel();
1507
1508 #ifdef CONFIG_SMP
1509 /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1510
1511 /* Get the current processor affinity, and exclude ourselves */
1512 TargetAffinity = KeActiveProcessors;
1513 TargetAffinity &= ~Prcb->SetMember;
1514
1515 /* Make sure this is MP */
1516 if (TargetAffinity)
1517 {
1518 /* Send an IPI TB flush to the other processors */
1519 KiIpiSendPacket(TargetAffinity,
1520 KiFlushTargetEntireTb,
1521 NULL,
1522 0,
1523 NULL);
1524 }
1525 #endif
1526
1527 /* Flush the TB for the Current CPU, and update the flush stamp */
1528 KeFlushCurrentTb();
1529
1530 #ifdef CONFIG_SMP
1531 /* If this is MP, wait for the other processors to finish */
1532 if (TargetAffinity)
1533 {
1534 /* Sanity check */
1535 ASSERT(Prcb == (volatile PKPRCB)KeGetCurrentPrcb());
1536
1537 /* FIXME: TODO */
1538 ASSERTMSG("Not yet implemented\n", FALSE);
1539 }
1540 #endif
1541
1542 /* Update the flush stamp and return to original IRQL */
1543 InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
1544 KeLowerIrql(OldIrql);
1545 }
1546
1547 /*
1548 * @implemented
1549 */
1550 VOID
1551 NTAPI
1552 KeSetDmaIoCoherency(IN ULONG Coherency)
1553 {
1554 /* Save the coherency globally */
1555 KiDmaIoCoherency = Coherency;
1556 }
1557
1558 /*
1559 * @implemented
1560 */
1561 KAFFINITY
1562 NTAPI
1563 KeQueryActiveProcessors(VOID)
1564 {
1565 PAGED_CODE();
1566
1567 /* Simply return the number of active processors */
1568 return KeActiveProcessors;
1569 }
1570
1571 /*
1572 * @implemented
1573 */
1574 VOID
1575 __cdecl
1576 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1577 {
1578 /* Capture the context */
1579 RtlCaptureContext(&State->ContextFrame);
1580
1581 /* Capture the control state */
1582 KiSaveProcessorControlState(State);
1583 }