[NTOSKRNL]
[reactos.git] / reactos / ntoskrnl / ke / i386 / cpu.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: GPL - See COPYING in the top level directory
4 * FILE: ntoskrnl/ke/i386/cpu.c
5 * PURPOSE: Routines for CPU-level support
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 */
8
9 /* INCLUDES *****************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14
15 /* GLOBALS *******************************************************************/
16
17 /* The TSS to use for Double Fault Traps (INT 0x9) */
18 UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
19
20 /* The TSS to use for NMI Fault Traps (INT 0x2) */
21 UCHAR KiNMITSS[KTSS_IO_MAPS];
22
23 /* CPU Features and Flags */
24 ULONG KeI386CpuType;
25 ULONG KeI386CpuStep;
26 ULONG KeProcessorArchitecture;
27 ULONG KeProcessorLevel;
28 ULONG KeProcessorRevision;
29 ULONG KeFeatureBits;
30 ULONG KiFastSystemCallDisable;
31 ULONG KeI386NpxPresent = 0;
32 ULONG KiMXCsrMask = 0;
33 ULONG MxcsrFeatureMask = 0;
34 ULONG KeI386XMMIPresent = 0;
35 ULONG KeI386FxsrPresent = 0;
36 ULONG KeI386MachineType;
37 ULONG Ke386Pae = FALSE;
38 ULONG Ke386NoExecute = FALSE;
39 ULONG KeLargestCacheLine = 0x40;
40 ULONG KeDcacheFlushCount = 0;
41 ULONG KeIcacheFlushCount = 0;
42 ULONG KiDmaIoCoherency = 0;
43 ULONG KePrefetchNTAGranularity = 32;
44 CHAR KeNumberProcessors;
45 KAFFINITY KeActiveProcessors = 1;
46 BOOLEAN KiI386PentiumLockErrataPresent;
47 BOOLEAN KiSMTProcessorsPresent;
48
49 /* The distance between SYSEXIT and IRETD return modes */
50 UCHAR KiSystemCallExitAdjust;
51
52 /* The offset that was applied -- either 0 or the value above */
53 UCHAR KiSystemCallExitAdjusted;
54
55 /* Whether the adjustment was already done once */
56 BOOLEAN KiFastCallCopyDoneOnce;
57
58 /* Flush data */
59 volatile LONG KiTbFlushTimeStamp;
60
61 /* CPU Signatures */
62 static const CHAR CmpIntelID[] = "GenuineIntel";
63 static const CHAR CmpAmdID[] = "AuthenticAMD";
64 static const CHAR CmpCyrixID[] = "CyrixInstead";
65 static const CHAR CmpTransmetaID[] = "GenuineTMx86";
66 static const CHAR CmpCentaurID[] = "CentaurHauls";
67 static const CHAR CmpRiseID[] = "RiseRiseRise";
68
69 /* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
70
71 VOID
72 NTAPI
73 CPUID(IN ULONG InfoType,
74 OUT PULONG CpuInfoEax,
75 OUT PULONG CpuInfoEbx,
76 OUT PULONG CpuInfoEcx,
77 OUT PULONG CpuInfoEdx)
78 {
79 ULONG CpuInfo[4];
80
81 /* Perform the CPUID Operation */
82 __cpuid((int*)CpuInfo, InfoType);
83
84 /* Return the results */
85 *CpuInfoEax = CpuInfo[0];
86 *CpuInfoEbx = CpuInfo[1];
87 *CpuInfoEcx = CpuInfo[2];
88 *CpuInfoEdx = CpuInfo[3];
89 }
90
91 VOID
92 NTAPI
93 WRMSR(IN ULONG Register,
94 IN LONGLONG Value)
95 {
96 /* Write to the MSR */
97 __writemsr(Register, Value);
98 }
99
100 LONGLONG
101 FASTCALL
102 RDMSR(IN ULONG Register)
103 {
104 /* Read from the MSR */
105 return __readmsr(Register);
106 }
107
108 /* NSC/Cyrix CPU configuration register index */
109 #define CX86_CCR1 0xc1
110
111 /* NSC/Cyrix CPU indexed register access macros */
112 static __inline
113 ULONG
114 getCx86(UCHAR reg)
115 {
116 WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22, reg);
117 return READ_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23);
118 }
119
120 #define setCx86(reg, data) do { \
121 WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22,(reg)); \
122 WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23,(data)); \
123 } while (0)
124
125 /* FUNCTIONS *****************************************************************/
126
127 VOID
128 NTAPI
129 INIT_FUNCTION
130 KiSetProcessorType(VOID)
131 {
132 ULONG EFlags, NewEFlags;
133 ULONG Reg, Dummy;
134 ULONG Stepping, Type;
135
136 /* Start by assuming no CPUID data */
137 KeGetCurrentPrcb()->CpuID = 0;
138
139 /* Save EFlags */
140 EFlags = __readeflags();
141
142 /* XOR out the ID bit and update EFlags */
143 NewEFlags = EFlags ^ EFLAGS_ID;
144 __writeeflags(NewEFlags);
145
146 /* Get them back and see if they were modified */
147 NewEFlags = __readeflags();
148 if (NewEFlags != EFlags)
149 {
150 /* The modification worked, so CPUID exists. Set the ID Bit again. */
151 EFlags |= EFLAGS_ID;
152 __writeeflags(EFlags);
153
154 /* Peform CPUID 0 to see if CPUID 1 is supported */
155 CPUID(0, &Reg, &Dummy, &Dummy, &Dummy);
156 if (Reg > 0)
157 {
158 /* Do CPUID 1 now */
159 CPUID(1, &Reg, &Dummy, &Dummy, &Dummy);
160
161 /*
162 * Get the Stepping and Type. The stepping contains both the
163 * Model and the Step, while the Type contains the returned Type.
164 * We ignore the family.
165 *
166 * For the stepping, we convert this: zzzzzzxy into this: x0y
167 */
168 Stepping = Reg & 0xF0;
169 Stepping <<= 4;
170 Stepping += (Reg & 0xFF);
171 Stepping &= 0xF0F;
172 Type = Reg & 0xF00;
173 Type >>= 8;
174
175 /* Save them in the PRCB */
176 KeGetCurrentPrcb()->CpuID = TRUE;
177 KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
178 KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
179 }
180 else
181 {
182 DPRINT1("CPUID Support lacking\n");
183 }
184 }
185 else
186 {
187 DPRINT1("CPUID Support lacking\n");
188 }
189
190 /* Restore EFLAGS */
191 __writeeflags(EFlags);
192 }
193
194 ULONG
195 NTAPI
196 INIT_FUNCTION
197 KiGetCpuVendor(VOID)
198 {
199 PKPRCB Prcb = KeGetCurrentPrcb();
200 ULONG Vendor[5];
201 ULONG Temp;
202
203 /* Assume no Vendor ID and fail if no CPUID Support. */
204 Prcb->VendorString[0] = 0;
205 if (!Prcb->CpuID) return 0;
206
207 /* Get the Vendor ID and null-terminate it */
208 CPUID(0, &Vendor[0], &Vendor[1], &Vendor[2], &Vendor[3]);
209 Vendor[4] = 0;
210
211 /* Re-arrange vendor string */
212 Temp = Vendor[2];
213 Vendor[2] = Vendor[3];
214 Vendor[3] = Temp;
215
216 /* Copy it to the PRCB and null-terminate it again */
217 RtlCopyMemory(Prcb->VendorString,
218 &Vendor[1],
219 sizeof(Prcb->VendorString) - sizeof(CHAR));
220 Prcb->VendorString[sizeof(Prcb->VendorString) - sizeof(CHAR)] = ANSI_NULL;
221
222 /* Now check the CPU Type */
223 if (!strcmp(Prcb->VendorString, CmpIntelID))
224 {
225 return CPU_INTEL;
226 }
227 else if (!strcmp(Prcb->VendorString, CmpAmdID))
228 {
229 return CPU_AMD;
230 }
231 else if (!strcmp(Prcb->VendorString, CmpCyrixID))
232 {
233 DPRINT1("Cyrix CPU support not fully tested!\n");
234 return CPU_CYRIX;
235 }
236 else if (!strcmp(Prcb->VendorString, CmpTransmetaID))
237 {
238 DPRINT1("Transmeta CPU support not fully tested!\n");
239 return CPU_TRANSMETA;
240 }
241 else if (!strcmp(Prcb->VendorString, CmpCentaurID))
242 {
243 DPRINT1("Centaur CPU support not fully tested!\n");
244 return CPU_CENTAUR;
245 }
246 else if (!strcmp(Prcb->VendorString, CmpRiseID))
247 {
248 DPRINT1("Rise CPU support not fully tested!\n");
249 return CPU_RISE;
250 }
251
252 /* Invalid CPU */
253 return 0;
254 }
255
256 ULONG
257 NTAPI
258 INIT_FUNCTION
259 KiGetFeatureBits(VOID)
260 {
261 PKPRCB Prcb = KeGetCurrentPrcb();
262 ULONG Vendor;
263 ULONG FeatureBits = KF_WORKING_PTE;
264 ULONG Reg[4], Dummy, Ccr1;
265 BOOLEAN ExtendedCPUID = TRUE;
266 ULONG CpuFeatures = 0;
267
268 /* Get the Vendor ID */
269 Vendor = KiGetCpuVendor();
270
271 /* Make sure we got a valid vendor ID at least. */
272 if (!Vendor) return FeatureBits;
273
274 /* Get the CPUID Info. Features are in Reg[3]. */
275 CPUID(1, &Reg[0], &Reg[1], &Dummy, &Reg[3]);
276
277 /* Set the initial APIC ID */
278 Prcb->InitialApicId = (UCHAR)(Reg[1] >> 24);
279
280 switch (Vendor)
281 {
282 /* Intel CPUs */
283 case CPU_INTEL:
284
285 /* Check if it's a P6 */
286 if (Prcb->CpuType == 6)
287 {
288 /* Perform the special sequence to get the MicroCode Signature */
289 WRMSR(0x8B, 0);
290 CPUID(1, &Dummy, &Dummy, &Dummy, &Dummy);
291 Prcb->UpdateSignature.QuadPart = RDMSR(0x8B);
292 }
293 else if (Prcb->CpuType == 5)
294 {
295 /* On P5, enable workaround for the LOCK errata. */
296 KiI386PentiumLockErrataPresent = TRUE;
297 }
298
299 /* Check for broken P6 with bad SMP PTE implementation */
300 if (((Reg[0] & 0x0FF0) == 0x0610 && (Reg[0] & 0x000F) <= 0x9) ||
301 ((Reg[0] & 0x0FF0) == 0x0630 && (Reg[0] & 0x000F) <= 0x4))
302 {
303 /* Remove support for correct PTE support. */
304 FeatureBits &= ~KF_WORKING_PTE;
305 }
306
307 /* Virtualbox claims to have no SYSENTER support,
308 * which is false for processors >= Pentium Pro */
309 if(Prcb->CpuType >= 6)
310 {
311 Reg[3] |= 0x800;
312 }
313
314 /* Check if the CPU is too old to support SYSENTER */
315 if ((Reg[0] & 0x0FFF3FFF) < 0x00000633)
316 {
317 /* Disable it */
318 Reg[3] &= ~0x800;
319 }
320
321 /* Set the current features */
322 CpuFeatures = Reg[3];
323
324 break;
325
326 /* AMD CPUs */
327 case CPU_AMD:
328
329 /* Check if this is a K5 or K6. (family 5) */
330 if ((Reg[0] & 0x0F00) == 0x0500)
331 {
332 /* Get the Model Number */
333 switch (Reg[0] & 0x00F0)
334 {
335 /* Model 1: K5 - 5k86 (initial models) */
336 case 0x0010:
337
338 /* Check if this is Step 0 or 1. They don't support PGE */
339 if ((Reg[0] & 0x000F) > 0x03) break;
340
341 /* Model 0: K5 - SSA5 */
342 case 0x0000:
343
344 /* Model 0 doesn't support PGE at all. */
345 Reg[3] &= ~0x2000;
346 break;
347
348 /* Model 8: K6-2 */
349 case 0x0080:
350
351 /* K6-2, Step 8 and over have support for MTRR. */
352 if ((Reg[0] & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
353 break;
354
355 /* Model 9: K6-III
356 Model D: K6-2+, K6-III+ */
357 case 0x0090:
358 case 0x00D0:
359
360 FeatureBits |= KF_AMDK6MTRR;
361 break;
362 }
363 }
364 else if((Reg[0] & 0x0F00) < 0x0500)
365 {
366 /* Families below 5 don't support PGE, PSE or CMOV at all */
367 Reg[3] &= ~(0x08 | 0x2000 | 0x8000);
368
369 /* They also don't support advanced CPUID functions. */
370 ExtendedCPUID = FALSE;
371 }
372
373 /* Set the current features */
374 CpuFeatures = Reg[3];
375
376 break;
377
378 /* Cyrix CPUs */
379 case CPU_CYRIX:
380
381 /* Workaround the "COMA" bug on 6x family of Cyrix CPUs */
382 if (Prcb->CpuType == 6 &&
383 Prcb->CpuStep <= 1)
384 {
385 /* Get CCR1 value */
386 Ccr1 = getCx86(CX86_CCR1);
387
388 /* Enable the NO_LOCK bit */
389 Ccr1 |= 0x10;
390
391 /* Set the new CCR1 value */
392 setCx86(CX86_CCR1, Ccr1);
393 }
394
395 /* Set the current features */
396 CpuFeatures = Reg[3];
397
398 break;
399
400 /* Transmeta CPUs */
401 case CPU_TRANSMETA:
402
403 /* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
404 if ((Reg[0] & 0x0FFF) >= 0x0542)
405 {
406 WRMSR(0x80860004, RDMSR(0x80860004) | 0x0100);
407 FeatureBits |= KF_CMPXCHG8B;
408 }
409
410 break;
411
412 /* Centaur, IDT, Rise and VIA CPUs */
413 case CPU_CENTAUR:
414 case CPU_RISE:
415
416 /* These CPUs don't report the presence of CMPXCHG8B through CPUID.
417 However, this feature exists and operates properly without any additional steps. */
418 FeatureBits |= KF_CMPXCHG8B;
419
420 break;
421 }
422
423 /* Convert all CPUID Feature bits into our format */
424 if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
425 if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
426 if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
427 if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
428 if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
429 if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
430 if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
431 if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
432 if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
433 if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
434 if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
435 if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
436 if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
437 if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
438
439 /* Check if the CPU has hyper-threading */
440 if (CpuFeatures & 0x10000000)
441 {
442 /* Set the number of logical CPUs */
443 Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(Reg[1] >> 16);
444 if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
445 {
446 /* We're on dual-core */
447 KiSMTProcessorsPresent = TRUE;
448 }
449 }
450 else
451 {
452 /* We only have a single CPU */
453 Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
454 }
455
456 /* Check if CPUID 0x80000000 is supported */
457 if (ExtendedCPUID)
458 {
459 /* Do the call */
460 CPUID(0x80000000, &Reg[0], &Dummy, &Dummy, &Dummy);
461 if ((Reg[0] & 0xffffff00) == 0x80000000)
462 {
463 /* Check if CPUID 0x80000001 is supported */
464 if (Reg[0] >= 0x80000001)
465 {
466 /* Check which extended features are available. */
467 CPUID(0x80000001, &Dummy, &Dummy, &Dummy, &Reg[3]);
468
469 /* Check if NX-bit is supported */
470 if (Reg[3] & 0x00100000) FeatureBits |= KF_NX_BIT;
471
472 /* Now handle each features for each CPU Vendor */
473 switch (Vendor)
474 {
475 case CPU_AMD:
476 case CPU_CENTAUR:
477 if (Reg[3] & 0x80000000) FeatureBits |= KF_3DNOW;
478 break;
479 }
480 }
481 }
482 }
483
484 DPRINT1("Supported CPU features :\n");
485 #define print_supported(kf_value) \
486 if(FeatureBits & kf_value) DPRINT1("\t" #kf_value "\n")
487 print_supported(KF_V86_VIS);
488 print_supported(KF_RDTSC);
489 print_supported(KF_CR4);
490 print_supported(KF_CMOV);
491 print_supported(KF_GLOBAL_PAGE);
492 print_supported(KF_LARGE_PAGE);
493 print_supported(KF_MTRR);
494 print_supported(KF_CMPXCHG8B);
495 print_supported(KF_MMX);
496 print_supported(KF_WORKING_PTE);
497 print_supported(KF_PAT);
498 print_supported(KF_FXSR);
499 print_supported(KF_FAST_SYSCALL);
500 print_supported(KF_XMMI);
501 print_supported(KF_3DNOW);
502 print_supported(KF_AMDK6MTRR);
503 print_supported(KF_XMMI64);
504 print_supported(KF_DTS);
505 print_supported(KF_NX_BIT);
506 print_supported(KF_NX_DISABLED);
507 print_supported(KF_NX_ENABLED);
508 #undef print_supported
509
510 /* Return the Feature Bits */
511 return FeatureBits;
512 }
513
514 VOID
515 NTAPI
516 INIT_FUNCTION
517 KiGetCacheInformation(VOID)
518 {
519 PKIPCR Pcr = (PKIPCR)KeGetPcr();
520 ULONG Vendor;
521 ULONG Data[4], Dummy;
522 ULONG CacheRequests = 0, i;
523 ULONG CurrentRegister;
524 UCHAR RegisterByte;
525 ULONG Size, Associativity = 0, CacheLine = 64, CurrentSize = 0;
526 BOOLEAN FirstPass = TRUE;
527
528 /* Set default L2 size */
529 Pcr->SecondLevelCacheSize = 0;
530
531 /* Get the Vendor ID and make sure we support CPUID */
532 Vendor = KiGetCpuVendor();
533 if (!Vendor) return;
534
535 /* Check the Vendor ID */
536 switch (Vendor)
537 {
538 /* Handle Intel case */
539 case CPU_INTEL:
540
541 /*Check if we support CPUID 2 */
542 CPUID(0, &Data[0], &Dummy, &Dummy, &Dummy);
543 if (Data[0] >= 2)
544 {
545 /* We need to loop for the number of times CPUID will tell us to */
546 do
547 {
548 /* Do the CPUID call */
549 CPUID(2, &Data[0], &Data[1], &Data[2], &Data[3]);
550
551 /* Check if it was the first call */
552 if (FirstPass)
553 {
554 /*
555 * The number of times to loop is the first byte. Read
556 * it and then destroy it so we don't get confused.
557 */
558 CacheRequests = Data[0] & 0xFF;
559 Data[0] &= 0xFFFFFF00;
560
561 /* Don't go over this again */
562 FirstPass = FALSE;
563 }
564
565 /* Loop all 4 registers */
566 for (i = 0; i < 4; i++)
567 {
568 /* Get the current register */
569 CurrentRegister = Data[i];
570
571 /*
572 * If the upper bit is set, then this register should
573 * be skipped.
574 */
575 if (CurrentRegister & 0x80000000) continue;
576
577 /* Keep looping for every byte inside this register */
578 while (CurrentRegister)
579 {
580 /* Read a byte, skip a byte. */
581 RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
582 CurrentRegister >>= 8;
583 if (!RegisterByte) continue;
584
585 /*
586 * Valid values are from 0x40 (0 bytes) to 0x49
587 * (32MB), or from 0x80 to 0x89 (same size but
588 * 8-way associative.
589 */
590 if (((RegisterByte > 0x40) && (RegisterByte <= 0x47)) ||
591 ((RegisterByte > 0x78) && (RegisterByte <= 0x7C)) ||
592 ((RegisterByte > 0x80) && (RegisterByte <= 0x85)))
593 {
594 /* Compute associativity */
595 Associativity = 4;
596 if (RegisterByte >= 0x79) Associativity = 8;
597
598 /* Mask out only the first nibble */
599 RegisterByte &= 0x07;
600
601 /* Check if this cache is bigger than the last */
602 Size = 0x10000 << RegisterByte;
603 if ((Size / Associativity) > CurrentSize)
604 {
605 /* Set the L2 Cache Size and Associativity */
606 CurrentSize = Size / Associativity;
607 Pcr->SecondLevelCacheSize = Size;
608 Pcr->SecondLevelCacheAssociativity = Associativity;
609 }
610 }
611 else if ((RegisterByte > 0x21) && (RegisterByte <= 0x29))
612 {
613 /* Set minimum cache line size */
614 if (CacheLine < 128) CacheLine = 128;
615
616 /* Hard-code size/associativity */
617 Associativity = 8;
618 switch (RegisterByte)
619 {
620 case 0x22:
621 Size = 512 * 1024;
622 Associativity = 4;
623 break;
624
625 case 0x23:
626 Size = 1024 * 1024;
627 break;
628
629 case 0x25:
630 Size = 2048 * 1024;
631 break;
632
633 case 0x29:
634 Size = 4096 * 1024;
635 break;
636
637 default:
638 Size = 0;
639 break;
640 }
641
642 /* Check if this cache is bigger than the last */
643 if ((Size / Associativity) > CurrentSize)
644 {
645 /* Set the L2 Cache Size and Associativity */
646 CurrentSize = Size / Associativity;
647 Pcr->SecondLevelCacheSize = Size;
648 Pcr->SecondLevelCacheAssociativity = Associativity;
649 }
650 }
651 else if (((RegisterByte > 0x65) && (RegisterByte < 0x69)) ||
652 (RegisterByte == 0x2C) || (RegisterByte == 0xF0))
653 {
654 /* Indicates L1 cache line of 64 bytes */
655 KePrefetchNTAGranularity = 64;
656 }
657 else if (RegisterByte == 0xF1)
658 {
659 /* Indicates L1 cache line of 128 bytes */
660 KePrefetchNTAGranularity = 128;
661 }
662 else if (((RegisterByte >= 0x4A) && (RegisterByte <= 0x4C)) ||
663 (RegisterByte == 0x78) ||
664 (RegisterByte == 0x7D) ||
665 (RegisterByte == 0x7F) ||
666 (RegisterByte == 0x86) ||
667 (RegisterByte == 0x87))
668 {
669 /* Set minimum cache line size */
670 if (CacheLine < 64) CacheLine = 64;
671
672 /* Hard-code size/associativity */
673 switch (RegisterByte)
674 {
675 case 0x4A:
676 Size = 4 * 1024 * 1024;
677 Associativity = 8;
678 break;
679
680 case 0x4B:
681 Size = 6 * 1024 * 1024;
682 Associativity = 12;
683 break;
684
685 case 0x4C:
686 Size = 8 * 1024 * 1024;
687 Associativity = 16;
688 break;
689
690 case 0x78:
691 Size = 1 * 1024 * 1024;
692 Associativity = 4;
693 break;
694
695 case 0x7D:
696 Size = 2 * 1024 * 1024;
697 Associativity = 8;
698 break;
699
700 case 0x7F:
701 Size = 512 * 1024;
702 Associativity = 2;
703 break;
704
705 case 0x86:
706 Size = 512 * 1024;
707 Associativity = 4;
708 break;
709
710 case 0x87:
711 Size = 1 * 1024 * 1024;
712 Associativity = 8;
713 break;
714
715 default:
716 Size = 0;
717 break;
718 }
719
720 /* Check if this cache is bigger than the last */
721 if ((Size / Associativity) > CurrentSize)
722 {
723 /* Set the L2 Cache Size and Associativity */
724 CurrentSize = Size / Associativity;
725 Pcr->SecondLevelCacheSize = Size;
726 Pcr->SecondLevelCacheAssociativity = Associativity;
727 }
728 }
729 }
730 }
731 } while (--CacheRequests);
732 }
733 break;
734
735 case CPU_AMD:
736
737 /* Check if we support CPUID 0x80000005 */
738 CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
739 if (Data[0] >= 0x80000006)
740 {
741 /* Get L1 size first */
742 CPUID(0x80000005, &Data[0], &Data[1], &Data[2], &Data[3]);
743 KePrefetchNTAGranularity = Data[2] & 0xFF;
744
745 /* Check if we support CPUID 0x80000006 */
746 CPUID(0x80000000, &Data[0], &Data[1], &Data[2], &Data[3]);
747 if (Data[0] >= 0x80000006)
748 {
749 /* Get 2nd level cache and tlb size */
750 CPUID(0x80000006, &Data[0], &Data[1], &Data[2], &Data[3]);
751
752 /* Cache line size */
753 CacheLine = Data[2] & 0xFF;
754
755 /* Hardcode associativity */
756 RegisterByte = Data[2] >> 12;
757 switch (RegisterByte)
758 {
759 case 2:
760 Associativity = 2;
761 break;
762
763 case 4:
764 Associativity = 4;
765 break;
766
767 case 6:
768 Associativity = 8;
769 break;
770
771 case 8:
772 case 15:
773 Associativity = 16;
774 break;
775
776 default:
777 Associativity = 1;
778 break;
779 }
780
781 /* Compute size */
782 Size = (Data[2] >> 16) << 10;
783
784 /* Hack for Model 6, Steping 300 */
785 if ((KeGetCurrentPrcb()->CpuType == 6) &&
786 (KeGetCurrentPrcb()->CpuStep == 0x300))
787 {
788 /* Stick 64K in there */
789 Size = 64 * 1024;
790 }
791
792 /* Set the L2 Cache Size and associativity */
793 Pcr->SecondLevelCacheSize = Size;
794 Pcr->SecondLevelCacheAssociativity = Associativity;
795 }
796 }
797 break;
798
799 case CPU_CYRIX:
800 case CPU_TRANSMETA:
801 case CPU_CENTAUR:
802 case CPU_RISE:
803
804 /* FIXME */
805 break;
806 }
807
808 /* Set the cache line */
809 if (CacheLine > KeLargestCacheLine) KeLargestCacheLine = CacheLine;
810 DPRINT1("Prefetch Cache: %d bytes\tL2 Cache: %d bytes\tL2 Cache Line: %d bytes\tL2 Cache Associativity: %d\n",
811 KePrefetchNTAGranularity,
812 Pcr->SecondLevelCacheSize,
813 KeLargestCacheLine,
814 Pcr->SecondLevelCacheAssociativity);
815 }
816
817 VOID
818 NTAPI
819 INIT_FUNCTION
820 KiSetCR0Bits(VOID)
821 {
822 ULONG Cr0;
823
824 /* Save current CR0 */
825 Cr0 = __readcr0();
826
827 /* If this is a 486, enable Write-Protection */
828 if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
829
830 /* Set new Cr0 */
831 __writecr0(Cr0);
832 }
833
834 VOID
835 NTAPI
836 INIT_FUNCTION
837 KiInitializeTSS2(IN PKTSS Tss,
838 IN PKGDTENTRY TssEntry OPTIONAL)
839 {
840 PUCHAR p;
841
842 /* Make sure the GDT Entry is valid */
843 if (TssEntry)
844 {
845 /* Set the Limit */
846 TssEntry->LimitLow = sizeof(KTSS) - 1;
847 TssEntry->HighWord.Bits.LimitHi = 0;
848 }
849
850 /* Now clear the I/O Map */
851 ASSERT(IOPM_COUNT == 1);
852 RtlFillMemory(Tss->IoMaps[0].IoMap, IOPM_FULL_SIZE, 0xFF);
853
854 /* Initialize Interrupt Direction Maps */
855 p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
856 RtlZeroMemory(p, IOPM_DIRECTION_MAP_SIZE);
857
858 /* Add DPMI support for interrupts */
859 p[0] = 4;
860 p[3] = 0x18;
861 p[4] = 0x18;
862
863 /* Initialize the default Interrupt Direction Map */
864 p = Tss->IntDirectionMap;
865 RtlZeroMemory(Tss->IntDirectionMap, IOPM_DIRECTION_MAP_SIZE);
866
867 /* Add DPMI support */
868 p[0] = 4;
869 p[3] = 0x18;
870 p[4] = 0x18;
871 }
872
873 VOID
874 NTAPI
875 KiInitializeTSS(IN PKTSS Tss)
876 {
877 /* Set an invalid map base */
878 Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
879
880 /* Disable traps during Task Switches */
881 Tss->Flags = 0;
882
883 /* Set LDT and Ring 0 SS */
884 Tss->LDT = 0;
885 Tss->Ss0 = KGDT_R0_DATA;
886 }
887
888 VOID
889 FASTCALL
890 INIT_FUNCTION
891 Ki386InitializeTss(IN PKTSS Tss,
892 IN PKIDTENTRY Idt,
893 IN PKGDTENTRY Gdt)
894 {
895 PKGDTENTRY TssEntry, TaskGateEntry;
896
897 /* Initialize the boot TSS. */
898 TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
899 TssEntry->HighWord.Bits.Type = I386_TSS;
900 TssEntry->HighWord.Bits.Pres = 1;
901 TssEntry->HighWord.Bits.Dpl = 0;
902 KiInitializeTSS2(Tss, TssEntry);
903 KiInitializeTSS(Tss);
904
905 /* Load the task register */
906 Ke386SetTr(KGDT_TSS);
907
908 /* Setup the Task Gate for Double Fault Traps */
909 TaskGateEntry = (PKGDTENTRY)&Idt[8];
910 TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
911 TaskGateEntry->HighWord.Bits.Pres = 1;
912 TaskGateEntry->HighWord.Bits.Dpl = 0;
913 ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
914
915 /* Initialize the TSS used for handling double faults. */
916 Tss = (PKTSS)KiDoubleFaultTSS;
917 KiInitializeTSS(Tss);
918 Tss->CR3 = __readcr3();
919 Tss->Esp0 = KiDoubleFaultStack;
920 Tss->Esp = KiDoubleFaultStack;
921 Tss->Eip = PtrToUlong(KiTrap08);
922 Tss->Cs = KGDT_R0_CODE;
923 Tss->Fs = KGDT_R0_PCR;
924 Tss->Ss = Ke386GetSs();
925 Tss->Es = KGDT_R3_DATA | RPL_MASK;
926 Tss->Ds = KGDT_R3_DATA | RPL_MASK;
927
928 /* Setup the Double Trap TSS entry in the GDT */
929 TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
930 TssEntry->HighWord.Bits.Type = I386_TSS;
931 TssEntry->HighWord.Bits.Pres = 1;
932 TssEntry->HighWord.Bits.Dpl = 0;
933 TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
934 TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
935 TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
936 TssEntry->LimitLow = KTSS_IO_MAPS;
937
938 /* Now setup the NMI Task Gate */
939 TaskGateEntry = (PKGDTENTRY)&Idt[2];
940 TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
941 TaskGateEntry->HighWord.Bits.Pres = 1;
942 TaskGateEntry->HighWord.Bits.Dpl = 0;
943 ((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
944
945 /* Initialize the actual TSS */
946 Tss = (PKTSS)KiNMITSS;
947 KiInitializeTSS(Tss);
948 Tss->CR3 = __readcr3();
949 Tss->Esp0 = KiDoubleFaultStack;
950 Tss->Esp = KiDoubleFaultStack;
951 Tss->Eip = PtrToUlong(KiTrap02);
952 Tss->Cs = KGDT_R0_CODE;
953 Tss->Fs = KGDT_R0_PCR;
954 Tss->Ss = Ke386GetSs();
955 Tss->Es = KGDT_R3_DATA | RPL_MASK;
956 Tss->Ds = KGDT_R3_DATA | RPL_MASK;
957
958 /* And its associated TSS Entry */
959 TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
960 TssEntry->HighWord.Bits.Type = I386_TSS;
961 TssEntry->HighWord.Bits.Pres = 1;
962 TssEntry->HighWord.Bits.Dpl = 0;
963 TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
964 TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
965 TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
966 TssEntry->LimitLow = KTSS_IO_MAPS;
967 }
968
969 VOID
970 NTAPI
971 KeFlushCurrentTb(VOID)
972 {
973 /* Flush the TLB by resetting CR3 */
974 __writecr3(__readcr3());
975 }
976
977 VOID
978 NTAPI
979 KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
980 {
981 PKGDTENTRY TssEntry;
982
983 //
984 // Restore the CR registers
985 //
986 __writecr0(ProcessorState->SpecialRegisters.Cr0);
987 Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
988 __writecr3(ProcessorState->SpecialRegisters.Cr3);
989 if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
990
991 //
992 // Restore the DR registers
993 //
994 __writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
995 __writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
996 __writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
997 __writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
998 __writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
999 __writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
1000
1001 //
1002 // Restore GDT and IDT
1003 //
1004 Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
1005 __lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
1006
1007 //
1008 // Clear the busy flag so we don't crash if we reload the same selector
1009 //
1010 TssEntry = (PKGDTENTRY)(ProcessorState->SpecialRegisters.Gdtr.Base +
1011 ProcessorState->SpecialRegisters.Tr);
1012 TssEntry->HighWord.Bytes.Flags1 &= ~0x2;
1013
1014 //
1015 // Restore TSS and LDT
1016 //
1017 Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
1018 Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
1019 }
1020
1021 VOID
1022 NTAPI
1023 KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
1024 {
1025 /* Save the CR registers */
1026 ProcessorState->SpecialRegisters.Cr0 = __readcr0();
1027 ProcessorState->SpecialRegisters.Cr2 = __readcr2();
1028 ProcessorState->SpecialRegisters.Cr3 = __readcr3();
1029 ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
1030 __readcr4() : 0;
1031
1032 /* Save the DR registers */
1033 ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
1034 ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
1035 ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
1036 ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
1037 ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
1038 ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
1039 __writedr(7, 0);
1040
1041 /* Save GDT, IDT, LDT and TSS */
1042 Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
1043 __sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
1044 ProcessorState->SpecialRegisters.Tr = Ke386GetTr();
1045 ProcessorState->SpecialRegisters.Ldtr = Ke386GetLocalDescriptorTable();
1046 }
1047
1048 VOID
1049 NTAPI
1050 INIT_FUNCTION
1051 KiInitializeMachineType(VOID)
1052 {
1053 /* Set the Machine Type we got from NTLDR */
1054 KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
1055 }
1056
1057 ULONG_PTR
1058 NTAPI
1059 INIT_FUNCTION
1060 KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
1061 {
1062 /* Set CS and ESP */
1063 WRMSR(0x174, KGDT_R0_CODE);
1064 WRMSR(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
1065
1066 /* Set LSTAR */
1067 WRMSR(0x176, (ULONG_PTR)KiFastCallEntry);
1068 return 0;
1069 }
1070
1071 VOID
1072 NTAPI
1073 INIT_FUNCTION
1074 KiRestoreFastSyscallReturnState(VOID)
1075 {
1076 /* Check if the CPU Supports fast system call */
1077 if (KeFeatureBits & KF_FAST_SYSCALL)
1078 {
1079 /* Check if it has been disabled */
1080 if (!KiFastSystemCallDisable)
1081 {
1082 /* Do an IPI to enable it */
1083 KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
1084
1085 /* It's enabled, so use the proper exit stub */
1086 KiFastCallExitHandler = KiSystemCallSysExitReturn;
1087 DPRINT1("Support for SYSENTER detected.\n");
1088 }
1089 else
1090 {
1091 /* Disable fast system call */
1092 KeFeatureBits &= ~KF_FAST_SYSCALL;
1093 KiFastCallExitHandler = KiSystemCallTrapReturn;
1094 DPRINT1("Support for SYSENTER disabled.\n");
1095 }
1096 }
1097 else
1098 {
1099 /* Use the IRET handler */
1100 KiFastCallExitHandler = KiSystemCallTrapReturn;
1101 DPRINT1("No support for SYSENTER detected.\n");
1102 }
1103 }
1104
1105 ULONG_PTR
1106 NTAPI
1107 INIT_FUNCTION
1108 Ki386EnableDE(IN ULONG_PTR Context)
1109 {
1110 /* Enable DE */
1111 __writecr4(__readcr4() | CR4_DE);
1112 return 0;
1113 }
1114
1115 ULONG_PTR
1116 NTAPI
1117 INIT_FUNCTION
1118 Ki386EnableFxsr(IN ULONG_PTR Context)
1119 {
1120 /* Enable FXSR */
1121 __writecr4(__readcr4() | CR4_FXSR);
1122 return 0;
1123 }
1124
1125 ULONG_PTR
1126 NTAPI
1127 INIT_FUNCTION
1128 Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
1129 {
1130 PKIDTENTRY IdtEntry;
1131
1132 /* Get the IDT Entry for Interrupt 0x13 */
1133 IdtEntry = &((PKIPCR)KeGetPcr())->IDT[0x13];
1134
1135 /* Set it up */
1136 IdtEntry->Selector = KGDT_R0_CODE;
1137 IdtEntry->Offset = ((ULONG_PTR)KiTrap13 & 0xFFFF);
1138 IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap13 >> 16) & 0xFFFF;
1139 ((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
1140 ((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
1141 ((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
1142
1143 /* Enable XMMI exceptions */
1144 __writecr4(__readcr4() | CR4_XMMEXCPT);
1145 return 0;
1146 }
1147
1148 VOID
1149 NTAPI
1150 INIT_FUNCTION
1151 KiI386PentiumLockErrataFixup(VOID)
1152 {
1153 KDESCRIPTOR IdtDescriptor;
1154 PKIDTENTRY NewIdt, NewIdt2;
1155
1156 /* Allocate memory for a new IDT */
1157 NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
1158
1159 /* Put everything after the first 7 entries on a new page */
1160 NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
1161
1162 /* Disable interrupts */
1163 _disable();
1164
1165 /* Get the current IDT and copy it */
1166 __sidt(&IdtDescriptor.Limit);
1167 RtlCopyMemory(NewIdt2,
1168 (PVOID)IdtDescriptor.Base,
1169 IdtDescriptor.Limit + 1);
1170 IdtDescriptor.Base = (ULONG)NewIdt2;
1171
1172 /* Set the new IDT */
1173 __lidt(&IdtDescriptor.Limit);
1174 ((PKIPCR)KeGetPcr())->IDT = NewIdt2;
1175
1176 /* Restore interrupts */
1177 _enable();
1178
1179 /* Set the first 7 entries as read-only to produce a fault */
1180 MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
1181 }
1182
1183 BOOLEAN
1184 NTAPI
1185 KeDisableInterrupts(VOID)
1186 {
1187 ULONG Flags;
1188 BOOLEAN Return;
1189
1190 /* Get EFLAGS and check if the interrupt bit is set */
1191 Flags = __readeflags();
1192 Return = (Flags & EFLAGS_INTERRUPT_MASK) ? TRUE: FALSE;
1193
1194 /* Disable interrupts */
1195 _disable();
1196 return Return;
1197 }
1198
1199 BOOLEAN
1200 NTAPI
1201 KeInvalidateAllCaches(VOID)
1202 {
1203 /* Only supported on Pentium Pro and higher */
1204 if (KeI386CpuType < 6) return FALSE;
1205
1206 /* Invalidate all caches */
1207 __wbinvd();
1208 return TRUE;
1209 }
1210
1211 VOID
1212 FASTCALL
1213 KeZeroPages(IN PVOID Address,
1214 IN ULONG Size)
1215 {
1216 /* Not using XMMI in this routine */
1217 RtlZeroMemory(Address, Size);
1218 }
1219
1220 VOID
1221 NTAPI
1222 KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame,
1223 IN PKEXCEPTION_FRAME ExceptionFrame)
1224 {
1225 PKPRCB Prcb = KeGetCurrentPrcb();
1226
1227 //
1228 // Save full context
1229 //
1230 Prcb->ProcessorState.ContextFrame.ContextFlags = CONTEXT_FULL |
1231 CONTEXT_DEBUG_REGISTERS;
1232 KeTrapFrameToContext(TrapFrame, NULL, &Prcb->ProcessorState.ContextFrame);
1233
1234 //
1235 // Save control registers
1236 //
1237 KiSaveProcessorControlState(&Prcb->ProcessorState);
1238 }
1239
1240 BOOLEAN
1241 NTAPI
1242 INIT_FUNCTION
1243 KiIsNpxPresent(VOID)
1244 {
1245 ULONG Cr0;
1246 USHORT Magic;
1247
1248 /* Set magic */
1249 Magic = 0xFFFF;
1250
1251 /* Read CR0 and mask out FPU flags */
1252 Cr0 = __readcr0() & ~(CR0_MP | CR0_TS | CR0_EM | CR0_ET);
1253
1254 /* Store on FPU stack */
1255 #ifdef _MSC_VER
1256 __asm fninit;
1257 __asm fnstsw Magic;
1258 #else
1259 asm volatile ("fninit;" "fnstsw %0" : "+m"(Magic));
1260 #endif
1261
1262 /* Magic should now be cleared */
1263 if (Magic & 0xFF)
1264 {
1265 /* You don't have an FPU -- enable emulation for now */
1266 __writecr0(Cr0 | CR0_EM | CR0_TS);
1267 return FALSE;
1268 }
1269
1270 /* You have an FPU, enable it */
1271 Cr0 |= CR0_ET;
1272
1273 /* Enable INT 16 on 486 and higher */
1274 if (KeGetCurrentPrcb()->CpuType >= 3) Cr0 |= CR0_NE;
1275
1276 /* Set FPU state */
1277 __writecr0(Cr0 | CR0_EM | CR0_TS);
1278 return TRUE;
1279 }
1280
1281 BOOLEAN
1282 NTAPI
1283 INIT_FUNCTION
1284 KiIsNpxErrataPresent(VOID)
1285 {
1286 BOOLEAN ErrataPresent;
1287 ULONG Cr0;
1288 volatile double Value1, Value2;
1289
1290 /* Disable interrupts */
1291 _disable();
1292
1293 /* Read CR0 and remove FPU flags */
1294 Cr0 = __readcr0();
1295 __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1296
1297 /* Initialize FPU state */
1298 Ke386FnInit();
1299
1300 /* Multiply the magic values and divide, we should get the result back */
1301 Value1 = 4195835.0;
1302 Value2 = 3145727.0;
1303 ErrataPresent = (Value1 * Value2 / 3145727.0) != 4195835.0;
1304
1305 /* Restore CR0 */
1306 __writecr0(Cr0);
1307
1308 /* Enable interrupts */
1309 _enable();
1310
1311 /* Return if there's an errata */
1312 return ErrataPresent;
1313 }
1314
1315 VOID
1316 NTAPI
1317 KiFlushNPXState(IN PFLOATING_SAVE_AREA SaveArea)
1318 {
1319 ULONG EFlags, Cr0;
1320 PKTHREAD Thread, NpxThread;
1321 PFX_SAVE_AREA FxSaveArea;
1322
1323 /* Save volatiles and disable interrupts */
1324 EFlags = __readeflags();
1325 _disable();
1326
1327 /* Save the PCR and get the current thread */
1328 Thread = KeGetCurrentThread();
1329
1330 /* Check if we're already loaded */
1331 if (Thread->NpxState != NPX_STATE_LOADED)
1332 {
1333 /* If there's nothing to load, quit */
1334 if (!SaveArea) return;
1335
1336 /* Need FXSR support for this */
1337 ASSERT(KeI386FxsrPresent == TRUE);
1338
1339 /* Check for sane CR0 */
1340 Cr0 = __readcr0();
1341 if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1342 {
1343 /* Mask out FPU flags */
1344 __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1345 }
1346
1347 /* Get the NPX thread and check its FPU state */
1348 NpxThread = KeGetCurrentPrcb()->NpxThread;
1349 if ((NpxThread) && (NpxThread->NpxState == NPX_STATE_LOADED))
1350 {
1351 /* Get the FX frame and store the state there */
1352 FxSaveArea = KiGetThreadNpxArea(NpxThread);
1353 Ke386FxSave(FxSaveArea);
1354
1355 /* NPX thread has lost its state */
1356 NpxThread->NpxState = NPX_STATE_NOT_LOADED;
1357 }
1358
1359 /* Now load NPX state from the NPX area */
1360 FxSaveArea = KiGetThreadNpxArea(Thread);
1361 Ke386FxStore(FxSaveArea);
1362 }
1363 else
1364 {
1365 /* Check for sane CR0 */
1366 Cr0 = __readcr0();
1367 if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
1368 {
1369 /* Mask out FPU flags */
1370 __writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
1371 }
1372
1373 /* Get FX frame */
1374 FxSaveArea = KiGetThreadNpxArea(Thread);
1375 Thread->NpxState = NPX_STATE_NOT_LOADED;
1376
1377 /* Save state if supported by CPU */
1378 if (KeI386FxsrPresent) Ke386FxSave(FxSaveArea);
1379 }
1380
1381 /* Now save the FN state wherever it was requested */
1382 if (SaveArea) Ke386FnSave(SaveArea);
1383
1384 /* Clear NPX thread */
1385 KeGetCurrentPrcb()->NpxThread = NULL;
1386
1387 /* Add the CR0 from the NPX frame */
1388 Cr0 |= NPX_STATE_NOT_LOADED;
1389 Cr0 |= FxSaveArea->Cr0NpxState;
1390 __writecr0(Cr0);
1391
1392 /* Restore interrupt state */
1393 __writeeflags(EFlags);
1394 }
1395
1396 /* PUBLIC FUNCTIONS **********************************************************/
1397
1398 /*
1399 * @implemented
1400 */
1401 VOID
1402 NTAPI
1403 KiCoprocessorError(VOID)
1404 {
1405 PFX_SAVE_AREA NpxArea;
1406
1407 /* Get the FPU area */
1408 NpxArea = KiGetThreadNpxArea(KeGetCurrentThread());
1409
1410 /* Set CR0_TS */
1411 NpxArea->Cr0NpxState = CR0_TS;
1412 __writecr0(__readcr0() | CR0_TS);
1413 }
1414
1415 /*
1416 * @implemented
1417 */
1418 NTSTATUS
1419 NTAPI
1420 KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
1421 {
1422 PFNSAVE_FORMAT FpState;
1423 ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
1424 DPRINT1("%s is not really implemented\n", __FUNCTION__);
1425
1426 /* check if we are doing software emulation */
1427 if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
1428
1429 FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
1430 if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
1431
1432 *((PVOID *) Save) = FpState;
1433 #ifdef __GNUC__
1434 asm volatile("fnsave %0\n\t" : "=m" (*FpState));
1435 #else
1436 __asm
1437 {
1438 fnsave [FpState]
1439 };
1440 #endif
1441
1442 KeGetCurrentThread()->DispatcherHeader.NpxIrql = KeGetCurrentIrql();
1443 return STATUS_SUCCESS;
1444 }
1445
1446 /*
1447 * @implemented
1448 */
1449 NTSTATUS
1450 NTAPI
1451 KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
1452 {
1453 PFNSAVE_FORMAT FpState = *((PVOID *) Save);
1454 ASSERT(KeGetCurrentThread()->DispatcherHeader.NpxIrql == KeGetCurrentIrql());
1455 DPRINT1("%s is not really implemented\n", __FUNCTION__);
1456
1457 #ifdef __GNUC__
1458 asm volatile("fnclex\n\t");
1459 asm volatile("frstor %0\n\t" : "=m" (*FpState));
1460 #else
1461 __asm
1462 {
1463 fnclex
1464 frstor [FpState]
1465 };
1466 #endif
1467
1468 ExFreePool(FpState);
1469 return STATUS_SUCCESS;
1470 }
1471
1472 /*
1473 * @implemented
1474 */
1475 ULONG
1476 NTAPI
1477 KeGetRecommendedSharedDataAlignment(VOID)
1478 {
1479 /* Return the global variable */
1480 return KeLargestCacheLine;
1481 }
1482
1483 VOID
1484 NTAPI
1485 KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
1486 IN PVOID Ignored1,
1487 IN PVOID Ignored2,
1488 IN PVOID Ignored3)
1489 {
1490 /* Signal this packet as done */
1491 KiIpiSignalPacketDone(PacketContext);
1492
1493 /* Flush the TB for the Current CPU */
1494 KeFlushCurrentTb();
1495 }
1496
1497 /*
1498 * @implemented
1499 */
1500 VOID
1501 NTAPI
1502 KeFlushEntireTb(IN BOOLEAN Invalid,
1503 IN BOOLEAN AllProcessors)
1504 {
1505 KIRQL OldIrql;
1506 #ifdef CONFIG_SMP
1507 KAFFINITY TargetAffinity;
1508 PKPRCB Prcb = KeGetCurrentPrcb();
1509 #endif
1510
1511 /* Raise the IRQL for the TB Flush */
1512 OldIrql = KeRaiseIrqlToSynchLevel();
1513
1514 #ifdef CONFIG_SMP
1515 /* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
1516
1517 /* Get the current processor affinity, and exclude ourselves */
1518 TargetAffinity = KeActiveProcessors;
1519 TargetAffinity &= ~Prcb->SetMember;
1520
1521 /* Make sure this is MP */
1522 if (TargetAffinity)
1523 {
1524 /* Send an IPI TB flush to the other processors */
1525 KiIpiSendPacket(TargetAffinity,
1526 KiFlushTargetEntireTb,
1527 NULL,
1528 0,
1529 NULL);
1530 }
1531 #endif
1532
1533 /* Flush the TB for the Current CPU, and update the flush stamp */
1534 KeFlushCurrentTb();
1535
1536 #ifdef CONFIG_SMP
1537 /* If this is MP, wait for the other processors to finish */
1538 if (TargetAffinity)
1539 {
1540 /* Sanity check */
1541 ASSERT(Prcb == (volatile PKPRCB)KeGetCurrentPrcb());
1542
1543 /* FIXME: TODO */
1544 ASSERTMSG("Not yet implemented\n", FALSE);
1545 }
1546 #endif
1547
1548 /* Update the flush stamp and return to original IRQL */
1549 InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
1550 KeLowerIrql(OldIrql);
1551 }
1552
1553 /*
1554 * @implemented
1555 */
1556 VOID
1557 NTAPI
1558 KeSetDmaIoCoherency(IN ULONG Coherency)
1559 {
1560 /* Save the coherency globally */
1561 KiDmaIoCoherency = Coherency;
1562 }
1563
1564 /*
1565 * @implemented
1566 */
1567 KAFFINITY
1568 NTAPI
1569 KeQueryActiveProcessors(VOID)
1570 {
1571 PAGED_CODE();
1572
1573 /* Simply return the number of active processors */
1574 return KeActiveProcessors;
1575 }
1576
1577 /*
1578 * @implemented
1579 */
1580 VOID
1581 __cdecl
1582 KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
1583 {
1584 /* Capture the context */
1585 RtlCaptureContext(&State->ContextFrame);
1586
1587 /* Capture the control state */
1588 KiSaveProcessorControlState(State);
1589 }