- Implement RtlPrefectMemoryNonTemporal. Patch by Patrick Baggett <baggett.patrick...
[reactos.git] / reactos / ntoskrnl / ke / i386 / kernel.c
1 /* $Id$
2 *
3 * COPYRIGHT: See COPYING in the top level directory
4 * PROJECT: ReactOS kernel
5 * FILE: ntoskrnl/ke/i386/kernel.c
6 * PURPOSE: Initializes the kernel
7 *
8 * PROGRAMMERS: David Welch (welch@mcmail.com)
9 */
10
11 /* INCLUDES *****************************************************************/
12
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <internal/debug.h>
16
17 /* GLOBALS *******************************************************************/
18
19 ULONG KiPcrInitDone = 0;
20 static ULONG PcrsAllocated = 0;
21 static ULONG Ke386CpuidFlags2, Ke386CpuidExFlags, Ke386CpuidExMisc;
22 ULONG Ke386CacheAlignment;
23 CHAR Ke386CpuidModel[49] = {0,};
24 ULONG Ke386L1CacheSize;
25 ULONG Ke386CacheGranularity = 0x40; /* FIXME: Default to 64 bytes for RtlPrefetchMemoryNonTemporal(), need real size */
26 BOOLEAN Ke386NoExecute = FALSE;
27 BOOLEAN Ke386Pae = FALSE;
28 BOOLEAN Ke386GlobalPagesEnabled = FALSE;
29 ULONG KiFastSystemCallDisable = 1;
30 extern PVOID Ki386InitialStackArray[MAXIMUM_PROCESSORS];
31 extern ULONG IdleProcessorMask;
32
33 static VOID INIT_FUNCTION Ki386GetCpuId(VOID);
34
35 #if defined (ALLOC_PRAGMA)
36 #pragma alloc_text(INIT, Ki386GetCpuId)
37 #pragma alloc_text(INIT, KeCreateApplicationProcessorIdleThread)
38 #pragma alloc_text(INIT, KePrepareForApplicationProcessorInit)
39 #pragma alloc_text(INIT, KeInit1)
40 #pragma alloc_text(INIT, KeInit2)
41 #pragma alloc_text(INIT, Ki386SetProcessorFeatures)
42 #endif
43
44 /* FUNCTIONS *****************************************************************/
45
46 static VOID INIT_FUNCTION
47 Ki386GetCpuId(VOID)
48 {
49 ULONG OrigFlags, Flags, FinalFlags;
50 ULONG MaxCpuidLevel;
51 ULONG Dummy, Eax, Ecx, Edx;
52 PKIPCR Pcr = (PKIPCR)KeGetCurrentKPCR();
53
54 Ke386CpuidFlags2 = Ke386CpuidExFlags = 0;
55 Ke386CacheAlignment = 32;
56
57 /* Try to toggle the id bit in eflags. */
58 Ke386SaveFlags(OrigFlags);
59 Flags = OrigFlags ^ X86_EFLAGS_ID;
60 Ke386RestoreFlags(Flags);
61 Ke386SaveFlags(FinalFlags);
62
63 Pcr->PrcbData.LogicalProcessorsPerPhysicalProcessor = 1;
64 Pcr->PrcbData.InitialApicId = 0xff;
65
66 if ((OrigFlags & X86_EFLAGS_ID) == (FinalFlags & X86_EFLAGS_ID))
67 {
68 /* No cpuid supported. */
69 Pcr->PrcbData.CpuID = FALSE;
70 Pcr->PrcbData.CpuType = 3;
71 return;
72 }
73 Pcr->PrcbData.CpuID = TRUE;
74
75 /* Get the vendor name and the maximum cpuid level supported. */
76 Ki386Cpuid(0, &MaxCpuidLevel, (PULONG)&Pcr->PrcbData.VendorString[0], (PULONG)&Pcr->PrcbData.VendorString[8], (PULONG)&Pcr->PrcbData.VendorString[4]);
77 if (MaxCpuidLevel > 0)
78 {
79 /* Get the feature flags. */
80 Ki386Cpuid(1, &Eax, &Ke386CpuidExMisc, &Ke386CpuidFlags2, &Pcr->PrcbData.FeatureBits);
81
82 DPRINT ("Model: %x\n", (Eax & 0xf00) == 0xf00 ? ((Eax >> 4) & 0xf) | ((Eax >> 12) & 0xf0) : (Eax >> 4) & 0xf);
83 DPRINT ("Family: %x\n", (Eax & 0xf00) == 0xf00 ? ((Eax >> 8) & 0xf) + ((Eax >> 20) & 0xff) : (Eax >> 8) & 0xf);
84
85 /* Get the cache alignment, if it is available */
86 if (Pcr->PrcbData.FeatureBits & (1<<19))
87 {
88 Ke386CacheAlignment = ((Ke386CpuidExMisc >> 8) & 0xff) * 8;
89 }
90 Pcr->PrcbData.CpuType = (Eax >> 8) & 0xf;
91 Pcr->PrcbData.CpuStep = (Eax & 0xf) | ((Eax << 4) & 0xf00);
92
93 Pcr->PrcbData.InitialApicId = (Ke386CpuidExMisc >> 24) & 0xff;
94
95 /* detect Hyper-Threading on Pentium 4 CPUs or later */
96 if ((Pcr->PrcbData.CpuType == 0xf || (Eax & 0x0f00000)) &&
97 !strncmp(Pcr->PrcbData.VendorString, "GenuineIntel", 12) &&
98 Pcr->PrcbData.FeatureBits & X86_FEATURE_HT)
99 {
100 Pcr->PrcbData.LogicalProcessorsPerPhysicalProcessor = (Ke386CpuidExMisc >> 16) & 0xff;
101 }
102 }
103 else
104 {
105 Pcr->PrcbData.CpuType = 4;
106 }
107
108 /* Get the maximum extended cpuid level supported. */
109 Ki386Cpuid(0x80000000, &MaxCpuidLevel, &Dummy, &Dummy, &Dummy);
110 if (MaxCpuidLevel > 0)
111 {
112 /* Get the extended feature flags. */
113 Ki386Cpuid(0x80000001, &Dummy, &Dummy, &Dummy, &Ke386CpuidExFlags);
114 }
115
116 /* Get the model name. */
117 if (MaxCpuidLevel >= 0x80000004)
118 {
119 PULONG v = (PULONG)Ke386CpuidModel;
120 Ki386Cpuid(0x80000002, v, v + 1, v + 2, v + 3);
121 Ki386Cpuid(0x80000003, v + 4, v + 5, v + 6, v + 7);
122 Ki386Cpuid(0x80000004, v + 8, v + 9, v + 10, v + 11);
123 }
124
125 /* Get the L1 cache size */
126 if (MaxCpuidLevel >= 0x80000005)
127 {
128 Ki386Cpuid(0x80000005, &Dummy, &Dummy, &Ecx, &Edx);
129 Ke386L1CacheSize = (Ecx >> 24)+(Edx >> 24);
130 if ((Ecx & 0xff) > 0)
131 {
132 Ke386CacheAlignment = Ecx & 0xff;
133 }
134 }
135
136 /* Get the L2 cache size */
137 if (MaxCpuidLevel >= 0x80000006)
138 {
139 Ki386Cpuid(0x80000006, &Dummy, &Dummy, &Ecx, &Dummy);
140 Pcr->L2CacheSize = Ecx >> 16;
141 }
142 }
143
144 VOID
145 KeApplicationProcessorInitDispatcher(VOID)
146 {
147 KIRQL oldIrql;
148 oldIrql = KeAcquireDispatcherDatabaseLock();
149 IdleProcessorMask |= (1 << KeGetCurrentProcessorNumber());
150 KeReleaseDispatcherDatabaseLock(oldIrql);
151 }
152
153 VOID
154 INIT_FUNCTION
155 KeCreateApplicationProcessorIdleThread(ULONG Id)
156 {
157 PETHREAD IdleThread;
158 PKPRCB Prcb = ((PKPCR)((ULONG_PTR)KPCR_BASE + Id * PAGE_SIZE))->Prcb;
159
160 PsInitializeIdleOrFirstThread(PsIdleProcess,
161 &IdleThread,
162 NULL,
163 KernelMode,
164 FALSE);
165 IdleThread->Tcb.State = Running;
166 IdleThread->Tcb.FreezeCount = 0;
167 IdleThread->Tcb.Affinity = 1 << Id;
168 IdleThread->Tcb.UserAffinity = 1 << Id;
169 IdleThread->Tcb.Priority = LOW_PRIORITY;
170 IdleThread->Tcb.BasePriority = LOW_PRIORITY;
171 Prcb->IdleThread = &IdleThread->Tcb;
172 Prcb->CurrentThread = &IdleThread->Tcb;
173
174 Ki386InitialStackArray[Id] = (PVOID)IdleThread->Tcb.StackLimit;
175
176 DPRINT("IdleThread for Processor %d has PID %d\n",
177 Id, IdleThread->Cid.UniqueThread);
178 }
179
180 VOID
181 INIT_FUNCTION
182 NTAPI
183 KePrepareForApplicationProcessorInit(ULONG Id)
184 {
185 PFN_TYPE PrcPfn;
186 PKIPCR Pcr;
187 PKIPCR BootPcr;
188
189 DPRINT("KePrepareForApplicationProcessorInit(Id %d)\n", Id);
190
191 BootPcr = (PKIPCR)KPCR_BASE;
192 Pcr = (PKIPCR)((ULONG_PTR)KPCR_BASE + Id * PAGE_SIZE);
193
194 MmRequestPageMemoryConsumer(MC_NPPOOL, TRUE, &PrcPfn);
195 MmCreateVirtualMappingForKernel((PVOID)Pcr,
196 PAGE_READWRITE,
197 &PrcPfn,
198 1);
199 /*
200 * Create a PCR for this processor
201 */
202 memset(Pcr, 0, PAGE_SIZE);
203 Pcr->Number = Id;
204 Pcr->SetMember = 1 << Id;
205 Pcr->NtTib.Self = &Pcr->NtTib;
206 Pcr->Self = (PKPCR)Pcr;
207 Pcr->Prcb = &Pcr->PrcbData;
208 Pcr->Irql = SYNCH_LEVEL;
209
210 Pcr->PrcbData.SetMember = 1 << Id;
211 Pcr->PrcbData.MHz = BootPcr->PrcbData.MHz;
212 Pcr->StallScaleFactor = BootPcr->StallScaleFactor;
213
214 /* Mark the end of the exception handler list */
215 Pcr->NtTib.ExceptionList = (PVOID)-1;
216
217 KiGdtPrepareForApplicationProcessorInit(Id);
218
219 KeActiveProcessors |= 1 << Id;
220 }
221
222 VOID
223 NTAPI
224 KeApplicationProcessorInit(VOID)
225 {
226 ULONG Offset;
227 PKIPCR Pcr;
228
229 DPRINT("KeApplicationProcessorInit()\n");
230
231 if (Ke386GlobalPagesEnabled)
232 {
233 /* Enable global pages */
234 Ke386SetCr4(Ke386GetCr4() | X86_CR4_PGE);
235 }
236
237
238 Offset = InterlockedIncrementUL(&PcrsAllocated) - 1;
239 Pcr = (PKIPCR)((ULONG_PTR)KPCR_BASE + Offset * PAGE_SIZE);
240
241 /*
242 * Initialize the GDT
243 */
244 KiInitializeGdt((PKPCR)Pcr);
245
246 /* Get processor information. */
247 Ki386GetCpuId();
248
249 /* Check FPU/MMX/SSE support. */
250 KiCheckFPU();
251
252 KeInitDpc(Pcr->Prcb);
253
254 if (Pcr->PrcbData.FeatureBits & X86_FEATURE_SYSCALL)
255 {
256 extern void KiFastCallEntry(void);
257
258 /* CS Selector of the target segment. */
259 Ke386Wrmsr(0x174, KGDT_R0_CODE, 0);
260 /* Target ESP. */
261 Ke386Wrmsr(0x175, 0, 0);
262 /* Target EIP. */
263 Ke386Wrmsr(0x176, (ULONG_PTR)KiFastCallEntry, 0);
264 }
265
266 /*
267 * It is now safe to process interrupts
268 */
269 KeLowerIrql(DISPATCH_LEVEL);
270
271 /*
272 * Initialize the TSS
273 */
274 Ki386ApplicationProcessorInitializeTSS();
275
276 /*
277 * Initialize a default LDT
278 */
279 Ki386InitializeLdt();
280
281 /* Now we can enable interrupts. */
282 Ke386EnableInterrupts();
283 }
284
285 VOID
286 INIT_FUNCTION
287 NTAPI
288 KeInit1(PCHAR CommandLine, PULONG LastKernelAddress)
289 {
290 PKIPCR KPCR;
291 BOOLEAN Pae = FALSE;
292 BOOLEAN NoExecute = FALSE;
293 PCHAR p1, p2;
294 extern USHORT KiBootGdt[];
295 extern KTSS KiBootTss;
296
297 /*
298 * Initialize the initial PCR region. We can't allocate a page
299 * with MmAllocPage() here because MmInit1() has not yet been
300 * called, so we use a predefined page in low memory
301 */
302
303 KPCR = (PKIPCR)KPCR_BASE;
304 memset(KPCR, 0, PAGE_SIZE);
305 KPCR->Self = (PKPCR)KPCR;
306 KPCR->Prcb = &KPCR->PrcbData;
307 KPCR->Irql = SYNCH_LEVEL;
308 KPCR->NtTib.Self = &KPCR->NtTib;
309 KPCR->GDT = KiBootGdt;
310 KPCR->IDT = (PUSHORT)KiIdt;
311 KPCR->TSS = &KiBootTss;
312 KPCR->Number = 0;
313 KPCR->SetMember = 1 << 0;
314 KeActiveProcessors = 1 << 0;
315 KPCR->PrcbData.SetMember = 1 << 0;
316 KiPcrInitDone = 1;
317 PcrsAllocated++;
318
319 KiInitializeGdt (NULL);
320 Ki386BootInitializeTSS();
321 Ki386InitializeLdt();
322
323 /* Get processor information. */
324 Ki386GetCpuId();
325
326 /* Check FPU/MMX/SSE support. */
327 KiCheckFPU();
328
329 /* Mark the end of the exception handler list */
330 KPCR->NtTib.ExceptionList = (PVOID)-1;
331
332 KeInitDpc(KPCR->Prcb);
333
334 KeInitExceptions ();
335 KeInitInterrupts ();
336
337 KeActiveProcessors |= 1 << 0;
338
339
340 if (KPCR->PrcbData.FeatureBits & X86_FEATURE_PGE)
341 {
342 ULONG Flags;
343 /* Enable global pages */
344 Ke386GlobalPagesEnabled = TRUE;
345 Ke386SaveFlags(Flags);
346 Ke386DisableInterrupts();
347 Ke386SetCr4(Ke386GetCr4() | X86_CR4_PGE);
348 Ke386RestoreFlags(Flags);
349 }
350
351 /* Search for pae and noexecute */
352 p1 = (PCHAR)KeLoaderBlock.CommandLine;
353 while(*p1 && (p2 = strchr(p1, '/')))
354 {
355 p2++;
356 if (!_strnicmp(p2, "PAE", 3))
357 {
358 if (p2[3] == ' ' || p2[3] == 0)
359 {
360 p2 += 3;
361 Pae = TRUE;
362 }
363 }
364 else if (!_strnicmp(p2, "NOEXECUTE", 9))
365 {
366 if (p2[9] == ' ' || p2[9] == '=' || p2[9] == 0)
367 {
368 p2 += 9;
369 NoExecute = TRUE;
370 }
371 }
372 p1 = p2;
373 }
374 #if 0
375 /*
376 * FIXME:
377 * Make the detection of the noexecute feature more portable.
378 */
379 if(KPCR->PrcbData.CpuType == 0xf &&
380 RtlCompareMemory("AuthenticAMD", KPCR->PrcbData.VendorString, 12) == 12)
381 {
382 if (NoExecute)
383 {
384 ULONG Flags, l, h;
385 Ke386SaveFlags(Flags);
386 Ke386DisableInterrupts();
387
388 Ke386Rdmsr(0xc0000080, l, h);
389 l |= (1 << 11);
390 Ke386Wrmsr(0xc0000080, l, h);
391 Ke386NoExecute = TRUE;
392 Ke386RestoreFlags(Flags);
393 }
394 }
395 else
396 {
397 NoExecute=FALSE;
398 }
399 #endif
400
401 Ke386Pae = Ke386GetCr4() & X86_CR4_PAE ? TRUE : FALSE;
402 #if 0
403 /* Enable PAE mode */
404 if ((Pae && (KPCR->PrcbData.FeatureBits & X86_FEATURE_PAE)) || NoExecute)
405 {
406 MiEnablePAE((PVOID*)LastKernelAddress);
407 Ke386PaeEnabled = TRUE;
408 }
409 #endif
410 if (KPCR->PrcbData.FeatureBits & X86_FEATURE_SYSCALL)
411 {
412 extern void KiFastCallEntry(void);
413
414 /* CS Selector of the target segment. */
415 Ke386Wrmsr(0x174, KGDT_R0_CODE, 0);
416 /* Target ESP. */
417 Ke386Wrmsr(0x175, 0, 0);
418 /* Target EIP. */
419 Ke386Wrmsr(0x176, (ULONG_PTR)KiFastCallEntry, 0);
420 }
421 }
422
423 VOID
424 INIT_FUNCTION
425 NTAPI
426 KeInit2(VOID)
427 {
428 PKIPCR Pcr = (PKIPCR)KeGetCurrentKPCR();
429
430 KiInitializeBugCheck();
431 KeInitializeDispatcher();
432 KiInitializeSystemClock();
433
434 if (Pcr->PrcbData.FeatureBits & X86_FEATURE_PAE)
435 {
436 DPRINT("CPU supports PAE mode\n");
437 if (Ke386Pae)
438 {
439 DPRINT("CPU runs in PAE mode\n");
440 if (Ke386NoExecute)
441 {
442 DPRINT("NoExecute is enabled\n");
443 }
444 }
445 else
446 {
447 DPRINT("CPU doesn't run in PAE mode\n");
448 }
449 }
450 if ((Pcr->PrcbData.FeatureBits & (X86_FEATURE_FXSR | X86_FEATURE_MMX | X86_FEATURE_SSE | X86_FEATURE_SSE2)) ||
451 (Ke386CpuidFlags2 & X86_EXT_FEATURE_SSE3))
452 {
453 DPRINT("CPU supports" "%s%s%s%s%s" ".\n",
454 ((Pcr->PrcbData.FeatureBits & X86_FEATURE_FXSR) ? " FXSR" : ""),
455 ((Pcr->PrcbData.FeatureBits & X86_FEATURE_MMX) ? " MMX" : ""),
456 ((Pcr->PrcbData.FeatureBits & X86_FEATURE_SSE) ? " SSE" : ""),
457 ((Pcr->PrcbData.FeatureBits & X86_FEATURE_SSE2) ? " SSE2" : ""),
458 ((Ke386CpuidFlags2 & X86_EXT_FEATURE_SSE3) ? " SSE3" : ""));
459 }
460 if (Ke386GetCr4() & X86_CR4_OSFXSR)
461 {
462 DPRINT("SSE enabled.\n");
463 }
464 if (Ke386GetCr4() & X86_CR4_OSXMMEXCPT)
465 {
466 DPRINT("Unmasked SIMD exceptions enabled.\n");
467 }
468 if (Pcr->PrcbData.VendorString[0])
469 {
470 DPRINT("CPU Vendor: %s\n", Pcr->PrcbData.VendorString);
471 }
472 if (Ke386CpuidModel[0])
473 {
474 DPRINT("CPU Model: %s\n", Ke386CpuidModel);
475 }
476
477 DPRINT("Ke386CacheAlignment: %d\n", Ke386CacheAlignment);
478 if (Ke386L1CacheSize)
479 {
480
481 DPRINT("Ke386L1CacheSize: %dkB\n", Ke386L1CacheSize);
482 }
483 if (Pcr->L2CacheSize)
484 {
485 DPRINT("Ke386L2CacheSize: %dkB\n", Pcr->L2CacheSize);
486 }
487 }
488
489 VOID INIT_FUNCTION
490 Ki386SetProcessorFeatures(VOID)
491 {
492 PKIPCR Pcr = (PKIPCR)KeGetCurrentKPCR();
493 OBJECT_ATTRIBUTES ObjectAttributes;
494 UNICODE_STRING KeyName =
495 RTL_CONSTANT_STRING(L"\\Registry\\Machine\\System\\CurrentControlSet\\Control\\Session Manager\\Kernel");
496 UNICODE_STRING ValueName = RTL_CONSTANT_STRING(L"FastSystemCallDisable");
497 HANDLE KeyHandle;
498 ULONG ResultLength;
499 KEY_VALUE_PARTIAL_INFORMATION ValueData;
500 NTSTATUS Status;
501 ULONG FastSystemCallDisable = 0;
502
503 SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_PRECISION_ERRATA] = FALSE;
504 SharedUserData->ProcessorFeatures[PF_FLOATING_POINT_EMULATED] = FALSE;
505 SharedUserData->ProcessorFeatures[PF_COMPARE_EXCHANGE_DOUBLE] =
506 (Pcr->PrcbData.FeatureBits & X86_FEATURE_CX8);
507 SharedUserData->ProcessorFeatures[PF_MMX_INSTRUCTIONS_AVAILABLE] =
508 (Pcr->PrcbData.FeatureBits & X86_FEATURE_MMX);
509 SharedUserData->ProcessorFeatures[PF_PPC_MOVEMEM_64BIT_OK] = FALSE;
510 SharedUserData->ProcessorFeatures[PF_ALPHA_BYTE_INSTRUCTIONS] = FALSE;
511 SharedUserData->ProcessorFeatures[PF_XMMI_INSTRUCTIONS_AVAILABLE] =
512 (Pcr->PrcbData.FeatureBits & X86_FEATURE_SSE);
513 SharedUserData->ProcessorFeatures[PF_3DNOW_INSTRUCTIONS_AVAILABLE] =
514 (Ke386CpuidExFlags & X86_EXT_FEATURE_3DNOW);
515 SharedUserData->ProcessorFeatures[PF_RDTSC_INSTRUCTION_AVAILABLE] =
516 (Pcr->PrcbData.FeatureBits & X86_FEATURE_TSC);
517 SharedUserData->ProcessorFeatures[PF_PAE_ENABLED] = Ke386Pae;
518 SharedUserData->ProcessorFeatures[PF_XMMI64_INSTRUCTIONS_AVAILABLE] =
519 (Pcr->PrcbData.FeatureBits & X86_FEATURE_SSE2);
520
521 /* Does the CPU Support 'prefetchnta' (SSE) */
522 if(Pcr->PrcbData.FeatureBits & X86_FEATURE_SSE)
523 {
524 /* Replace the ret by a nop */
525 *(PCHAR)RtlPrefetchMemoryNonTemporal = 0x90;
526 }
527
528 /* Does the CPU Support Fast System Call? */
529 if (Pcr->PrcbData.FeatureBits & X86_FEATURE_SYSCALL) {
530
531 /* FIXME: Check for Family == 6, Model < 3 and Stepping < 3 and disable */
532
533 /* Make sure it's not disabled in registry */
534 InitializeObjectAttributes(&ObjectAttributes,
535 &KeyName,
536 OBJ_CASE_INSENSITIVE,
537 NULL,
538 NULL);
539 Status = NtOpenKey(&KeyHandle, KEY_ALL_ACCESS, &ObjectAttributes);
540
541 if (NT_SUCCESS(Status)) {
542
543 /* Read the Value then Close the Key */
544 Status = NtQueryValueKey(KeyHandle,
545 &ValueName,
546 KeyValuePartialInformation,
547 &ValueData,
548 sizeof(ValueData),
549 &ResultLength);
550 RtlMoveMemory(&FastSystemCallDisable, ValueData.Data, sizeof(ULONG));
551
552 NtClose(KeyHandle);
553 }
554
555 } else {
556
557 /* Disable SYSENTER/SYSEXIT, because the CPU doesn't support it */
558 FastSystemCallDisable = 1;
559
560 }
561
562 if (FastSystemCallDisable) {
563 /* Use INT2E */
564 const unsigned char Entry[7] = {0x8D, 0x54, 0x24, 0x08, /* lea 0x8(%esp),%edx */
565 0xCD, 0x2E, /* int 0x2e */
566 0xC3}; /* ret */
567 memcpy(&SharedUserData->SystemCall, Entry, sizeof(Entry));
568 } else {
569 /* Use SYSENTER */
570 const unsigned char Entry[5] = {0x8B, 0xD4, /* movl %esp,%edx */
571 0x0F, 0x34, /* sysenter */
572 0xC3}; /* ret */
573 memcpy(&SharedUserData->SystemCall, Entry, sizeof(Entry));
574 /* Enable SYSENTER/SYSEXIT */
575 KiFastSystemCallDisable = 0;
576 }
577 }