[NTOS]: This is why you shouldn't let Antoine Dodson commit code.
[reactos.git] / reactos / ntoskrnl / mm / ARM3 / pagfault.c
1 /*
2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/pagfault.c
5 * PURPOSE: ARM Memory Manager Page Fault Handling
6 * PROGRAMMERS: ReactOS Portable Systems Group
7 */
8
9 /* INCLUDES *******************************************************************/
10
11 #include <ntoskrnl.h>
12 #define NDEBUG
13 #include <debug.h>
14
15 #line 15 "ARMĀ³::PAGFAULT"
16 #define MODULE_INVOLVED_IN_ARM3
17 #include "../ARM3/miarm.h"
18
19 /* GLOBALS ********************************************************************/
20
21 /* PRIVATE FUNCTIONS **********************************************************/
22
23 PMMPTE
24 NTAPI
25 MiCheckVirtualAddress(IN PVOID VirtualAddress,
26 OUT PULONG ProtectCode,
27 OUT PMMVAD *ProtoVad)
28 {
29 PMMVAD Vad;
30
31 /* No prototype/section support for now */
32 *ProtoVad = NULL;
33
34 /* Only valid for user VADs for now */
35 ASSERT(VirtualAddress <= MM_HIGHEST_USER_ADDRESS);
36
37 /* Special case for shared data */
38 if (PAGE_ALIGN(VirtualAddress) == (PVOID)USER_SHARED_DATA)
39 {
40 /* It's a read-only page */
41 *ProtectCode = MM_READONLY;
42 return MmSharedUserDataPte;
43 }
44
45 /* Find the VAD, it might not exist if the address is bogus */
46 Vad = MiLocateAddress(VirtualAddress);
47 if (!Vad)
48 {
49 /* Bogus virtual address */
50 *ProtectCode = MM_NOACCESS;
51 return NULL;
52 }
53
54 /* This must be a TEB/PEB VAD */
55 ASSERT(Vad->u.VadFlags.PrivateMemory == TRUE);
56 ASSERT(Vad->u.VadFlags.MemCommit == TRUE);
57 ASSERT(Vad->u.VadFlags.VadType == VadNone);
58
59 /* Return the protection on it */
60 *ProtectCode = Vad->u.VadFlags.Protection;
61 return NULL;
62 }
63
64 NTSTATUS
65 FASTCALL
66 MiCheckPdeForPagedPool(IN PVOID Address)
67 {
68 PMMPDE PointerPde;
69 NTSTATUS Status = STATUS_SUCCESS;
70
71 /* No session support in ReactOS yet */
72 ASSERT(MI_IS_SESSION_ADDRESS(Address) == FALSE);
73 ASSERT(MI_IS_SESSION_PTE(Address) == FALSE);
74
75 //
76 // Check if this is a fault while trying to access the page table itself
77 //
78 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
79 {
80 //
81 // Send a hint to the page fault handler that this is only a valid fault
82 // if we already detected this was access within the page table range
83 //
84 PointerPde = (PMMPDE)MiAddressToPte(Address);
85 Status = STATUS_WAIT_1;
86 }
87 else if (Address < MmSystemRangeStart)
88 {
89 //
90 // This is totally illegal
91 //
92 return STATUS_ACCESS_VIOLATION;
93 }
94 else
95 {
96 //
97 // Get the PDE for the address
98 //
99 PointerPde = MiAddressToPde(Address);
100 }
101
102 //
103 // Check if it's not valid
104 //
105 if (PointerPde->u.Hard.Valid == 0)
106 {
107 #ifdef _M_AMD64
108 ASSERT(FALSE);
109 #else
110 //
111 // Copy it from our double-mapped system page directory
112 //
113 InterlockedExchangePte(PointerPde,
114 MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)].u.Long);
115 #endif
116 }
117
118 //
119 // Return status
120 //
121 return Status;
122 }
123
124 VOID
125 NTAPI
126 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
127 {
128 PMMPTE ZeroPte;
129 MMPTE TempPte;
130 PMMPFN Pfn1;
131 PVOID ZeroAddress;
132
133 /* Get the PFN for this page */
134 Pfn1 = MiGetPfnEntry(PageFrameNumber);
135 ASSERT(Pfn1);
136
137 /* Grab a system PTE we can use to zero the page */
138 ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
139 ASSERT(ZeroPte);
140
141 /* Initialize the PTE for it */
142 TempPte = ValidKernelPte;
143 TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
144
145 /* Setup caching */
146 if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
147 {
148 /* Write combining, no caching */
149 MI_PAGE_DISABLE_CACHE(&TempPte);
150 MI_PAGE_WRITE_COMBINED(&TempPte);
151 }
152 else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
153 {
154 /* Write through, no caching */
155 MI_PAGE_DISABLE_CACHE(&TempPte);
156 MI_PAGE_WRITE_THROUGH(&TempPte);
157 }
158
159 /* Make the system PTE valid with our PFN */
160 MI_WRITE_VALID_PTE(ZeroPte, TempPte);
161
162 /* Get the address it maps to, and zero it out */
163 ZeroAddress = MiPteToAddress(ZeroPte);
164 KeZeroPages(ZeroAddress, PAGE_SIZE);
165
166 /* Now get rid of it */
167 MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
168 }
169
170 NTSTATUS
171 NTAPI
172 MiResolveDemandZeroFault(IN PVOID Address,
173 IN PMMPTE PointerPte,
174 IN PEPROCESS Process,
175 IN KIRQL OldIrql)
176 {
177 PFN_NUMBER PageFrameNumber = 0;
178 MMPTE TempPte;
179 BOOLEAN NeedZero = FALSE;
180 ULONG Color;
181 DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
182 Address,
183 Process);
184
185 /* Must currently only be called by paging path */
186 ASSERT(OldIrql == MM_NOIRQL);
187 if (Process)
188 {
189 /* Sanity check */
190 ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
191
192 /* No forking yet */
193 ASSERT(Process->ForkInProgress == NULL);
194
195 /* Get process color */
196 Color = MI_GET_NEXT_PROCESS_COLOR(Process);
197
198 /* We'll need a zero page */
199 NeedZero = TRUE;
200 }
201 else
202 {
203 /* Get the next system page color */
204 Color = MI_GET_NEXT_COLOR();
205 }
206
207 //
208 // Lock the PFN database
209 //
210 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
211 ASSERT(PointerPte->u.Hard.Valid == 0);
212
213 /* Do we need a zero page? */
214 if (NeedZero)
215 {
216 /* Try to get one, if we couldn't grab a free page and zero it */
217 PageFrameNumber = MiRemoveZeroPageSafe(Color);
218 if (PageFrameNumber) NeedZero = FALSE;
219 }
220
221 /* Did we get a page? */
222 if (!PageFrameNumber)
223 {
224 /* We either failed to find a zero page, or this is a system request */
225 PageFrameNumber = MiRemoveAnyPage(Color);
226 DPRINT("New pool page: %lx\n", PageFrameNumber);
227 }
228
229 /* Initialize it */
230 MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
231
232 //
233 // Release PFN lock
234 //
235 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
236
237 //
238 // Increment demand zero faults
239 //
240 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
241
242 /* Zero the page if need be */
243 if (NeedZero) MiZeroPfn(PageFrameNumber);
244
245 /* Build the PTE */
246 if (PointerPte <= MiHighestUserPte)
247 {
248 /* For user mode */
249 MI_MAKE_HARDWARE_PTE_USER(&TempPte,
250 PointerPte,
251 PointerPte->u.Soft.Protection,
252 PageFrameNumber);
253 }
254 else
255 {
256 /* For kernel mode */
257 MI_MAKE_HARDWARE_PTE(&TempPte,
258 PointerPte,
259 PointerPte->u.Soft.Protection,
260 PageFrameNumber);
261 }
262
263 /* Set it dirty if it's a writable page */
264 if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
265
266 /* Write it */
267 MI_WRITE_VALID_PTE(PointerPte, TempPte);
268
269 //
270 // It's all good now
271 //
272 DPRINT("Paged pool page has now been paged in\n");
273 return STATUS_PAGE_FAULT_DEMAND_ZERO;
274 }
275
276 NTSTATUS
277 NTAPI
278 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
279 IN PVOID Address,
280 IN PMMPTE PointerPte,
281 IN PMMPTE PointerProtoPte,
282 IN KIRQL OldIrql,
283 IN PMMPFN Pfn1)
284 {
285 MMPTE TempPte;
286 PFN_NUMBER PageFrameIndex;
287
288 /* Must be called with an valid prototype PTE, with the PFN lock held */
289 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
290 ASSERT(PointerProtoPte->u.Hard.Valid == 1);
291
292 /* Quick-n-dirty */
293 ASSERT(PointerPte->u.Soft.PageFileHigh == 0xFFFFF);
294
295 /* Get the page */
296 PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
297
298 /* Release the PFN lock */
299 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
300
301 /* Build the user PTE */
302 ASSERT(Address < MmSystemRangeStart);
303 MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, MM_READONLY, PageFrameIndex);
304
305 /* Write the PTE */
306 MI_WRITE_VALID_PTE(PointerPte, TempPte);
307
308 /* Return success */
309 return STATUS_SUCCESS;
310 }
311
312 NTSTATUS
313 NTAPI
314 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
315 IN PVOID Address,
316 IN PMMPTE PointerPte,
317 IN PMMPTE PointerProtoPte,
318 IN OUT PMMPFN *OutPfn,
319 OUT PVOID *PageFileData,
320 OUT PMMPTE PteValue,
321 IN PEPROCESS Process,
322 IN KIRQL OldIrql,
323 IN PVOID TrapInformation)
324 {
325 MMPTE TempPte;
326 PMMPFN Pfn1;
327 PFN_NUMBER PageFrameIndex;
328
329 /* Must be called with an invalid, prototype PTE, with the PFN lock held */
330 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
331 ASSERT(PointerPte->u.Hard.Valid == 0);
332 ASSERT(PointerPte->u.Soft.Prototype == 1);
333
334 /* Read the prototype PTE -- it must be valid since we only handle shared data */
335 TempPte = *PointerProtoPte;
336 ASSERT(TempPte.u.Hard.Valid == 1);
337
338 /* One more user of this mapped page */
339 PageFrameIndex = PFN_FROM_PTE(&TempPte);
340 Pfn1 = MiGetPfnEntry(PageFrameIndex);
341 Pfn1->u2.ShareCount++;
342
343 /* Call it a transition */
344 InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
345
346 /* Complete the prototype PTE fault -- this will release the PFN lock */
347 return MiCompleteProtoPteFault(StoreInstruction,
348 Address,
349 PointerPte,
350 PointerProtoPte,
351 OldIrql,
352 NULL);
353 }
354
355 NTSTATUS
356 NTAPI
357 MiDispatchFault(IN BOOLEAN StoreInstruction,
358 IN PVOID Address,
359 IN PMMPTE PointerPte,
360 IN PMMPTE PointerProtoPte,
361 IN BOOLEAN Recursive,
362 IN PEPROCESS Process,
363 IN PVOID TrapInformation,
364 IN PVOID Vad)
365 {
366 MMPTE TempPte;
367 KIRQL OldIrql, LockIrql;
368 NTSTATUS Status;
369 PMMPTE SuperProtoPte;
370 DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
371 Address,
372 Process);
373
374 //
375 // Make sure APCs are off and we're not at dispatch
376 //
377 OldIrql = KeGetCurrentIrql();
378 ASSERT(OldIrql <= APC_LEVEL);
379 ASSERT(KeAreAllApcsDisabled() == TRUE);
380
381 //
382 // Grab a copy of the PTE
383 //
384 TempPte = *PointerPte;
385
386 /* Do we have a prototype PTE? */
387 if (PointerProtoPte)
388 {
389 /* This should never happen */
390 ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
391
392 /* We currently only handle the shared user data PTE path */
393 ASSERT(Address < MmSystemRangeStart);
394 ASSERT(PointerPte->u.Soft.Prototype == 1);
395 ASSERT(PointerPte->u.Soft.PageFileHigh == 0xFFFFF);
396 ASSERT(Vad == NULL);
397
398 /* Lock the PFN database */
399 LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
400
401 /* For the shared data page, this should be true */
402 SuperProtoPte = MiAddressToPte(PointerProtoPte);
403 ASSERT(SuperProtoPte->u.Hard.Valid == 1);
404 ASSERT(TempPte.u.Hard.Valid == 0);
405
406 /* Resolve the fault -- this will release the PFN lock */
407 Status = MiResolveProtoPteFault(StoreInstruction,
408 Address,
409 PointerPte,
410 PointerProtoPte,
411 NULL,
412 NULL,
413 NULL,
414 Process,
415 LockIrql,
416 TrapInformation);
417 ASSERT(Status == STATUS_SUCCESS);
418
419 /* Complete this as a transition fault */
420 ASSERT(OldIrql == KeGetCurrentIrql());
421 ASSERT(OldIrql <= APC_LEVEL);
422 ASSERT(KeAreAllApcsDisabled() == TRUE);
423 return STATUS_PAGE_FAULT_TRANSITION;
424 }
425
426 //
427 // The PTE must be invalid, but not totally blank
428 //
429 ASSERT(TempPte.u.Hard.Valid == 0);
430 ASSERT(TempPte.u.Long != 0);
431
432 //
433 // No prototype, transition or page file software PTEs in ARM3 yet
434 //
435 ASSERT(TempPte.u.Soft.Prototype == 0);
436 ASSERT(TempPte.u.Soft.Transition == 0);
437 ASSERT(TempPte.u.Soft.PageFileHigh == 0);
438
439 //
440 // If we got this far, the PTE can only be a demand zero PTE, which is what
441 // we want. Go handle it!
442 //
443 Status = MiResolveDemandZeroFault(Address,
444 PointerPte,
445 Process,
446 MM_NOIRQL);
447 ASSERT(KeAreAllApcsDisabled () == TRUE);
448 if (NT_SUCCESS(Status))
449 {
450 //
451 // Make sure we're returning in a sane state and pass the status down
452 //
453 ASSERT(OldIrql == KeGetCurrentIrql ());
454 ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
455 return Status;
456 }
457
458 //
459 // Generate an access fault
460 //
461 return STATUS_ACCESS_VIOLATION;
462 }
463
464 NTSTATUS
465 NTAPI
466 MmArmAccessFault(IN BOOLEAN StoreInstruction,
467 IN PVOID Address,
468 IN KPROCESSOR_MODE Mode,
469 IN PVOID TrapInformation)
470 {
471 KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
472 PMMPTE PointerPte, ProtoPte;
473 PMMPDE PointerPde;
474 MMPTE TempPte;
475 PETHREAD CurrentThread;
476 PEPROCESS CurrentProcess;
477 NTSTATUS Status;
478 PMMSUPPORT WorkingSet;
479 ULONG ProtectionCode;
480 PMMVAD Vad;
481 PFN_NUMBER PageFrameIndex;
482 ULONG Color;
483 DPRINT("ARM3 FAULT AT: %p\n", Address);
484
485 //
486 // Get the PTE and PDE
487 //
488 PointerPte = MiAddressToPte(Address);
489 PointerPde = MiAddressToPde(Address);
490 #if (_MI_PAGING_LEVELS >= 3)
491 /* We need the PPE and PXE addresses */
492 ASSERT(FALSE);
493 #endif
494
495 //
496 // Check for dispatch-level snafu
497 //
498 if (OldIrql > APC_LEVEL)
499 {
500 //
501 // There are some special cases where this is okay, but not in ARM3 yet
502 //
503 DbgPrint("MM:***PAGE FAULT AT IRQL > 1 Va %p, IRQL %lx\n",
504 Address,
505 OldIrql);
506 ASSERT(OldIrql <= APC_LEVEL);
507 }
508
509 //
510 // Check for kernel fault
511 //
512 if (Address >= MmSystemRangeStart)
513 {
514 //
515 // What are you even DOING here?
516 //
517 if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
518
519 #if (_MI_PAGING_LEVELS >= 3)
520 /* Need to check PXE and PDE validity */
521 ASSERT(FALSE);
522 #endif
523
524 //
525 // Is the PDE valid?
526 //
527 if (!PointerPde->u.Hard.Valid == 0)
528 {
529 //
530 // Debug spew (eww!)
531 //
532 DPRINT("Invalid PDE\n");
533 #if (_MI_PAGING_LEVELS == 2)
534 //
535 // Handle mapping in "Special" PDE directoreis
536 //
537 MiCheckPdeForPagedPool(Address);
538 #endif
539 //
540 // Now we SHOULD be good
541 //
542 if (PointerPde->u.Hard.Valid == 0)
543 {
544 //
545 // FIXFIX: Do the S-LIST hack
546 //
547
548 //
549 // Kill the system
550 //
551 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
552 (ULONG_PTR)Address,
553 StoreInstruction,
554 (ULONG_PTR)TrapInformation,
555 2);
556 }
557 }
558
559 //
560 // The PDE is valid, so read the PTE
561 //
562 TempPte = *PointerPte;
563 if (TempPte.u.Hard.Valid == 1)
564 {
565 //
566 // Only two things can go wrong here:
567 // Executing NX page (we couldn't care less)
568 // Writing to a read-only page (the stuff ARM3 works with is write,
569 // so again, moot point).
570 //
571 if (StoreInstruction)
572 {
573 DPRINT1("Should NEVER happen on ARM3!!!\n");
574 return STATUS_ACCESS_VIOLATION;
575 }
576
577 //
578 // Otherwise, the PDE was probably invalid, and all is good now
579 //
580 return STATUS_SUCCESS;
581 }
582
583 //
584 // Check for a fault on the page table or hyperspace itself
585 //
586 if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
587 {
588 //
589 // This might happen...not sure yet
590 //
591 DPRINT1("FAULT ON PAGE TABLES: %p %lx %lx!\n", Address, *PointerPte, *PointerPde);
592 #if (_MI_PAGING_LEVELS == 2)
593 //
594 // Map in the page table
595 //
596 if (MiCheckPdeForPagedPool(Address) == STATUS_WAIT_1)
597 {
598 DPRINT1("PAGE TABLES FAULTED IN!\n");
599 return STATUS_SUCCESS;
600 }
601 #endif
602 //
603 // Otherwise the page table doesn't actually exist
604 //
605 DPRINT1("FAILING\n");
606 return STATUS_ACCESS_VIOLATION;
607 }
608
609 /* In this path, we are using the system working set */
610 CurrentThread = PsGetCurrentThread();
611 WorkingSet = &MmSystemCacheWs;
612
613 /* Acquire it */
614 KeRaiseIrql(APC_LEVEL, &LockIrql);
615 MiLockWorkingSet(CurrentThread, WorkingSet);
616
617 //
618 // Re-read PTE now that the IRQL has been raised
619 //
620 TempPte = *PointerPte;
621 if (TempPte.u.Hard.Valid == 1)
622 {
623 //
624 // Only two things can go wrong here:
625 // Executing NX page (we couldn't care less)
626 // Writing to a read-only page (the stuff ARM3 works with is write,
627 // so again, moot point.
628 //
629 if (StoreInstruction)
630 {
631 DPRINT1("Should NEVER happen on ARM3!!!\n");
632 return STATUS_ACCESS_VIOLATION;
633 }
634
635 /* Release the working set */
636 MiUnlockWorkingSet(CurrentThread, WorkingSet);
637 KeLowerIrql(LockIrql);
638
639 //
640 // Otherwise, the PDE was probably invalid, and all is good now
641 //
642 return STATUS_SUCCESS;
643 }
644
645 /* Check one kind of prototype PTE */
646 if (TempPte.u.Soft.Prototype)
647 {
648 /* The one used for protected pool... */
649 ASSERT(MmProtectFreedNonPagedPool == TRUE);
650
651 /* Make sure protected pool is on, and that this is a pool address */
652 if ((MmProtectFreedNonPagedPool) &&
653 (((Address >= MmNonPagedPoolStart) &&
654 (Address < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
655 MmSizeOfNonPagedPoolInBytes))) ||
656 ((Address >= MmNonPagedPoolExpansionStart) &&
657 (Address < MmNonPagedPoolEnd))))
658 {
659 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
660 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL,
661 (ULONG_PTR)Address,
662 StoreInstruction,
663 Mode,
664 4);
665 }
666 }
667
668 //
669 // We don't implement transition PTEs
670 //
671 ASSERT(TempPte.u.Soft.Transition == 0);
672
673 //
674 // Now do the real fault handling
675 //
676 Status = MiDispatchFault(StoreInstruction,
677 Address,
678 PointerPte,
679 NULL,
680 FALSE,
681 NULL,
682 TrapInformation,
683 NULL);
684
685 /* Release the working set */
686 ASSERT(KeAreAllApcsDisabled() == TRUE);
687 MiUnlockWorkingSet(CurrentThread, WorkingSet);
688 KeLowerIrql(LockIrql);
689
690 //
691 // We are done!
692 //
693 DPRINT("Fault resolved with status: %lx\n", Status);
694 return Status;
695 }
696
697 /* This is a user fault */
698 CurrentThread = PsGetCurrentThread();
699 CurrentProcess = PsGetCurrentProcess();
700
701 /* Lock the working set */
702 MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
703
704 #if (_MI_PAGING_LEVELS >= 3)
705 /* Need to check/handle PPE and PXE validity too */
706 ASSERT(FALSE);
707 #endif
708
709 /* First things first, is the PDE valid? */
710 ASSERT(PointerPde != MiAddressToPde(PTE_BASE));
711 ASSERT(PointerPde->u.Hard.LargePage == 0);
712 if (PointerPde->u.Hard.Valid == 0)
713 {
714 /* Right now, we only handle scenarios where the PDE is totally empty */
715 ASSERT(PointerPde->u.Long == 0);
716
717 /* Check if this address range belongs to a valid allocation (VAD) */
718 MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
719
720 /* Right now, we expect a valid protection mask on the VAD */
721 ASSERT(ProtectionCode != MM_NOACCESS);
722
723 /* Make the PDE demand-zero */
724 MI_WRITE_INVALID_PTE(PointerPde, DemandZeroPde);
725
726 /* And go dispatch the fault on the PDE. This should handle the demand-zero */
727 Status = MiDispatchFault(TRUE,
728 PointerPte,
729 PointerPde,
730 NULL,
731 FALSE,
732 PsGetCurrentProcess(),
733 TrapInformation,
734 NULL);
735
736 /* We should come back with APCs enabled, and with a valid PDE */
737 ASSERT(KeAreAllApcsDisabled() == TRUE);
738 #if (_MI_PAGING_LEVELS >= 3)
739 /* Need to check/handle PPE and PXE validity too */
740 ASSERT(FALSE);
741 #endif
742 ASSERT(PointerPde->u.Hard.Valid == 1);
743 }
744
745 /* Now capture the PTE. We only handle cases where it's totally empty */
746 TempPte = *PointerPte;
747 ASSERT(TempPte.u.Long == 0);
748
749 /* Check if this address range belongs to a valid allocation (VAD) */
750 ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
751 if (ProtectionCode == MM_NOACCESS)
752 {
753 /* This is a bogus VA */
754 Status = STATUS_ACCESS_VIOLATION;
755
756 /* Could be a not-yet-mapped paged pool page table */
757 #if (_MI_PAGING_LEVELS == 2)
758 MiCheckPdeForPagedPool(Address);
759 #endif
760 /* See if that fixed it */
761 if (PointerPte->u.Hard.Valid == 1) Status = STATUS_SUCCESS;
762
763 /* Return the status */
764 MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
765 return Status;
766 }
767
768 /* Did we get a prototype PTE back? */
769 if (!ProtoPte)
770 {
771 /* No, create a new PTE. First, write the protection */
772 PointerPte->u.Soft.Protection = ProtectionCode;
773
774 /* Lock the PFN database since we're going to grab a page */
775 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
776
777 /* Try to get a zero page */
778 Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
779 PageFrameIndex = MiRemoveZeroPageSafe(Color);
780 if (!PageFrameIndex)
781 {
782 /* Grab a page out of there. Later we should grab a colored zero page */
783 PageFrameIndex = MiRemoveAnyPage(Color);
784 ASSERT(PageFrameIndex);
785
786 /* Release the lock since we need to do some zeroing */
787 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
788
789 /* Zero out the page, since it's for user-mode */
790 MiZeroPfn(PageFrameIndex);
791
792 /* Grab the lock again so we can initialize the PFN entry */
793 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
794 }
795
796 /* Initialize the PFN entry now */
797 MiInitializePfn(PageFrameIndex, PointerPte, 1);
798
799 /* And we're done with the lock */
800 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
801
802 /* One more demand-zero fault */
803 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
804
805 /* Was the fault on an actual user page, or a kernel page for the user? */
806 if (PointerPte <= MiHighestUserPte)
807 {
808 /* User fault, build a user PTE */
809 MI_MAKE_HARDWARE_PTE_USER(&TempPte,
810 PointerPte,
811 PointerPte->u.Soft.Protection,
812 PageFrameIndex);
813 }
814 else
815 {
816 /* Session, kernel, or user PTE, figure it out and build it */
817 MI_MAKE_HARDWARE_PTE(&TempPte,
818 PointerPte,
819 PointerPte->u.Soft.Protection,
820 PageFrameIndex);
821 }
822
823 /* Write the dirty bit for writeable pages */
824 if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
825
826 /* And now write down the PTE, making the address valid */
827 MI_WRITE_VALID_PTE(PointerPte, TempPte);
828
829 /* Demand zero */
830 Status = STATUS_PAGE_FAULT_DEMAND_ZERO;
831 }
832 else
833 {
834 /* The only "prototype PTE" we support is the shared user data path */
835 ASSERT(ProtectionCode == MM_READONLY);
836
837 /* Write the prototype PTE */
838 TempPte = PrototypePte;
839 TempPte.u.Soft.Protection = ProtectionCode;
840 MI_WRITE_INVALID_PTE(PointerPte, TempPte);
841
842 /* Handle the fault */
843 Status = MiDispatchFault(StoreInstruction,
844 Address,
845 PointerPte,
846 ProtoPte,
847 FALSE,
848 CurrentProcess,
849 TrapInformation,
850 Vad);
851 ASSERT(Status == STATUS_PAGE_FAULT_TRANSITION);
852 ASSERT(PointerPte->u.Hard.Valid == 1);
853 ASSERT(PointerPte->u.Hard.PageFrameNumber == MmSharedUserDataPte->u.Hard.PageFrameNumber);
854 }
855
856 /* Release the working set */
857 MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
858 return Status;
859 }
860
861 /* EOF */