2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/pagfault.c
5 * PURPOSE: ARM Memory Manager Page Fault Handling
6 * PROGRAMMERS: ReactOS Portable Systems Group
9 /* INCLUDES *******************************************************************/
15 #line 15 "ARMĀ³::PAGFAULT"
16 #define MODULE_INVOLVED_IN_ARM3
17 #include "../ARM3/miarm.h"
19 /* GLOBALS ********************************************************************/
22 BOOLEAN UserPdeFault
= FALSE
;
25 /* PRIVATE FUNCTIONS **********************************************************/
29 MiCheckVirtualAddress(IN PVOID VirtualAddress
,
30 OUT PULONG ProtectCode
,
36 /* No prototype/section support for now */
39 /* Check if this is a page table address */
40 if (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress
))
42 /* This should never happen, as these addresses are handled by the double-maping */
43 if (((PMMPTE
)VirtualAddress
>= MiAddressToPte(MmPagedPoolStart
)) &&
44 ((PMMPTE
)VirtualAddress
<= MmPagedPoolInfo
.LastPteForPagedPool
))
46 /* Fail such access */
47 *ProtectCode
= MM_NOACCESS
;
51 /* Return full access rights */
52 *ProtectCode
= MM_READWRITE
;
56 /* Should not be a session address */
57 ASSERT(MI_IS_SESSION_ADDRESS(VirtualAddress
) == FALSE
);
59 /* Special case for shared data */
60 if (PAGE_ALIGN(VirtualAddress
) == (PVOID
)USER_SHARED_DATA
)
62 /* It's a read-only page */
63 *ProtectCode
= MM_READONLY
;
64 return MmSharedUserDataPte
;
67 /* Find the VAD, it might not exist if the address is bogus */
68 Vad
= MiLocateAddress(VirtualAddress
);
71 /* Bogus virtual address */
72 *ProtectCode
= MM_NOACCESS
;
76 /* This must be a VM VAD */
77 ASSERT(Vad
->u
.VadFlags
.VadType
== VadNone
);
79 /* Check if it's a section, or just an allocation */
80 if (Vad
->u
.VadFlags
.PrivateMemory
== TRUE
)
82 /* This must be a TEB/PEB VAD */
83 ASSERT(Vad
->u
.VadFlags
.MemCommit
== TRUE
);
84 *ProtectCode
= Vad
->u
.VadFlags
.Protection
;
89 /* Return the proto VAD */
90 ASSERT(Vad
->u2
.VadFlags2
.ExtendableFile
== 0);
93 /* Get the prototype PTE for this page */
94 PointerPte
= (((ULONG_PTR
)VirtualAddress
>> PAGE_SHIFT
) - Vad
->StartingVpn
) + Vad
->FirstPrototypePte
;
95 ASSERT(PointerPte
<= Vad
->LastContiguousPte
);
96 ASSERT(PointerPte
!= NULL
);
98 /* Return the Prototype PTE and the protection for the page mapping */
99 *ProtectCode
= Vad
->u
.VadFlags
.Protection
;
106 MiCheckPdeForPagedPool(IN PVOID Address
)
109 NTSTATUS Status
= STATUS_SUCCESS
;
111 /* No session support in ReactOS yet */
112 ASSERT(MI_IS_SESSION_ADDRESS(Address
) == FALSE
);
113 ASSERT(MI_IS_SESSION_PTE(Address
) == FALSE
);
116 // Check if this is a fault while trying to access the page table itself
118 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address
))
121 // Send a hint to the page fault handler that this is only a valid fault
122 // if we already detected this was access within the page table range
124 PointerPde
= (PMMPDE
)MiAddressToPte(Address
);
125 Status
= STATUS_WAIT_1
;
127 else if (Address
< MmSystemRangeStart
)
130 // This is totally illegal
132 return STATUS_ACCESS_VIOLATION
;
137 // Get the PDE for the address
139 PointerPde
= MiAddressToPde(Address
);
143 // Check if it's not valid
145 if (PointerPde
->u
.Hard
.Valid
== 0)
151 // Copy it from our double-mapped system page directory
153 InterlockedExchangePte(PointerPde
,
154 MmSystemPagePtes
[((ULONG_PTR
)PointerPde
& (SYSTEM_PD_SIZE
- 1)) / sizeof(MMPTE
)].u
.Long
);
166 MiZeroPfn(IN PFN_NUMBER PageFrameNumber
)
173 /* Get the PFN for this page */
174 Pfn1
= MiGetPfnEntry(PageFrameNumber
);
177 /* Grab a system PTE we can use to zero the page */
178 ZeroPte
= MiReserveSystemPtes(1, SystemPteSpace
);
181 /* Initialize the PTE for it */
182 TempPte
= ValidKernelPte
;
183 TempPte
.u
.Hard
.PageFrameNumber
= PageFrameNumber
;
186 if (Pfn1
->u3
.e1
.CacheAttribute
== MiWriteCombined
)
188 /* Write combining, no caching */
189 MI_PAGE_DISABLE_CACHE(&TempPte
);
190 MI_PAGE_WRITE_COMBINED(&TempPte
);
192 else if (Pfn1
->u3
.e1
.CacheAttribute
== MiNonCached
)
194 /* Write through, no caching */
195 MI_PAGE_DISABLE_CACHE(&TempPte
);
196 MI_PAGE_WRITE_THROUGH(&TempPte
);
199 /* Make the system PTE valid with our PFN */
200 MI_WRITE_VALID_PTE(ZeroPte
, TempPte
);
202 /* Get the address it maps to, and zero it out */
203 ZeroAddress
= MiPteToAddress(ZeroPte
);
204 KeZeroPages(ZeroAddress
, PAGE_SIZE
);
206 /* Now get rid of it */
207 MiReleaseSystemPtes(ZeroPte
, 1, SystemPteSpace
);
212 MiResolveDemandZeroFault(IN PVOID Address
,
213 IN PMMPTE PointerPte
,
214 IN PEPROCESS Process
,
217 PFN_NUMBER PageFrameNumber
= 0;
219 BOOLEAN NeedZero
= FALSE
, HaveLock
= FALSE
;
221 DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
225 /* Must currently only be called by paging path */
226 if ((Process
) && (OldIrql
== MM_NOIRQL
))
229 ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte
));
232 ASSERT(Process
->ForkInProgress
== NULL
);
234 /* Get process color */
235 Color
= MI_GET_NEXT_PROCESS_COLOR(Process
);
236 ASSERT(Color
!= 0xFFFFFFFF);
238 /* We'll need a zero page */
243 /* Check if we need a zero page */
244 NeedZero
= (OldIrql
!= MM_NOIRQL
);
246 /* Get the next system page color */
247 Color
= MI_GET_NEXT_COLOR();
250 /* Check if the PFN database should be acquired */
251 if (OldIrql
== MM_NOIRQL
)
253 /* Acquire it and remember we should release it after */
254 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
258 /* We either manually locked the PFN DB, or already came with it locked */
259 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
261 /* Do we need a zero page? */
262 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
264 if (UserPdeFault
) MI_SET_USAGE(MI_USAGE_PAGE_TABLE
);
265 if (!UserPdeFault
) MI_SET_USAGE(MI_USAGE_DEMAND_ZERO
);
267 if (Process
) MI_SET_PROCESS2(Process
->ImageFileName
);
268 if (!Process
) MI_SET_PROCESS2("Kernel Demand 0");
269 if ((NeedZero
) && (Process
))
271 /* Try to get one, if we couldn't grab a free page and zero it */
272 PageFrameNumber
= MiRemoveZeroPageSafe(Color
);
275 /* We got a genuine zero page, stop worrying about it */
280 /* We'll need a free page and zero it manually */
281 PageFrameNumber
= MiRemoveAnyPage(Color
);
286 /* Process or system doesn't want a zero page, grab anything */
287 PageFrameNumber
= MiRemoveAnyPage(Color
);
291 /* System wants a zero page, obtain one */
292 PageFrameNumber
= MiRemoveZeroPage(Color
);
296 MiInitializePfn(PageFrameNumber
, PointerPte
, TRUE
);
298 /* Release PFN lock if needed */
299 if (HaveLock
) KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
302 // Increment demand zero faults
304 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount
);
306 /* Zero the page if need be */
307 if (NeedZero
) MiZeroPfn(PageFrameNumber
);
310 if (PointerPte
<= MiHighestUserPte
)
313 MI_MAKE_HARDWARE_PTE_USER(&TempPte
,
315 PointerPte
->u
.Soft
.Protection
,
320 /* For kernel mode */
321 MI_MAKE_HARDWARE_PTE(&TempPte
,
323 PointerPte
->u
.Soft
.Protection
,
327 /* Set it dirty if it's a writable page */
328 if (MI_IS_PAGE_WRITEABLE(&TempPte
)) MI_MAKE_DIRTY_PAGE(&TempPte
);
331 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
336 DPRINT("Paged pool page has now been paged in\n");
337 return STATUS_PAGE_FAULT_DEMAND_ZERO
;
342 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction
,
344 IN PMMPTE PointerPte
,
345 IN PMMPTE PointerProtoPte
,
352 PFN_NUMBER PageFrameIndex
;
354 /* Must be called with an valid prototype PTE, with the PFN lock held */
355 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
356 ASSERT(PointerProtoPte
->u
.Hard
.Valid
== 1);
359 PageFrameIndex
= PFN_FROM_PTE(PointerProtoPte
);
361 /* Get the PFN entry and set it as a prototype PTE */
362 Pfn1
= MiGetPfnEntry(PageFrameIndex
);
363 Pfn1
->u3
.e1
.PrototypePte
= 1;
365 /* FIXME: Increment the share count for the page table */
367 /* Check where we should be getting the protection information from */
368 if (PointerPte
->u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
)
370 /* Get the protection from the PTE, there's no real Proto PTE data */
371 Protection
= PointerPte
->u
.Soft
.Protection
;
375 /* Get the protection from the original PTE link */
376 OriginalPte
= &Pfn1
->OriginalPte
;
377 Protection
= OriginalPte
->u
.Soft
.Protection
;
380 /* Release the PFN lock */
381 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
383 /* Remove caching bits */
384 Protection
&= ~(MM_NOCACHE
| MM_NOACCESS
);
386 /* Check if this is a kernel or user address */
387 if (Address
< MmSystemRangeStart
)
389 /* Build the user PTE */
390 MI_MAKE_HARDWARE_PTE_USER(&TempPte
, PointerPte
, Protection
, PageFrameIndex
);
394 /* Build the kernel PTE */
395 MI_MAKE_HARDWARE_PTE(&TempPte
, PointerPte
, Protection
, PageFrameIndex
);
399 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
402 return STATUS_SUCCESS
;
407 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction
,
409 IN PMMPTE PointerPte
,
410 IN PMMPTE PointerProtoPte
,
411 IN OUT PMMPFN
*OutPfn
,
412 OUT PVOID
*PageFileData
,
414 IN PEPROCESS Process
,
416 IN PVOID TrapInformation
)
420 PFN_NUMBER PageFrameIndex
;
423 /* Must be called with an invalid, prototype PTE, with the PFN lock held */
424 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
425 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
426 ASSERT(PointerPte
->u
.Soft
.Prototype
== 1);
428 /* Read the prototype PTE and check if it's valid */
429 TempPte
= *PointerProtoPte
;
430 if (TempPte
.u
.Hard
.Valid
== 1)
432 /* One more user of this mapped page */
433 PageFrameIndex
= PFN_FROM_PTE(&TempPte
);
434 Pfn1
= MiGetPfnEntry(PageFrameIndex
);
435 Pfn1
->u2
.ShareCount
++;
437 /* Call it a transition */
438 InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount
);
440 /* Complete the prototype PTE fault -- this will release the PFN lock */
441 return MiCompleteProtoPteFault(StoreInstruction
,
449 /* Make sure there's some protection mask */
450 if (TempPte
.u
.Long
== 0)
452 /* Release the lock */
453 DPRINT1("Access on reserved section?\n");
454 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
455 return STATUS_ACCESS_VIOLATION
;
458 /* This is the only thing we support right now */
459 ASSERT(TempPte
.u
.Soft
.PageFileHigh
== 0);
460 ASSERT(TempPte
.u
.Proto
.ReadOnly
== 0);
461 ASSERT(PointerPte
> MiHighestUserPte
);
462 ASSERT(TempPte
.u
.Soft
.Prototype
== 0);
463 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
465 /* Resolve the demand zero fault */
466 Status
= MiResolveDemandZeroFault(Address
, PointerProtoPte
, Process
, OldIrql
);
467 ASSERT(NT_SUCCESS(Status
));
469 /* Complete the prototype PTE fault -- this will release the PFN lock */
470 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
471 return MiCompleteProtoPteFault(StoreInstruction
,
481 MiDispatchFault(IN BOOLEAN StoreInstruction
,
483 IN PMMPTE PointerPte
,
484 IN PMMPTE PointerProtoPte
,
485 IN BOOLEAN Recursive
,
486 IN PEPROCESS Process
,
487 IN PVOID TrapInformation
,
491 KIRQL OldIrql
, LockIrql
;
493 PMMPTE SuperProtoPte
;
494 DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
499 // Make sure APCs are off and we're not at dispatch
501 OldIrql
= KeGetCurrentIrql();
502 ASSERT(OldIrql
<= APC_LEVEL
);
503 ASSERT(KeAreAllApcsDisabled() == TRUE
);
506 // Grab a copy of the PTE
508 TempPte
= *PointerPte
;
510 /* Do we have a prototype PTE? */
513 /* This should never happen */
514 ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte
));
516 /* Check if this is a kernel-mode address */
517 SuperProtoPte
= MiAddressToPte(PointerProtoPte
);
518 if (Address
>= MmSystemRangeStart
)
520 /* Lock the PFN database */
521 LockIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
523 /* Has the PTE been made valid yet? */
524 if (!SuperProtoPte
->u
.Hard
.Valid
)
531 /* Resolve the fault -- this will release the PFN lock */
532 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
533 Status
= MiResolveProtoPteFault(StoreInstruction
,
543 ASSERT(Status
== STATUS_SUCCESS
);
545 /* Complete this as a transition fault */
546 ASSERT(OldIrql
== KeGetCurrentIrql());
547 ASSERT(OldIrql
<= APC_LEVEL
);
548 ASSERT(KeAreAllApcsDisabled() == TRUE
);
554 /* We currently only handle very limited paths */
555 ASSERT(PointerPte
->u
.Soft
.Prototype
== 1);
556 ASSERT(PointerPte
->u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
);
558 /* Lock the PFN database */
559 LockIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
561 /* For our current usage, this should be true */
562 ASSERT(SuperProtoPte
->u
.Hard
.Valid
== 1);
563 ASSERT(TempPte
.u
.Hard
.Valid
== 0);
565 /* Resolve the fault -- this will release the PFN lock */
566 Status
= MiResolveProtoPteFault(StoreInstruction
,
576 ASSERT(Status
== STATUS_SUCCESS
);
578 /* Complete this as a transition fault */
579 ASSERT(OldIrql
== KeGetCurrentIrql());
580 ASSERT(OldIrql
<= APC_LEVEL
);
581 ASSERT(KeAreAllApcsDisabled() == TRUE
);
582 return STATUS_PAGE_FAULT_TRANSITION
;
587 // The PTE must be invalid, but not totally blank
589 ASSERT(TempPte
.u
.Hard
.Valid
== 0);
590 ASSERT(TempPte
.u
.Long
!= 0);
593 // No prototype, transition or page file software PTEs in ARM3 yet
595 ASSERT(TempPte
.u
.Soft
.Prototype
== 0);
596 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
597 ASSERT(TempPte
.u
.Soft
.PageFileHigh
== 0);
600 // If we got this far, the PTE can only be a demand zero PTE, which is what
601 // we want. Go handle it!
603 Status
= MiResolveDemandZeroFault(Address
,
607 ASSERT(KeAreAllApcsDisabled() == TRUE
);
608 if (NT_SUCCESS(Status
))
611 // Make sure we're returning in a sane state and pass the status down
613 ASSERT(OldIrql
== KeGetCurrentIrql());
614 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
619 // Generate an access fault
621 return STATUS_ACCESS_VIOLATION
;
626 MmArmAccessFault(IN BOOLEAN StoreInstruction
,
628 IN KPROCESSOR_MODE Mode
,
629 IN PVOID TrapInformation
)
631 KIRQL OldIrql
= KeGetCurrentIrql(), LockIrql
;
632 PMMPTE PointerPte
, ProtoPte
= NULL
;
635 PETHREAD CurrentThread
;
636 PEPROCESS CurrentProcess
;
638 PMMSUPPORT WorkingSet
;
639 ULONG ProtectionCode
;
641 PFN_NUMBER PageFrameIndex
;
643 DPRINT("ARM3 FAULT AT: %p\n", Address
);
646 // Get the PTE and PDE
648 PointerPte
= MiAddressToPte(Address
);
649 PointerPde
= MiAddressToPde(Address
);
650 #if (_MI_PAGING_LEVELS >= 3)
651 /* We need the PPE and PXE addresses */
656 // Check for dispatch-level snafu
658 if (OldIrql
> APC_LEVEL
)
661 // There are some special cases where this is okay, but not in ARM3 yet
663 DbgPrint("MM:***PAGE FAULT AT IRQL > 1 Va %p, IRQL %lx\n",
666 ASSERT(OldIrql
<= APC_LEVEL
);
670 // Check for kernel fault
672 while (Address
>= MmSystemRangeStart
)
675 // What are you even DOING here?
677 if (Mode
== UserMode
) return STATUS_ACCESS_VIOLATION
;
679 #if (_MI_PAGING_LEVELS >= 3)
680 /* Need to check PXE and PDE validity */
687 if (!PointerPde
->u
.Hard
.Valid
== 0)
692 DPRINT("Invalid PDE\n");
693 #if (_MI_PAGING_LEVELS == 2)
695 // Handle mapping in "Special" PDE directoreis
697 MiCheckPdeForPagedPool(Address
);
700 // Now we SHOULD be good
702 if (PointerPde
->u
.Hard
.Valid
== 0)
705 // FIXFIX: Do the S-LIST hack
711 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
714 (ULONG_PTR
)TrapInformation
,
720 // The PDE is valid, so read the PTE
722 TempPte
= *PointerPte
;
723 if (TempPte
.u
.Hard
.Valid
== 1)
726 // Only two things can go wrong here:
727 // Executing NX page (we couldn't care less)
728 // Writing to a read-only page (the stuff ARM3 works with is write,
729 // so again, moot point).
731 if (StoreInstruction
)
733 DPRINT1("Should NEVER happen on ARM3!!!\n");
734 return STATUS_ACCESS_VIOLATION
;
738 // Otherwise, the PDE was probably invalid, and all is good now
740 return STATUS_SUCCESS
;
744 // Check for a fault on the page table or hyperspace itself
746 if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address
))
748 #if (_MI_PAGING_LEVELS == 2)
749 /* Could be paged pool access from a new process -- synchronize the page directories */
750 if (MiCheckPdeForPagedPool(Address
) == STATUS_WAIT_1
)
752 DPRINT1("PAGE TABLES FAULTED IN!\n");
753 return STATUS_SUCCESS
;
756 /* Otherwise this could be a commit of a virtual address */
760 /* In this path, we are using the system working set */
761 CurrentThread
= PsGetCurrentThread();
762 WorkingSet
= &MmSystemCacheWs
;
765 KeRaiseIrql(APC_LEVEL
, &LockIrql
);
766 MiLockWorkingSet(CurrentThread
, WorkingSet
);
769 // Re-read PTE now that the IRQL has been raised
771 TempPte
= *PointerPte
;
772 if (TempPte
.u
.Hard
.Valid
== 1)
775 // Only two things can go wrong here:
776 // Executing NX page (we couldn't care less)
777 // Writing to a read-only page (the stuff ARM3 works with is write,
778 // so again, moot point.
780 if (StoreInstruction
)
782 DPRINT1("Should NEVER happen on ARM3!!!\n");
783 return STATUS_ACCESS_VIOLATION
;
786 /* Release the working set */
787 MiUnlockWorkingSet(CurrentThread
, WorkingSet
);
788 KeLowerIrql(LockIrql
);
791 // Otherwise, the PDE was probably invalid, and all is good now
793 return STATUS_SUCCESS
;
796 /* Check one kind of prototype PTE */
797 if (TempPte
.u
.Soft
.Prototype
)
799 /* Make sure protected pool is on, and that this is a pool address */
800 if ((MmProtectFreedNonPagedPool
) &&
801 (((Address
>= MmNonPagedPoolStart
) &&
802 (Address
< (PVOID
)((ULONG_PTR
)MmNonPagedPoolStart
+
803 MmSizeOfNonPagedPoolInBytes
))) ||
804 ((Address
>= MmNonPagedPoolExpansionStart
) &&
805 (Address
< MmNonPagedPoolEnd
))))
807 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
808 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL
,
815 /* Get the prototype PTE! */
816 ProtoPte
= MiProtoPteToPte(&TempPte
);
821 // We don't implement transition PTEs
823 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
825 /* Check for no-access PTE */
826 if (TempPte
.u
.Soft
.Protection
== MM_NOACCESS
)
828 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
829 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
832 (ULONG_PTR
)TrapInformation
,
837 /* Check for demand page */
838 if ((StoreInstruction
) && !(ProtoPte
) && !(TempPte
.u
.Hard
.Valid
))
840 /* Get the protection code */
841 if (!(TempPte
.u
.Soft
.Protection
& MM_READWRITE
))
843 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
844 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
847 (ULONG_PTR
)TrapInformation
,
853 // Now do the real fault handling
855 Status
= MiDispatchFault(StoreInstruction
,
864 /* Release the working set */
865 ASSERT(KeAreAllApcsDisabled() == TRUE
);
866 MiUnlockWorkingSet(CurrentThread
, WorkingSet
);
867 KeLowerIrql(LockIrql
);
872 DPRINT("Fault resolved with status: %lx\n", Status
);
876 /* This is a user fault */
877 CurrentThread
= PsGetCurrentThread();
878 CurrentProcess
= PsGetCurrentProcess();
880 /* Lock the working set */
881 MiLockProcessWorkingSet(CurrentProcess
, CurrentThread
);
883 #if (_MI_PAGING_LEVELS >= 3)
884 /* Need to check/handle PPE and PXE validity too */
888 /* First things first, is the PDE valid? */
889 ASSERT(PointerPde
->u
.Hard
.LargePage
== 0);
890 if (PointerPde
->u
.Hard
.Valid
== 0)
892 /* Right now, we only handle scenarios where the PDE is totally empty */
893 ASSERT(PointerPde
->u
.Long
== 0);
895 /* Check if this address range belongs to a valid allocation (VAD) */
896 MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
898 /* Right now, we expect a valid protection mask on the VAD */
899 ASSERT(ProtectionCode
!= MM_NOACCESS
);
901 /* Make the PDE demand-zero */
902 MI_WRITE_INVALID_PDE(PointerPde
, DemandZeroPde
);
904 /* And go dispatch the fault on the PDE. This should handle the demand-zero */
908 Status
= MiDispatchFault(TRUE
,
913 PsGetCurrentProcess(),
917 UserPdeFault
= FALSE
;
919 /* We should come back with APCs enabled, and with a valid PDE */
920 ASSERT(KeAreAllApcsDisabled() == TRUE
);
921 #if (_MI_PAGING_LEVELS >= 3)
922 /* Need to check/handle PPE and PXE validity too */
925 ASSERT(PointerPde
->u
.Hard
.Valid
== 1);
928 /* Now capture the PTE. Ignore virtual faults for now */
929 TempPte
= *PointerPte
;
930 ASSERT(TempPte
.u
.Hard
.Valid
== 0);
932 /* Quick check for demand-zero */
933 if (TempPte
.u
.Long
== (MM_READWRITE
<< MM_PTE_SOFTWARE_PROTECTION_BITS
))
935 /* Resolve the fault */
936 MiResolveDemandZeroFault(Address
,
941 /* Return the status */
942 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
943 return STATUS_PAGE_FAULT_DEMAND_ZERO
;
946 /* Get protection and check if it's a prototype PTE */
947 ProtectionCode
= TempPte
.u
.Soft
.Protection
;
948 ASSERT(TempPte
.u
.Soft
.Prototype
== 0);
950 /* Check for non-demand zero PTE */
951 if (TempPte
.u
.Long
!= 0)
953 /* This is a page fault, check for valid protection */
954 ASSERT(ProtectionCode
!= 0x100);
956 /* FIXME: Run MiAccessCheck */
958 /* Dispatch the fault */
959 Status
= MiDispatchFault(StoreInstruction
,
964 PsGetCurrentProcess(),
968 /* Return the status */
969 ASSERT(NT_SUCCESS(Status
));
970 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
971 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
975 /* Check if this address range belongs to a valid allocation (VAD) */
976 ASSERT(TempPte
.u
.Long
== 0);
977 ProtoPte
= MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
978 if (ProtectionCode
== MM_NOACCESS
)
980 /* This is a bogus VA */
981 Status
= STATUS_ACCESS_VIOLATION
;
983 /* Could be a not-yet-mapped paged pool page table */
984 #if (_MI_PAGING_LEVELS == 2)
985 MiCheckPdeForPagedPool(Address
);
987 /* See if that fixed it */
988 if (PointerPte
->u
.Hard
.Valid
== 1) Status
= STATUS_SUCCESS
;
990 /* Return the status */
991 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
995 /* Is this a user address? */
996 if (Address
<= MM_HIGHEST_USER_ADDRESS
)
998 /* Add an additional page table reference */
999 MmWorkingSetList
->UsedPageTableEntries
[MiGetPdeOffset(Address
)]++;
1000 ASSERT(MmWorkingSetList
->UsedPageTableEntries
[MiGetPdeOffset(Address
)] <= PTE_COUNT
);
1003 /* Did we get a prototype PTE back? */
1006 /* No, create a new PTE. First, write the protection */
1007 PointerPte
->u
.Soft
.Protection
= ProtectionCode
;
1009 /* Lock the PFN database since we're going to grab a page */
1010 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1012 /* Try to get a zero page */
1013 MI_SET_USAGE(MI_USAGE_PEB_TEB
);
1014 MI_SET_PROCESS2(CurrentProcess
->ImageFileName
);
1015 Color
= MI_GET_NEXT_PROCESS_COLOR(CurrentProcess
);
1016 PageFrameIndex
= MiRemoveZeroPageSafe(Color
);
1017 if (!PageFrameIndex
)
1019 /* Grab a page out of there. Later we should grab a colored zero page */
1020 PageFrameIndex
= MiRemoveAnyPage(Color
);
1021 ASSERT(PageFrameIndex
);
1023 /* Release the lock since we need to do some zeroing */
1024 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
1026 /* Zero out the page, since it's for user-mode */
1027 MiZeroPfn(PageFrameIndex
);
1029 /* Grab the lock again so we can initialize the PFN entry */
1030 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1033 /* Initialize the PFN entry now */
1034 MiInitializePfn(PageFrameIndex
, PointerPte
, 1);
1036 /* And we're done with the lock */
1037 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
1039 /* One more demand-zero fault */
1040 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount
);
1042 /* Was the fault on an actual user page, or a kernel page for the user? */
1043 if (PointerPte
<= MiHighestUserPte
)
1045 /* User fault, build a user PTE */
1046 MI_MAKE_HARDWARE_PTE_USER(&TempPte
,
1048 PointerPte
->u
.Soft
.Protection
,
1053 /* Session, kernel, or user PTE, figure it out and build it */
1054 MI_MAKE_HARDWARE_PTE(&TempPte
,
1056 PointerPte
->u
.Soft
.Protection
,
1060 /* Write the dirty bit for writeable pages */
1061 if (MI_IS_PAGE_WRITEABLE(&TempPte
)) MI_MAKE_DIRTY_PAGE(&TempPte
);
1063 /* And now write down the PTE, making the address valid */
1064 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
1067 Status
= STATUS_PAGE_FAULT_DEMAND_ZERO
;
1071 /* No guard page support yet */
1072 ASSERT((ProtectionCode
& MM_DECOMMIT
) == 0);
1073 ASSERT(ProtectionCode
!= 0x100);
1075 /* Write the prototype PTE */
1076 TempPte
= PrototypePte
;
1077 TempPte
.u
.Soft
.Protection
= ProtectionCode
;
1078 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
1080 /* Handle the fault */
1081 Status
= MiDispatchFault(StoreInstruction
,
1089 ASSERT(Status
== STATUS_PAGE_FAULT_TRANSITION
);
1090 ASSERT(PointerPte
->u
.Hard
.Valid
== 1);
1091 ASSERT(PointerPte
->u
.Hard
.PageFrameNumber
!= 0);
1094 /* Release the working set */
1095 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);