2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/pagfault.c
5 * PURPOSE: ARM Memory Manager Page Fault Handling
6 * PROGRAMMERS: ReactOS Portable Systems Group
9 /* INCLUDES *******************************************************************/
15 #line 15 "ARMĀ³::PAGFAULT"
16 #define MODULE_INVOLVED_IN_ARM3
17 #include "../ARM3/miarm.h"
19 /* GLOBALS ********************************************************************/
21 /* PRIVATE FUNCTIONS **********************************************************/
25 MiCheckVirtualAddress(IN PVOID VirtualAddress
,
26 OUT PULONG ProtectCode
,
31 /* No prototype/section support for now */
34 /* Only valid for user VADs for now */
35 ASSERT(VirtualAddress
<= MM_HIGHEST_USER_ADDRESS
);
37 /* Special case for shared data */
38 if (PAGE_ALIGN(VirtualAddress
) == (PVOID
)USER_SHARED_DATA
)
40 /* It's a read-only page */
41 *ProtectCode
= MM_READONLY
;
42 return MmSharedUserDataPte
;
45 /* Find the VAD, it might not exist if the address is bogus */
46 Vad
= MiLocateAddress(VirtualAddress
);
49 /* Bogus virtual address */
50 *ProtectCode
= MM_NOACCESS
;
54 /* This must be a TEB/PEB VAD */
55 ASSERT(Vad
->u
.VadFlags
.PrivateMemory
== TRUE
);
56 ASSERT(Vad
->u
.VadFlags
.MemCommit
== TRUE
);
57 ASSERT(Vad
->u
.VadFlags
.VadType
== VadNone
);
59 /* Return the protection on it */
60 *ProtectCode
= Vad
->u
.VadFlags
.Protection
;
66 MiCheckPdeForPagedPool(IN PVOID Address
)
69 NTSTATUS Status
= STATUS_SUCCESS
;
71 /* No session support in ReactOS yet */
72 ASSERT(MI_IS_SESSION_ADDRESS(Address
) == FALSE
);
73 ASSERT(MI_IS_SESSION_PTE(Address
) == FALSE
);
76 // Check if this is a fault while trying to access the page table itself
78 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address
))
81 // Send a hint to the page fault handler that this is only a valid fault
82 // if we already detected this was access within the page table range
84 PointerPde
= (PMMPDE
)MiAddressToPte(Address
);
85 Status
= STATUS_WAIT_1
;
87 else if (Address
< MmSystemRangeStart
)
90 // This is totally illegal
92 return STATUS_ACCESS_VIOLATION
;
97 // Get the PDE for the address
99 PointerPde
= MiAddressToPde(Address
);
103 // Check if it's not valid
105 if (PointerPde
->u
.Hard
.Valid
== 0)
110 /* This seems to be making the assumption that one PDE is one page long */
111 C_ASSERT(PAGE_SIZE
== (PD_COUNT
* (sizeof(MMPTE
) * PDE_COUNT
)));
114 // Copy it from our double-mapped system page directory
116 InterlockedExchangePte(PointerPde
,
117 MmSystemPagePtes
[((ULONG_PTR
)PointerPde
&
119 sizeof(MMPTE
)].u
.Long
);
131 MiZeroPfn(IN PFN_NUMBER PageFrameNumber
)
138 /* Get the PFN for this page */
139 Pfn1
= MiGetPfnEntry(PageFrameNumber
);
142 /* Grab a system PTE we can use to zero the page */
143 ZeroPte
= MiReserveSystemPtes(1, SystemPteSpace
);
146 /* Initialize the PTE for it */
147 TempPte
= ValidKernelPte
;
148 TempPte
.u
.Hard
.PageFrameNumber
= PageFrameNumber
;
151 if (Pfn1
->u3
.e1
.CacheAttribute
== MiWriteCombined
)
153 /* Write combining, no caching */
154 MI_PAGE_DISABLE_CACHE(&TempPte
);
155 MI_PAGE_WRITE_COMBINED(&TempPte
);
157 else if (Pfn1
->u3
.e1
.CacheAttribute
== MiNonCached
)
159 /* Write through, no caching */
160 MI_PAGE_DISABLE_CACHE(&TempPte
);
161 MI_PAGE_WRITE_THROUGH(&TempPte
);
164 /* Make the system PTE valid with our PFN */
165 MI_WRITE_VALID_PTE(ZeroPte
, TempPte
);
167 /* Get the address it maps to, and zero it out */
168 ZeroAddress
= MiPteToAddress(ZeroPte
);
169 KeZeroPages(ZeroAddress
, PAGE_SIZE
);
171 /* Now get rid of it */
172 MiReleaseSystemPtes(ZeroPte
, 1, SystemPteSpace
);
177 MiResolveDemandZeroFault(IN PVOID Address
,
178 IN PMMPTE PointerPte
,
179 IN PEPROCESS Process
,
182 PFN_NUMBER PageFrameNumber
;
184 BOOLEAN NeedZero
= FALSE
;
185 DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
189 /* Must currently only be called by paging path */
190 ASSERT(OldIrql
== MM_NOIRQL
);
194 ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte
));
197 ASSERT(Process
->ForkInProgress
== NULL
);
199 /* We'll need a zero page */
204 // Lock the PFN database
206 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
207 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
210 PageFrameNumber
= MiRemoveAnyPage(0);
211 DPRINT("New pool page: %lx\n", PageFrameNumber
);
214 MiInitializePfn(PageFrameNumber
, PointerPte
, TRUE
);
219 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
222 // Increment demand zero faults
224 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount
);
226 /* Zero the page if need be */
227 if (NeedZero
) MiZeroPfn(PageFrameNumber
);
230 if (PointerPte
<= MiHighestUserPte
)
233 MI_MAKE_HARDWARE_PTE_USER(&TempPte
,
235 PointerPte
->u
.Soft
.Protection
,
240 /* For kernel mode */
241 MI_MAKE_HARDWARE_PTE(&TempPte
,
243 PointerPte
->u
.Soft
.Protection
,
247 /* Set it dirty if it's a writable page */
248 if (TempPte
.u
.Hard
.Write
) TempPte
.u
.Hard
.Dirty
= TRUE
;
251 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
256 DPRINT("Paged pool page has now been paged in\n");
257 return STATUS_PAGE_FAULT_DEMAND_ZERO
;
262 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction
,
264 IN PMMPTE PointerPte
,
265 IN PMMPTE PointerProtoPte
,
270 PFN_NUMBER PageFrameIndex
;
272 /* Must be called with an valid prototype PTE, with the PFN lock held */
273 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
274 ASSERT(PointerProtoPte
->u
.Hard
.Valid
== 1);
277 ASSERT(PointerPte
->u
.Soft
.PageFileHigh
== 0xFFFFF);
280 PageFrameIndex
= PFN_FROM_PTE(PointerProtoPte
);
282 /* Release the PFN lock */
283 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
285 /* Build the user PTE */
286 ASSERT(Address
< MmSystemRangeStart
);
287 MI_MAKE_HARDWARE_PTE_USER(&TempPte
, PointerPte
, MM_READONLY
, PageFrameIndex
);
290 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
293 return STATUS_SUCCESS
;
298 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction
,
300 IN PMMPTE PointerPte
,
301 IN PMMPTE PointerProtoPte
,
302 IN OUT PMMPFN
*OutPfn
,
303 OUT PVOID
*PageFileData
,
305 IN PEPROCESS Process
,
307 IN PVOID TrapInformation
)
311 PFN_NUMBER PageFrameIndex
;
313 /* Must be called with an invalid, prototype PTE, with the PFN lock held */
314 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
315 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
316 ASSERT(PointerPte
->u
.Soft
.Prototype
== 1);
318 /* Read the prototype PTE -- it must be valid since we only handle shared data */
319 TempPte
= *PointerProtoPte
;
320 ASSERT(TempPte
.u
.Hard
.Valid
== 1);
322 /* One more user of this mapped page */
323 PageFrameIndex
= PFN_FROM_PTE(&TempPte
);
324 Pfn1
= MiGetPfnEntry(PageFrameIndex
);
325 Pfn1
->u2
.ShareCount
++;
327 /* Call it a transition */
328 InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount
);
330 /* Complete the prototype PTE fault -- this will release the PFN lock */
331 return MiCompleteProtoPteFault(StoreInstruction
,
341 MiDispatchFault(IN BOOLEAN StoreInstruction
,
343 IN PMMPTE PointerPte
,
344 IN PMMPTE PointerProtoPte
,
345 IN BOOLEAN Recursive
,
346 IN PEPROCESS Process
,
347 IN PVOID TrapInformation
,
351 KIRQL OldIrql
, LockIrql
;
353 PMMPTE SuperProtoPte
;
354 DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
359 // Make sure APCs are off and we're not at dispatch
361 OldIrql
= KeGetCurrentIrql();
362 ASSERT(OldIrql
<= APC_LEVEL
);
363 ASSERT(KeAreAllApcsDisabled() == TRUE
);
366 // Grab a copy of the PTE
368 TempPte
= *PointerPte
;
370 /* Do we have a prototype PTE? */
373 /* This should never happen */
374 ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte
));
376 /* We currently only handle the shared user data PTE path */
377 ASSERT(Address
< MmSystemRangeStart
);
378 ASSERT(PointerPte
->u
.Soft
.Prototype
== 1);
379 ASSERT(PointerPte
->u
.Soft
.PageFileHigh
== 0xFFFFF);
382 /* Lock the PFN database */
383 LockIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
385 /* For the shared data page, this should be true */
386 SuperProtoPte
= MiAddressToPte(PointerProtoPte
);
387 ASSERT(SuperProtoPte
->u
.Hard
.Valid
== 1);
388 ASSERT(TempPte
.u
.Hard
.Valid
== 0);
390 /* Resolve the fault -- this will release the PFN lock */
391 Status
= MiResolveProtoPteFault(StoreInstruction
,
401 ASSERT(Status
== STATUS_SUCCESS
);
403 /* Complete this as a transition fault */
404 ASSERT(OldIrql
== KeGetCurrentIrql());
405 ASSERT(OldIrql
<= APC_LEVEL
);
406 ASSERT(KeAreAllApcsDisabled() == TRUE
);
407 return STATUS_PAGE_FAULT_TRANSITION
;
411 // The PTE must be invalid, but not totally blank
413 ASSERT(TempPte
.u
.Hard
.Valid
== 0);
414 ASSERT(TempPte
.u
.Long
!= 0);
417 // No prototype, transition or page file software PTEs in ARM3 yet
419 ASSERT(TempPte
.u
.Soft
.Prototype
== 0);
420 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
421 ASSERT(TempPte
.u
.Soft
.PageFileHigh
== 0);
424 // If we got this far, the PTE can only be a demand zero PTE, which is what
425 // we want. Go handle it!
427 Status
= MiResolveDemandZeroFault(Address
,
431 ASSERT(KeAreAllApcsDisabled () == TRUE
);
432 if (NT_SUCCESS(Status
))
435 // Make sure we're returning in a sane state and pass the status down
437 ASSERT(OldIrql
== KeGetCurrentIrql ());
438 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
443 // Generate an access fault
445 return STATUS_ACCESS_VIOLATION
;
450 MmArmAccessFault(IN BOOLEAN StoreInstruction
,
452 IN KPROCESSOR_MODE Mode
,
453 IN PVOID TrapInformation
)
455 KIRQL OldIrql
= KeGetCurrentIrql(), LockIrql
;
456 PMMPTE PointerPte
, ProtoPte
;
459 PETHREAD CurrentThread
;
460 PEPROCESS CurrentProcess
;
462 PMMSUPPORT WorkingSet
;
463 ULONG ProtectionCode
;
465 PFN_NUMBER PageFrameIndex
;
466 DPRINT("ARM3 FAULT AT: %p\n", Address
);
469 // Get the PTE and PDE
471 PointerPte
= MiAddressToPte(Address
);
472 PointerPde
= MiAddressToPde(Address
);
473 #if (_MI_PAGING_LEVELS >= 3)
474 /* We need the PPE and PXE addresses */
479 // Check for dispatch-level snafu
481 if (OldIrql
> APC_LEVEL
)
484 // There are some special cases where this is okay, but not in ARM3 yet
486 DbgPrint("MM:***PAGE FAULT AT IRQL > 1 Va %p, IRQL %lx\n",
489 ASSERT(OldIrql
<= APC_LEVEL
);
493 // Check for kernel fault
495 if (Address
>= MmSystemRangeStart
)
498 // What are you even DOING here?
500 if (Mode
== UserMode
) return STATUS_ACCESS_VIOLATION
;
502 #if (_MI_PAGING_LEVELS >= 3)
503 /* Need to check PXE and PDE validity */
510 if (!PointerPde
->u
.Hard
.Valid
== 0)
515 DPRINT("Invalid PDE\n");
516 #if (_MI_PAGING_LEVELS == 2)
518 // Handle mapping in "Special" PDE directoreis
520 MiCheckPdeForPagedPool(Address
);
523 // Now we SHOULD be good
525 if (PointerPde
->u
.Hard
.Valid
== 0)
528 // FIXFIX: Do the S-LIST hack
534 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
537 (ULONG_PTR
)TrapInformation
,
543 // The PDE is valid, so read the PTE
545 TempPte
= *PointerPte
;
546 if (TempPte
.u
.Hard
.Valid
== 1)
549 // Only two things can go wrong here:
550 // Executing NX page (we couldn't care less)
551 // Writing to a read-only page (the stuff ARM3 works with is write,
552 // so again, moot point).
554 if (StoreInstruction
)
556 DPRINT1("Should NEVER happen on ARM3!!!\n");
557 return STATUS_ACCESS_VIOLATION
;
561 // Otherwise, the PDE was probably invalid, and all is good now
563 return STATUS_SUCCESS
;
567 // Check for a fault on the page table or hyperspace itself
569 if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address
))
572 // This might happen...not sure yet
574 DPRINT1("FAULT ON PAGE TABLES: %p %lx %lx!\n", Address
, *PointerPte
, *PointerPde
);
575 #if (_MI_PAGING_LEVELS == 2)
577 // Map in the page table
579 if (MiCheckPdeForPagedPool(Address
) == STATUS_WAIT_1
)
581 DPRINT1("PAGE TABLES FAULTED IN!\n");
582 return STATUS_SUCCESS
;
586 // Otherwise the page table doesn't actually exist
588 DPRINT1("FAILING\n");
589 return STATUS_ACCESS_VIOLATION
;
592 /* In this path, we are using the system working set */
593 CurrentThread
= PsGetCurrentThread();
594 WorkingSet
= &MmSystemCacheWs
;
597 KeRaiseIrql(APC_LEVEL
, &LockIrql
);
598 MiLockWorkingSet(CurrentThread
, WorkingSet
);
601 // Re-read PTE now that the IRQL has been raised
603 TempPte
= *PointerPte
;
604 if (TempPte
.u
.Hard
.Valid
== 1)
607 // Only two things can go wrong here:
608 // Executing NX page (we couldn't care less)
609 // Writing to a read-only page (the stuff ARM3 works with is write,
610 // so again, moot point.
612 if (StoreInstruction
)
614 DPRINT1("Should NEVER happen on ARM3!!!\n");
615 return STATUS_ACCESS_VIOLATION
;
618 /* Release the working set */
619 MiUnlockWorkingSet(CurrentThread
, WorkingSet
);
620 KeLowerIrql(LockIrql
);
623 // Otherwise, the PDE was probably invalid, and all is good now
625 return STATUS_SUCCESS
;
628 /* Check one kind of prototype PTE */
629 if (TempPte
.u
.Soft
.Prototype
)
631 /* The one used for protected pool... */
632 ASSERT(MmProtectFreedNonPagedPool
== TRUE
);
634 /* Make sure protected pool is on, and that this is a pool address */
635 if ((MmProtectFreedNonPagedPool
) &&
636 (((Address
>= MmNonPagedPoolStart
) &&
637 (Address
< (PVOID
)((ULONG_PTR
)MmNonPagedPoolStart
+
638 MmSizeOfNonPagedPoolInBytes
))) ||
639 ((Address
>= MmNonPagedPoolExpansionStart
) &&
640 (Address
< MmNonPagedPoolEnd
))))
642 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
643 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL
,
652 // We don't implement transition PTEs
654 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
657 // Now do the real fault handling
659 Status
= MiDispatchFault(StoreInstruction
,
668 /* Release the working set */
669 ASSERT(KeAreAllApcsDisabled() == TRUE
);
670 MiUnlockWorkingSet(CurrentThread
, WorkingSet
);
671 KeLowerIrql(LockIrql
);
676 DPRINT("Fault resolved with status: %lx\n", Status
);
680 /* This is a user fault */
681 CurrentThread
= PsGetCurrentThread();
682 CurrentProcess
= PsGetCurrentProcess();
684 /* Lock the working set */
685 MiLockProcessWorkingSet(CurrentProcess
, CurrentThread
);
687 #if (_MI_PAGING_LEVELS >= 3)
688 /* Need to check/handle PPE and PXE validity too */
692 /* First things first, is the PDE valid? */
693 ASSERT(PointerPde
!= MiAddressToPde(PTE_BASE
));
694 ASSERT(PointerPde
->u
.Hard
.LargePage
== 0);
695 if (PointerPde
->u
.Hard
.Valid
== 0)
697 /* Right now, we only handle scenarios where the PDE is totally empty */
698 ASSERT(PointerPde
->u
.Long
== 0);
700 /* Check if this address range belongs to a valid allocation (VAD) */
701 MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
703 /* Right now, we expect a valid protection mask on the VAD */
704 ASSERT(ProtectionCode
!= MM_NOACCESS
);
706 /* Make the PDE demand-zero */
707 MI_WRITE_INVALID_PTE(PointerPde
, DemandZeroPde
);
709 /* And go dispatch the fault on the PDE. This should handle the demand-zero */
710 Status
= MiDispatchFault(TRUE
,
715 PsGetCurrentProcess(),
719 /* We should come back with APCs enabled, and with a valid PDE */
720 ASSERT(KeAreAllApcsDisabled() == TRUE
);
721 #if (_MI_PAGING_LEVELS >= 3)
722 /* Need to check/handle PPE and PXE validity too */
725 ASSERT(PointerPde
->u
.Hard
.Valid
== 1);
728 /* Now capture the PTE. We only handle cases where it's totally empty */
729 TempPte
= *PointerPte
;
730 ASSERT(TempPte
.u
.Long
== 0);
732 /* Check if this address range belongs to a valid allocation (VAD) */
733 ProtoPte
= MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
734 if (ProtectionCode
== MM_NOACCESS
)
736 /* This is a bogus VA */
737 Status
= STATUS_ACCESS_VIOLATION
;
739 /* Could be a not-yet-mapped paged pool page table */
740 #if (_MI_PAGING_LEVELS == 2)
741 MiCheckPdeForPagedPool(Address
);
743 /* See if that fixed it */
744 if (PointerPte
->u
.Hard
.Valid
== 1) Status
= STATUS_SUCCESS
;
746 /* Return the status */
747 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
751 /* Did we get a prototype PTE back? */
754 /* No, create a new PTE. First, write the protection */
755 PointerPte
->u
.Soft
.Protection
= ProtectionCode
;
757 /* Lock the PFN database since we're going to grab a page */
758 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
760 /* Grab a page out of there. Later we should grab a colored zero page */
761 PageFrameIndex
= MiRemoveAnyPage(0);
762 ASSERT(PageFrameIndex
);
764 /* Release the lock since we need to do some zeroing */
765 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
767 /* Zero out the page, since it's for user-mode */
768 MiZeroPfn(PageFrameIndex
);
770 /* Grab the lock again so we can initialize the PFN entry */
771 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
773 /* Initialize the PFN entry now */
774 MiInitializePfn(PageFrameIndex
, PointerPte
, 1);
776 /* And we're done with the lock */
777 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
779 /* One more demand-zero fault */
780 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount
);
782 /* Was the fault on an actual user page, or a kernel page for the user? */
783 if (PointerPte
<= MiHighestUserPte
)
785 /* User fault, build a user PTE */
786 MI_MAKE_HARDWARE_PTE_USER(&TempPte
,
788 PointerPte
->u
.Soft
.Protection
,
793 /* Session, kernel, or user PTE, figure it out and build it */
794 MI_MAKE_HARDWARE_PTE(&TempPte
,
796 PointerPte
->u
.Soft
.Protection
,
800 /* Write the dirty bit for writeable pages */
801 if (TempPte
.u
.Hard
.Write
) TempPte
.u
.Hard
.Dirty
= TRUE
;
803 /* And now write down the PTE, making the address valid */
804 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
807 Status
= STATUS_PAGE_FAULT_DEMAND_ZERO
;
811 /* The only "prototype PTE" we support is the shared user data path */
812 ASSERT(ProtectionCode
== MM_READONLY
);
814 /* Write the prototype PTE */
815 TempPte
= PrototypePte
;
816 TempPte
.u
.Soft
.Protection
= ProtectionCode
;
817 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
819 /* Handle the fault */
820 Status
= MiDispatchFault(StoreInstruction
,
828 ASSERT(Status
== STATUS_PAGE_FAULT_TRANSITION
);
829 ASSERT(PointerPte
->u
.Hard
.Valid
== 1);
830 ASSERT(PointerPte
->u
.Hard
.PageFrameNumber
== MmSharedUserDataPte
->u
.Hard
.PageFrameNumber
);
833 /* Release the working set */
834 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);