2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/pagfault.c
5 * PURPOSE: ARM Memory Manager Page Fault Handling
6 * PROGRAMMERS: ReactOS Portable Systems Group
9 /* INCLUDES *******************************************************************/
15 #define MODULE_INVOLVED_IN_ARM3
16 #include <mm/ARM3/miarm.h>
18 /* GLOBALS ********************************************************************/
20 #define HYDRA_PROCESS (PEPROCESS)1
22 BOOLEAN UserPdeFault
= FALSE
;
25 /* PRIVATE FUNCTIONS **********************************************************/
30 MiCheckForUserStackOverflow(IN PVOID Address
,
31 IN PVOID TrapInformation
)
33 PETHREAD CurrentThread
= PsGetCurrentThread();
34 PTEB Teb
= CurrentThread
->Tcb
.Teb
;
35 PVOID StackBase
, DeallocationStack
, NextStackAddress
;
36 SIZE_T GuaranteedSize
;
39 /* Do we own the address space lock? */
40 if (CurrentThread
->AddressSpaceOwner
== 1)
42 /* This isn't valid */
43 DPRINT1("Process owns address space lock\n");
44 ASSERT(KeAreAllApcsDisabled() == TRUE
);
45 return STATUS_GUARD_PAGE_VIOLATION
;
48 /* Are we attached? */
49 if (KeIsAttachedProcess())
51 /* This isn't valid */
52 DPRINT1("Process is attached\n");
53 return STATUS_GUARD_PAGE_VIOLATION
;
56 /* Read the current settings */
57 StackBase
= Teb
->NtTib
.StackBase
;
58 DeallocationStack
= Teb
->DeallocationStack
;
59 GuaranteedSize
= Teb
->GuaranteedStackBytes
;
60 DPRINT("Handling guard page fault with Stacks Addresses 0x%p and 0x%p, guarantee: %lx\n",
61 StackBase
, DeallocationStack
, GuaranteedSize
);
63 /* Guarantees make this code harder, for now, assume there aren't any */
64 ASSERT(GuaranteedSize
== 0);
66 /* So allocate only the minimum guard page size */
67 GuaranteedSize
= PAGE_SIZE
;
69 /* Does this faulting stack address actually exist in the stack? */
70 if ((Address
>= StackBase
) || (Address
< DeallocationStack
))
73 DPRINT1("Faulting address outside of stack bounds. Address=%p, StackBase=%p, DeallocationStack=%p\n",
74 Address
, StackBase
, DeallocationStack
);
75 return STATUS_GUARD_PAGE_VIOLATION
;
78 /* This is where the stack will start now */
79 NextStackAddress
= (PVOID
)((ULONG_PTR
)PAGE_ALIGN(Address
) - GuaranteedSize
);
81 /* Do we have at least one page between here and the end of the stack? */
82 if (((ULONG_PTR
)NextStackAddress
- PAGE_SIZE
) <= (ULONG_PTR
)DeallocationStack
)
84 /* We don't -- Trying to make this guard page valid now */
85 DPRINT1("Close to our death...\n");
87 /* Calculate the next memory address */
88 NextStackAddress
= (PVOID
)((ULONG_PTR
)PAGE_ALIGN(DeallocationStack
) + GuaranteedSize
);
90 /* Allocate the memory */
91 Status
= ZwAllocateVirtualMemory(NtCurrentProcess(),
97 if (NT_SUCCESS(Status
))
100 Teb
->NtTib
.StackLimit
= NextStackAddress
;
104 DPRINT1("Failed to allocate memory\n");
107 return STATUS_STACK_OVERFLOW
;
110 /* Don't handle this flag yet */
111 ASSERT((PsGetCurrentProcess()->Peb
->NtGlobalFlag
& FLG_DISABLE_STACK_EXTENSION
) == 0);
113 /* Update the stack limit */
114 Teb
->NtTib
.StackLimit
= (PVOID
)((ULONG_PTR
)NextStackAddress
+ GuaranteedSize
);
116 /* Now move the guard page to the next page */
117 Status
= ZwAllocateVirtualMemory(NtCurrentProcess(),
122 PAGE_READWRITE
| PAGE_GUARD
);
123 if ((NT_SUCCESS(Status
) || (Status
== STATUS_ALREADY_COMMITTED
)))
126 DPRINT("Guard page handled successfully for %p\n", Address
);
127 return STATUS_PAGE_FAULT_GUARD_PAGE
;
130 /* Fail, we couldn't move the guard page */
131 DPRINT1("Guard page failure: %lx\n", Status
);
133 return STATUS_STACK_OVERFLOW
;
139 _In_ ULONG ProtectionMask
,
141 _In_ BOOLEAN Execute
)
143 #define _BYTE_MASK(Bit0, Bit1, Bit2, Bit3, Bit4, Bit5, Bit6, Bit7) \
144 (Bit0) | ((Bit1) << 1) | ((Bit2) << 2) | ((Bit3) << 3) | \
145 ((Bit4) << 4) | ((Bit5) << 5) | ((Bit6) << 6) | ((Bit7) << 7)
146 static const UCHAR AccessAllowedMask
[2][2] =
148 { // Protect 0 1 2 3 4 5 6 7
149 _BYTE_MASK(0, 1, 1, 1, 1, 1, 1, 1), // READ
150 _BYTE_MASK(0, 0, 1, 1, 0, 0, 1, 1), // EXECUTE READ
153 _BYTE_MASK(0, 0, 0, 0, 1, 1, 1, 1), // WRITE
154 _BYTE_MASK(0, 0, 0, 0, 0, 0, 1, 1), // EXECUTE WRITE
158 /* We want only the lower access bits */
159 ProtectionMask
&= MM_PROTECT_ACCESS
;
161 /* Look it up in the table */
162 return (AccessAllowedMask
[Write
!= 0][Execute
!= 0] >> ProtectionMask
) & 1;
168 MiAccessCheck(IN PMMPTE PointerPte
,
169 IN BOOLEAN StoreInstruction
,
170 IN KPROCESSOR_MODE PreviousMode
,
171 IN ULONG_PTR ProtectionMask
,
177 /* Check for invalid user-mode access */
178 if ((PreviousMode
== UserMode
) && (PointerPte
> MiHighestUserPte
))
180 return STATUS_ACCESS_VIOLATION
;
183 /* Capture the PTE -- is it valid? */
184 TempPte
= *PointerPte
;
185 if (TempPte
.u
.Hard
.Valid
)
187 /* Was someone trying to write to it? */
188 if (StoreInstruction
)
191 if (MI_IS_PAGE_WRITEABLE(&TempPte
) ||
192 MI_IS_PAGE_COPY_ON_WRITE(&TempPte
))
194 /* Then there's nothing to worry about */
195 return STATUS_SUCCESS
;
198 /* Oops! This isn't allowed */
199 return STATUS_ACCESS_VIOLATION
;
202 /* Someone was trying to read from a valid PTE, that's fine too */
203 return STATUS_SUCCESS
;
206 /* Check if the protection on the page allows what is being attempted */
207 if (!MiIsAccessAllowed(ProtectionMask
, StoreInstruction
, FALSE
))
209 return STATUS_ACCESS_VIOLATION
;
212 /* Check if this is a guard page */
213 if ((ProtectionMask
& MM_PROTECT_SPECIAL
) == MM_GUARDPAGE
)
215 ASSERT(ProtectionMask
!= MM_DECOMMIT
);
217 /* Attached processes can't expand their stack */
218 if (KeIsAttachedProcess()) return STATUS_ACCESS_VIOLATION
;
220 /* No support for prototype PTEs yet */
221 ASSERT(TempPte
.u
.Soft
.Prototype
== 0);
223 /* Remove the guard page bit, and return a guard page violation */
224 TempPte
.u
.Soft
.Protection
= ProtectionMask
& ~MM_GUARDPAGE
;
225 ASSERT(TempPte
.u
.Long
!= 0);
226 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
227 return STATUS_GUARD_PAGE_VIOLATION
;
231 return STATUS_SUCCESS
;
237 MiCheckVirtualAddress(IN PVOID VirtualAddress
,
238 OUT PULONG ProtectCode
,
239 OUT PMMVAD
*ProtoVad
)
244 /* No prototype/section support for now */
247 /* User or kernel fault? */
248 if (VirtualAddress
<= MM_HIGHEST_USER_ADDRESS
)
250 /* Special case for shared data */
251 if (PAGE_ALIGN(VirtualAddress
) == (PVOID
)MM_SHARED_USER_DATA_VA
)
253 /* It's a read-only page */
254 *ProtectCode
= MM_READONLY
;
255 return MmSharedUserDataPte
;
258 /* Find the VAD, it might not exist if the address is bogus */
259 Vad
= MiLocateAddress(VirtualAddress
);
262 /* Bogus virtual address */
263 *ProtectCode
= MM_NOACCESS
;
267 /* ReactOS does not handle physical memory VADs yet */
268 ASSERT(Vad
->u
.VadFlags
.VadType
!= VadDevicePhysicalMemory
);
270 /* Check if it's a section, or just an allocation */
271 if (Vad
->u
.VadFlags
.PrivateMemory
)
273 /* ReactOS does not handle AWE VADs yet */
274 ASSERT(Vad
->u
.VadFlags
.VadType
!= VadAwe
);
276 /* This must be a TEB/PEB VAD */
277 if (Vad
->u
.VadFlags
.MemCommit
)
279 /* It's committed, so return the VAD protection */
280 *ProtectCode
= (ULONG
)Vad
->u
.VadFlags
.Protection
;
284 /* It has not yet been committed, so return no access */
285 *ProtectCode
= MM_NOACCESS
;
288 /* In both cases, return no PTE */
293 /* ReactOS does not supoprt these VADs yet */
294 ASSERT(Vad
->u
.VadFlags
.VadType
!= VadImageMap
);
295 ASSERT(Vad
->u2
.VadFlags2
.ExtendableFile
== 0);
297 /* Return the proto VAD */
300 /* Get the prototype PTE for this page */
301 PointerPte
= (((ULONG_PTR
)VirtualAddress
>> PAGE_SHIFT
) - Vad
->StartingVpn
) + Vad
->FirstPrototypePte
;
302 ASSERT(PointerPte
!= NULL
);
303 ASSERT(PointerPte
<= Vad
->LastContiguousPte
);
305 /* Return the Prototype PTE and the protection for the page mapping */
306 *ProtectCode
= (ULONG
)Vad
->u
.VadFlags
.Protection
;
310 else if (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress
))
312 /* This should never happen, as these addresses are handled by the double-maping */
313 if (((PMMPTE
)VirtualAddress
>= MiAddressToPte(MmPagedPoolStart
)) &&
314 ((PMMPTE
)VirtualAddress
<= MmPagedPoolInfo
.LastPteForPagedPool
))
316 /* Fail such access */
317 *ProtectCode
= MM_NOACCESS
;
321 /* Return full access rights */
322 *ProtectCode
= MM_READWRITE
;
325 else if (MI_IS_SESSION_ADDRESS(VirtualAddress
))
327 /* ReactOS does not have an image list yet, so bail out to failure case */
328 ASSERT(IsListEmpty(&MmSessionSpace
->ImageList
));
331 /* Default case -- failure */
332 *ProtectCode
= MM_NOACCESS
;
336 #if (_MI_PAGING_LEVELS == 2)
340 MiCheckPdeForSessionSpace(IN PVOID Address
)
344 PVOID SessionAddress
;
347 /* Is this a session PTE? */
348 if (MI_IS_SESSION_PTE(Address
))
350 /* Make sure the PDE for session space is valid */
351 PointerPde
= MiAddressToPde(MmSessionSpace
);
352 if (!PointerPde
->u
.Hard
.Valid
)
354 /* This means there's no valid session, bail out */
355 DbgPrint("MiCheckPdeForSessionSpace: No current session for PTE %p\n",
358 return STATUS_ACCESS_VIOLATION
;
361 /* Now get the session-specific page table for this address */
362 SessionAddress
= MiPteToAddress(Address
);
363 PointerPde
= MiAddressToPte(Address
);
364 if (PointerPde
->u
.Hard
.Valid
) return STATUS_WAIT_1
;
366 /* It's not valid, so find it in the page table array */
367 Index
= ((ULONG_PTR
)SessionAddress
- (ULONG_PTR
)MmSessionBase
) >> 22;
368 TempPde
.u
.Long
= MmSessionSpace
->PageTables
[Index
].u
.Long
;
369 if (TempPde
.u
.Hard
.Valid
)
371 /* The copy is valid, so swap it in */
372 InterlockedExchange((PLONG
)PointerPde
, TempPde
.u
.Long
);
373 return STATUS_WAIT_1
;
376 /* We don't seem to have allocated a page table for this address yet? */
377 DbgPrint("MiCheckPdeForSessionSpace: No Session PDE for PTE %p, %p\n",
378 PointerPde
->u
.Long
, SessionAddress
);
380 return STATUS_ACCESS_VIOLATION
;
383 /* Is the address also a session address? If not, we're done */
384 if (!MI_IS_SESSION_ADDRESS(Address
)) return STATUS_SUCCESS
;
386 /* It is, so again get the PDE for session space */
387 PointerPde
= MiAddressToPde(MmSessionSpace
);
388 if (!PointerPde
->u
.Hard
.Valid
)
390 /* This means there's no valid session, bail out */
391 DbgPrint("MiCheckPdeForSessionSpace: No current session for VA %p\n",
394 return STATUS_ACCESS_VIOLATION
;
397 /* Now get the PDE for the address itself */
398 PointerPde
= MiAddressToPde(Address
);
399 if (!PointerPde
->u
.Hard
.Valid
)
401 /* Do the swap, we should be good to go */
402 Index
= ((ULONG_PTR
)Address
- (ULONG_PTR
)MmSessionBase
) >> 22;
403 PointerPde
->u
.Long
= MmSessionSpace
->PageTables
[Index
].u
.Long
;
404 if (PointerPde
->u
.Hard
.Valid
) return STATUS_WAIT_1
;
406 /* We had not allocated a page table for this session address yet, fail! */
407 DbgPrint("MiCheckPdeForSessionSpace: No Session PDE for VA %p, %p\n",
408 PointerPde
->u
.Long
, Address
);
410 return STATUS_ACCESS_VIOLATION
;
413 /* It's valid, so there's nothing to do */
414 return STATUS_SUCCESS
;
419 MiCheckPdeForPagedPool(IN PVOID Address
)
422 NTSTATUS Status
= STATUS_SUCCESS
;
424 /* Check session PDE */
425 if (MI_IS_SESSION_ADDRESS(Address
)) return MiCheckPdeForSessionSpace(Address
);
426 if (MI_IS_SESSION_PTE(Address
)) return MiCheckPdeForSessionSpace(Address
);
429 // Check if this is a fault while trying to access the page table itself
431 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address
))
434 // Send a hint to the page fault handler that this is only a valid fault
435 // if we already detected this was access within the page table range
437 PointerPde
= (PMMPDE
)MiAddressToPte(Address
);
438 Status
= STATUS_WAIT_1
;
440 else if (Address
< MmSystemRangeStart
)
443 // This is totally illegal
445 return STATUS_ACCESS_VIOLATION
;
450 // Get the PDE for the address
452 PointerPde
= MiAddressToPde(Address
);
456 // Check if it's not valid
458 if (PointerPde
->u
.Hard
.Valid
== 0)
461 // Copy it from our double-mapped system page directory
463 InterlockedExchangePte(PointerPde
,
464 MmSystemPagePtes
[((ULONG_PTR
)PointerPde
& (SYSTEM_PD_SIZE
- 1)) / sizeof(MMPTE
)].u
.Long
);
475 MiCheckPdeForPagedPool(IN PVOID Address
)
477 return STATUS_ACCESS_VIOLATION
;
483 MiZeroPfn(IN PFN_NUMBER PageFrameNumber
)
490 /* Get the PFN for this page */
491 Pfn1
= MiGetPfnEntry(PageFrameNumber
);
494 /* Grab a system PTE we can use to zero the page */
495 ZeroPte
= MiReserveSystemPtes(1, SystemPteSpace
);
498 /* Initialize the PTE for it */
499 TempPte
= ValidKernelPte
;
500 TempPte
.u
.Hard
.PageFrameNumber
= PageFrameNumber
;
503 if (Pfn1
->u3
.e1
.CacheAttribute
== MiWriteCombined
)
505 /* Write combining, no caching */
506 MI_PAGE_DISABLE_CACHE(&TempPte
);
507 MI_PAGE_WRITE_COMBINED(&TempPte
);
509 else if (Pfn1
->u3
.e1
.CacheAttribute
== MiNonCached
)
511 /* Write through, no caching */
512 MI_PAGE_DISABLE_CACHE(&TempPte
);
513 MI_PAGE_WRITE_THROUGH(&TempPte
);
516 /* Make the system PTE valid with our PFN */
517 MI_WRITE_VALID_PTE(ZeroPte
, TempPte
);
519 /* Get the address it maps to, and zero it out */
520 ZeroAddress
= MiPteToAddress(ZeroPte
);
521 KeZeroPages(ZeroAddress
, PAGE_SIZE
);
523 /* Now get rid of it */
524 MiReleaseSystemPtes(ZeroPte
, 1, SystemPteSpace
);
530 _In_ PFN_NUMBER DestPage
,
531 _In_ PFN_NUMBER SrcPage
)
535 PMMPFN DestPfn
, SrcPfn
;
537 const VOID
* SrcAddress
;
540 DestPfn
= MiGetPfnEntry(DestPage
);
542 SrcPfn
= MiGetPfnEntry(SrcPage
);
545 /* Grab 2 system PTEs */
546 SysPtes
= MiReserveSystemPtes(2, SystemPteSpace
);
549 /* Initialize the destination PTE */
550 TempPte
= ValidKernelPte
;
551 TempPte
.u
.Hard
.PageFrameNumber
= DestPage
;
554 if (DestPfn
->u3
.e1
.CacheAttribute
== MiWriteCombined
)
556 /* Write combining, no caching */
557 MI_PAGE_DISABLE_CACHE(&TempPte
);
558 MI_PAGE_WRITE_COMBINED(&TempPte
);
560 else if (DestPfn
->u3
.e1
.CacheAttribute
== MiNonCached
)
562 /* Write through, no caching */
563 MI_PAGE_DISABLE_CACHE(&TempPte
);
564 MI_PAGE_WRITE_THROUGH(&TempPte
);
567 /* Make the system PTE valid with our PFN */
568 MI_WRITE_VALID_PTE(&SysPtes
[0], TempPte
);
570 /* Initialize the source PTE */
571 TempPte
= ValidKernelPte
;
572 TempPte
.u
.Hard
.PageFrameNumber
= SrcPage
;
575 if (SrcPfn
->u3
.e1
.CacheAttribute
== MiNonCached
)
577 MI_PAGE_DISABLE_CACHE(&TempPte
);
580 /* Make the system PTE valid with our PFN */
581 MI_WRITE_VALID_PTE(&SysPtes
[1], TempPte
);
583 /* Get the addresses and perform the copy */
584 DestAddress
= MiPteToAddress(&SysPtes
[0]);
585 SrcAddress
= MiPteToAddress(&SysPtes
[1]);
586 RtlCopyMemory(DestAddress
, SrcAddress
, PAGE_SIZE
);
588 /* Now get rid of it */
589 MiReleaseSystemPtes(SysPtes
, 2, SystemPteSpace
);
595 MiResolveDemandZeroFault(IN PVOID Address
,
596 IN PMMPTE PointerPte
,
598 IN PEPROCESS Process
,
601 PFN_NUMBER PageFrameNumber
= 0;
603 BOOLEAN NeedZero
= FALSE
, HaveLock
= FALSE
;
606 DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
610 /* Must currently only be called by paging path */
611 if ((Process
> HYDRA_PROCESS
) && (OldIrql
== MM_NOIRQL
))
614 ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte
));
617 ASSERT(Process
->ForkInProgress
== NULL
);
619 /* Get process color */
620 Color
= MI_GET_NEXT_PROCESS_COLOR(Process
);
621 ASSERT(Color
!= 0xFFFFFFFF);
623 /* We'll need a zero page */
628 /* Check if we need a zero page */
629 NeedZero
= (OldIrql
!= MM_NOIRQL
);
631 /* Session-backed image views must be zeroed */
632 if ((Process
== HYDRA_PROCESS
) &&
633 ((MI_IS_SESSION_IMAGE_ADDRESS(Address
)) ||
634 ((Address
>= MiSessionViewStart
) && (Address
< MiSessionSpaceWs
))))
639 /* Hardcode unknown color */
643 /* Check if the PFN database should be acquired */
644 if (OldIrql
== MM_NOIRQL
)
646 /* Acquire it and remember we should release it after */
647 OldIrql
= MiAcquirePfnLock();
651 /* We either manually locked the PFN DB, or already came with it locked */
652 MI_ASSERT_PFN_LOCK_HELD();
653 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
655 /* Assert we have enough pages */
656 ASSERT(MmAvailablePages
>= 32);
659 if (UserPdeFault
) MI_SET_USAGE(MI_USAGE_PAGE_TABLE
);
660 if (!UserPdeFault
) MI_SET_USAGE(MI_USAGE_DEMAND_ZERO
);
662 if (Process
== HYDRA_PROCESS
) MI_SET_PROCESS2("Hydra");
663 else if (Process
) MI_SET_PROCESS2(Process
->ImageFileName
);
664 else MI_SET_PROCESS2("Kernel Demand 0");
666 /* Do we need a zero page? */
667 if (Color
!= 0xFFFFFFFF)
669 /* Try to get one, if we couldn't grab a free page and zero it */
670 PageFrameNumber
= MiRemoveZeroPageSafe(Color
);
671 if (!PageFrameNumber
)
673 /* We'll need a free page and zero it manually */
674 PageFrameNumber
= MiRemoveAnyPage(Color
);
680 /* Get a color, and see if we should grab a zero or non-zero page */
681 Color
= MI_GET_NEXT_COLOR();
684 /* Process or system doesn't want a zero page, grab anything */
685 PageFrameNumber
= MiRemoveAnyPage(Color
);
689 /* System wants a zero page, obtain one */
690 PageFrameNumber
= MiRemoveZeroPage(Color
);
695 MiInitializePfn(PageFrameNumber
, PointerPte
, TRUE
);
697 /* Increment demand zero faults */
698 KeGetCurrentPrcb()->MmDemandZeroCount
++;
700 /* Do we have the lock? */
704 MiReleasePfnLock(OldIrql
);
706 /* Update performance counters */
707 if (Process
> HYDRA_PROCESS
) Process
->NumberOfPrivatePages
++;
710 /* Zero the page if need be */
711 if (NeedZero
) MiZeroPfn(PageFrameNumber
);
713 /* Fault on user PDE, or fault on user PTE? */
714 if (PointerPte
<= MiHighestUserPte
)
716 /* User fault, build a user PTE */
717 MI_MAKE_HARDWARE_PTE_USER(&TempPte
,
724 /* This is a user-mode PDE, create a kernel PTE for it */
725 MI_MAKE_HARDWARE_PTE(&TempPte
,
731 /* Set it dirty if it's a writable page */
732 if (MI_IS_PAGE_WRITEABLE(&TempPte
)) MI_MAKE_DIRTY_PAGE(&TempPte
);
735 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
737 /* Did we manually acquire the lock */
740 /* Get the PFN entry */
741 Pfn1
= MI_PFN_ELEMENT(PageFrameNumber
);
743 /* Windows does these sanity checks */
744 ASSERT(Pfn1
->u1
.Event
== 0);
745 ASSERT(Pfn1
->u3
.e1
.PrototypePte
== 0);
751 DPRINT("Demand zero page has now been paged in\n");
752 return STATUS_PAGE_FAULT_DEMAND_ZERO
;
758 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction
,
760 IN PMMPTE PointerPte
,
761 IN PMMPTE PointerProtoPte
,
763 IN PMMPFN
* LockedProtoPfn
)
766 PMMPTE OriginalPte
, PageTablePte
;
767 ULONG_PTR Protection
;
768 PFN_NUMBER PageFrameIndex
;
770 BOOLEAN OriginalProtection
, DirtyPage
;
772 /* Must be called with an valid prototype PTE, with the PFN lock held */
773 MI_ASSERT_PFN_LOCK_HELD();
774 ASSERT(PointerProtoPte
->u
.Hard
.Valid
== 1);
777 PageFrameIndex
= PFN_FROM_PTE(PointerProtoPte
);
779 /* Get the PFN entry and set it as a prototype PTE */
780 Pfn1
= MiGetPfnEntry(PageFrameIndex
);
781 Pfn1
->u3
.e1
.PrototypePte
= 1;
783 /* Increment the share count for the page table */
784 PageTablePte
= MiAddressToPte(PointerPte
);
785 Pfn2
= MiGetPfnEntry(PageTablePte
->u
.Hard
.PageFrameNumber
);
786 Pfn2
->u2
.ShareCount
++;
788 /* Check where we should be getting the protection information from */
789 if (PointerPte
->u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
)
791 /* Get the protection from the PTE, there's no real Proto PTE data */
792 Protection
= PointerPte
->u
.Soft
.Protection
;
794 /* Remember that we did not use the proto protection */
795 OriginalProtection
= FALSE
;
799 /* Get the protection from the original PTE link */
800 OriginalPte
= &Pfn1
->OriginalPte
;
801 Protection
= OriginalPte
->u
.Soft
.Protection
;
803 /* Remember that we used the original protection */
804 OriginalProtection
= TRUE
;
806 /* Check if this was a write on a read only proto */
807 if ((StoreInstruction
) && !(Protection
& MM_READWRITE
))
810 StoreInstruction
= 0;
814 /* Check if this was a write on a non-COW page */
816 if ((StoreInstruction
) && ((Protection
& MM_WRITECOPY
) != MM_WRITECOPY
))
818 /* Then the page should be marked dirty */
822 ASSERT(Pfn1
->OriginalPte
.u
.Soft
.Prototype
!= 0);
825 /* Did we get a locked incoming PFN? */
828 /* Drop a reference */
829 ASSERT((*LockedProtoPfn
)->u3
.e2
.ReferenceCount
>= 1);
830 MiDereferencePfnAndDropLockCount(*LockedProtoPfn
);
831 *LockedProtoPfn
= NULL
;
834 /* Release the PFN lock */
835 MiReleasePfnLock(OldIrql
);
837 /* Remove special/caching bits */
838 Protection
&= ~MM_PROTECT_SPECIAL
;
841 if (Pfn1
->u3
.e1
.CacheAttribute
== MiWriteCombined
)
843 /* Write combining, no caching */
844 MI_PAGE_DISABLE_CACHE(&TempPte
);
845 MI_PAGE_WRITE_COMBINED(&TempPte
);
847 else if (Pfn1
->u3
.e1
.CacheAttribute
== MiNonCached
)
849 /* Write through, no caching */
850 MI_PAGE_DISABLE_CACHE(&TempPte
);
851 MI_PAGE_WRITE_THROUGH(&TempPte
);
854 /* Check if this is a kernel or user address */
855 if (Address
< MmSystemRangeStart
)
857 /* Build the user PTE */
858 MI_MAKE_HARDWARE_PTE_USER(&TempPte
, PointerPte
, Protection
, PageFrameIndex
);
862 /* Build the kernel PTE */
863 MI_MAKE_HARDWARE_PTE(&TempPte
, PointerPte
, Protection
, PageFrameIndex
);
866 /* Set the dirty flag if needed */
867 if (DirtyPage
) MI_MAKE_DIRTY_PAGE(&TempPte
);
870 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
872 /* Reset the protection if needed */
873 if (OriginalProtection
) Protection
= MM_ZERO_ACCESS
;
876 ASSERT(PointerPte
== MiAddressToPte(Address
));
877 return STATUS_SUCCESS
;
883 MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction
,
884 _In_ PVOID FaultingAddress
,
885 _In_ PMMPTE PointerPte
,
886 _In_ PEPROCESS CurrentProcess
,
887 _Inout_ KIRQL
*OldIrql
)
892 MMPTE TempPte
= *PointerPte
;
894 ULONG PageFileIndex
= TempPte
.u
.Soft
.PageFileLow
;
895 ULONG_PTR PageFileOffset
= TempPte
.u
.Soft
.PageFileHigh
;
896 ULONG Protection
= TempPte
.u
.Soft
.Protection
;
898 /* Things we don't support yet */
899 ASSERT(CurrentProcess
> HYDRA_PROCESS
);
900 ASSERT(*OldIrql
!= MM_NOIRQL
);
902 /* We must hold the PFN lock */
903 MI_ASSERT_PFN_LOCK_HELD();
905 /* Some sanity checks */
906 ASSERT(TempPte
.u
.Hard
.Valid
== 0);
907 ASSERT(TempPte
.u
.Soft
.PageFileHigh
!= 0);
908 ASSERT(TempPte
.u
.Soft
.PageFileHigh
!= MI_PTE_LOOKUP_NEEDED
);
910 /* Get any page, it will be overwritten */
911 Color
= MI_GET_NEXT_PROCESS_COLOR(CurrentProcess
);
912 Page
= MiRemoveAnyPage(Color
);
914 /* Initialize this PFN */
915 MiInitializePfn(Page
, PointerPte
, StoreInstruction
);
917 /* Sets the PFN as being in IO operation */
918 Pfn1
= MI_PFN_ELEMENT(Page
);
919 ASSERT(Pfn1
->u1
.Event
== NULL
);
920 ASSERT(Pfn1
->u3
.e1
.ReadInProgress
== 0);
921 ASSERT(Pfn1
->u3
.e1
.WriteInProgress
== 0);
922 Pfn1
->u3
.e1
.ReadInProgress
= 1;
924 /* We must write the PTE now as the PFN lock will be released while performing the IO operation */
925 MI_MAKE_TRANSITION_PTE(&TempPte
, Page
, Protection
);
927 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
929 /* Release the PFN lock while we proceed */
930 MiReleasePfnLock(*OldIrql
);
932 /* Do the paging IO */
933 Status
= MiReadPageFile(Page
, PageFileIndex
, PageFileOffset
);
935 /* Lock the PFN database again */
936 *OldIrql
= MiAcquirePfnLock();
938 /* Nobody should have changed that while we were not looking */
939 ASSERT(Pfn1
->u3
.e1
.ReadInProgress
== 1);
940 ASSERT(Pfn1
->u3
.e1
.WriteInProgress
== 0);
942 if (!NT_SUCCESS(Status
))
946 Pfn1
->u4
.InPageError
= 1;
947 Pfn1
->u1
.ReadStatus
= Status
;
950 /* And the PTE can finally be valid */
951 MI_MAKE_HARDWARE_PTE(&TempPte
, PointerPte
, Protection
, Page
);
952 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
954 Pfn1
->u3
.e1
.ReadInProgress
= 0;
955 /* Did someone start to wait on us while we proceeded ? */
958 /* Tell them we're done */
959 KeSetEvent(Pfn1
->u1
.Event
, IO_NO_INCREMENT
, FALSE
);
968 MiResolveTransitionFault(IN BOOLEAN StoreInstruction
,
969 IN PVOID FaultingAddress
,
970 IN PMMPTE PointerPte
,
971 IN PEPROCESS CurrentProcess
,
973 OUT PKEVENT
**InPageBlock
)
975 PFN_NUMBER PageFrameIndex
;
978 PMMPTE PointerToPteForProtoPage
;
979 DPRINT("Transition fault on 0x%p with PTE 0x%p in process %s\n",
980 FaultingAddress
, PointerPte
, CurrentProcess
->ImageFileName
);
982 /* Windowss does this check */
983 ASSERT(*InPageBlock
== NULL
);
985 /* ARM3 doesn't support this path */
986 ASSERT(OldIrql
!= MM_NOIRQL
);
988 /* Capture the PTE and make sure it's in transition format */
989 TempPte
= *PointerPte
;
990 ASSERT((TempPte
.u
.Soft
.Valid
== 0) &&
991 (TempPte
.u
.Soft
.Prototype
== 0) &&
992 (TempPte
.u
.Soft
.Transition
== 1));
994 /* Get the PFN and the PFN entry */
995 PageFrameIndex
= TempPte
.u
.Trans
.PageFrameNumber
;
996 DPRINT("Transition PFN: %lx\n", PageFrameIndex
);
997 Pfn1
= MiGetPfnEntry(PageFrameIndex
);
999 /* One more transition fault! */
1000 InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount
);
1002 /* This is from ARM3 -- Windows normally handles this here */
1003 ASSERT(Pfn1
->u4
.InPageError
== 0);
1005 /* See if we should wait before terminating the fault */
1006 if ((Pfn1
->u3
.e1
.ReadInProgress
== 1)
1007 || ((Pfn1
->u3
.e1
.WriteInProgress
== 1) && StoreInstruction
))
1009 DPRINT1("The page is currently in a page transition !\n");
1010 *InPageBlock
= &Pfn1
->u1
.Event
;
1011 if (PointerPte
== Pfn1
->PteAddress
)
1013 DPRINT1("And this if for this particular PTE.\n");
1014 /* The PTE will be made valid by the thread serving the fault */
1015 return STATUS_SUCCESS
; // FIXME: Maybe something more descriptive
1019 /* Windows checks there's some free pages and this isn't an in-page error */
1020 ASSERT(MmAvailablePages
> 0);
1021 ASSERT(Pfn1
->u4
.InPageError
== 0);
1023 /* ReactOS checks for this */
1024 ASSERT(MmAvailablePages
> 32);
1026 /* Was this a transition page in the valid list, or free/zero list? */
1027 if (Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
)
1029 /* All Windows does here is a bunch of sanity checks */
1030 DPRINT("Transition in active list\n");
1031 ASSERT((Pfn1
->PteAddress
>= MiAddressToPte(MmPagedPoolStart
)) &&
1032 (Pfn1
->PteAddress
<= MiAddressToPte(MmPagedPoolEnd
)));
1033 ASSERT(Pfn1
->u2
.ShareCount
!= 0);
1034 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
!= 0);
1038 /* Otherwise, the page is removed from its list */
1039 DPRINT("Transition page in free/zero list\n");
1040 MiUnlinkPageFromList(Pfn1
);
1041 MiReferenceUnusedPageAndBumpLockCount(Pfn1
);
1044 /* At this point, there should no longer be any in-page errors */
1045 ASSERT(Pfn1
->u4
.InPageError
== 0);
1047 /* Check if this was a PFN with no more share references */
1048 if (Pfn1
->u2
.ShareCount
== 0) MiDropLockCount(Pfn1
);
1050 /* Bump the share count and make the page valid */
1051 Pfn1
->u2
.ShareCount
++;
1052 Pfn1
->u3
.e1
.PageLocation
= ActiveAndValid
;
1054 /* Prototype PTEs are in paged pool, which itself might be in transition */
1055 if (FaultingAddress
>= MmSystemRangeStart
)
1057 /* Check if this is a paged pool PTE in transition state */
1058 PointerToPteForProtoPage
= MiAddressToPte(PointerPte
);
1059 TempPte
= *PointerToPteForProtoPage
;
1060 if ((TempPte
.u
.Hard
.Valid
== 0) && (TempPte
.u
.Soft
.Transition
== 1))
1062 /* This isn't yet supported */
1063 DPRINT1("Double transition fault not yet supported\n");
1068 /* Build the final PTE */
1069 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1070 ASSERT(PointerPte
->u
.Trans
.Prototype
== 0);
1071 ASSERT(PointerPte
->u
.Trans
.Transition
== 1);
1072 TempPte
.u
.Long
= (PointerPte
->u
.Long
& ~0xFFF) |
1073 (MmProtectToPteMask
[PointerPte
->u
.Trans
.Protection
]) |
1074 MiDetermineUserGlobalPteMask(PointerPte
);
1076 /* Is the PTE writeable? */
1077 if ((Pfn1
->u3
.e1
.Modified
) &&
1078 MI_IS_PAGE_WRITEABLE(&TempPte
) &&
1079 !MI_IS_PAGE_COPY_ON_WRITE(&TempPte
))
1082 MI_MAKE_DIRTY_PAGE(&TempPte
);
1087 MI_MAKE_CLEAN_PAGE(&TempPte
);
1090 /* Write the valid PTE */
1091 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
1093 /* Return success */
1094 return STATUS_PAGE_FAULT_TRANSITION
;
1100 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction
,
1102 IN PMMPTE PointerPte
,
1103 IN PMMPTE PointerProtoPte
,
1104 IN OUT PMMPFN
*OutPfn
,
1105 OUT PVOID
*PageFileData
,
1106 OUT PMMPTE PteValue
,
1107 IN PEPROCESS Process
,
1109 IN PVOID TrapInformation
)
1111 MMPTE TempPte
, PteContents
;
1113 PFN_NUMBER PageFrameIndex
;
1115 PKEVENT
* InPageBlock
= NULL
;
1118 /* Must be called with an invalid, prototype PTE, with the PFN lock held */
1119 MI_ASSERT_PFN_LOCK_HELD();
1120 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1121 ASSERT(PointerPte
->u
.Soft
.Prototype
== 1);
1123 /* Read the prototype PTE and check if it's valid */
1124 TempPte
= *PointerProtoPte
;
1125 if (TempPte
.u
.Hard
.Valid
== 1)
1127 /* One more user of this mapped page */
1128 PageFrameIndex
= PFN_FROM_PTE(&TempPte
);
1129 Pfn1
= MiGetPfnEntry(PageFrameIndex
);
1130 Pfn1
->u2
.ShareCount
++;
1132 /* Call it a transition */
1133 InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount
);
1135 /* Complete the prototype PTE fault -- this will release the PFN lock */
1136 return MiCompleteProtoPteFault(StoreInstruction
,
1144 /* Make sure there's some protection mask */
1145 if (TempPte
.u
.Long
== 0)
1147 /* Release the lock */
1148 DPRINT1("Access on reserved section?\n");
1149 MiReleasePfnLock(OldIrql
);
1150 return STATUS_ACCESS_VIOLATION
;
1153 /* There is no such thing as a decommitted prototype PTE */
1154 ASSERT(TempPte
.u
.Long
!= MmDecommittedPte
.u
.Long
);
1156 /* Check for access rights on the PTE proper */
1157 PteContents
= *PointerPte
;
1158 if (PteContents
.u
.Soft
.PageFileHigh
!= MI_PTE_LOOKUP_NEEDED
)
1160 if (!PteContents
.u
.Proto
.ReadOnly
)
1162 Protection
= TempPte
.u
.Soft
.Protection
;
1166 Protection
= MM_READONLY
;
1168 /* Check for page acess in software */
1169 Status
= MiAccessCheck(PointerProtoPte
,
1172 TempPte
.u
.Soft
.Protection
,
1175 ASSERT(Status
== STATUS_SUCCESS
);
1179 Protection
= PteContents
.u
.Soft
.Protection
;
1182 /* Check for writing copy on write page */
1183 if (((Protection
& MM_WRITECOPY
) == MM_WRITECOPY
) && StoreInstruction
)
1185 PFN_NUMBER PageFrameIndex
, ProtoPageFrameIndex
;
1188 /* Resolve the proto fault as if it was a read operation */
1189 Status
= MiResolveProtoPteFault(FALSE
,
1200 if (!NT_SUCCESS(Status
))
1205 /* Lock again the PFN lock, MiResolveProtoPteFault unlocked it */
1206 OldIrql
= MiAcquirePfnLock();
1208 /* And re-read the proto PTE */
1209 TempPte
= *PointerProtoPte
;
1210 ASSERT(TempPte
.u
.Hard
.Valid
== 1);
1211 ProtoPageFrameIndex
= PFN_FROM_PTE(&TempPte
);
1213 /* Get a new page for the private copy */
1214 if (Process
> HYDRA_PROCESS
)
1215 Color
= MI_GET_NEXT_PROCESS_COLOR(Process
);
1217 Color
= MI_GET_NEXT_COLOR();
1219 PageFrameIndex
= MiRemoveAnyPage(Color
);
1221 /* Perform the copy */
1222 MiCopyPfn(PageFrameIndex
, ProtoPageFrameIndex
);
1224 /* This will drop everything MiResolveProtoPteFault referenced */
1225 MiDeletePte(PointerPte
, Address
, Process
, PointerProtoPte
);
1227 /* Because now we use this */
1228 Pfn1
= MI_PFN_ELEMENT(PageFrameIndex
);
1229 MiInitializePfn(PageFrameIndex
, PointerPte
, TRUE
);
1231 /* Fix the protection */
1232 Protection
&= ~MM_WRITECOPY
;
1233 Protection
|= MM_READWRITE
;
1234 if (Address
< MmSystemRangeStart
)
1236 /* Build the user PTE */
1237 MI_MAKE_HARDWARE_PTE_USER(&PteContents
, PointerPte
, Protection
, PageFrameIndex
);
1241 /* Build the kernel PTE */
1242 MI_MAKE_HARDWARE_PTE(&PteContents
, PointerPte
, Protection
, PageFrameIndex
);
1245 /* And finally, write the valid PTE */
1246 MI_WRITE_VALID_PTE(PointerPte
, PteContents
);
1248 /* The caller expects us to release the PFN lock */
1249 MiReleasePfnLock(OldIrql
);
1253 /* Check for clone PTEs */
1254 if (PointerPte
<= MiHighestUserPte
) ASSERT(Process
->CloneRoot
== NULL
);
1256 /* We don't support mapped files yet */
1257 ASSERT(TempPte
.u
.Soft
.Prototype
== 0);
1259 /* We might however have transition PTEs */
1260 if (TempPte
.u
.Soft
.Transition
== 1)
1262 /* Resolve the transition fault */
1263 ASSERT(OldIrql
!= MM_NOIRQL
);
1264 Status
= MiResolveTransitionFault(StoreInstruction
,
1270 ASSERT(NT_SUCCESS(Status
));
1274 /* We also don't support paged out pages */
1275 ASSERT(TempPte
.u
.Soft
.PageFileHigh
== 0);
1277 /* Resolve the demand zero fault */
1278 Status
= MiResolveDemandZeroFault(Address
,
1280 (ULONG
)TempPte
.u
.Soft
.Protection
,
1283 ASSERT(NT_SUCCESS(Status
));
1286 /* Complete the prototype PTE fault -- this will release the PFN lock */
1287 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1288 return MiCompleteProtoPteFault(StoreInstruction
,
1298 MiDispatchFault(IN ULONG FaultCode
,
1300 IN PMMPTE PointerPte
,
1301 IN PMMPTE PointerProtoPte
,
1302 IN BOOLEAN Recursive
,
1303 IN PEPROCESS Process
,
1304 IN PVOID TrapInformation
,
1308 KIRQL OldIrql
, LockIrql
;
1310 PMMPTE SuperProtoPte
;
1311 PMMPFN Pfn1
, OutPfn
= NULL
;
1312 PFN_NUMBER PageFrameIndex
;
1313 PFN_COUNT PteCount
, ProcessedPtes
;
1314 DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
1318 /* Make sure the addresses are ok */
1319 ASSERT(PointerPte
== MiAddressToPte(Address
));
1322 // Make sure APCs are off and we're not at dispatch
1324 OldIrql
= KeGetCurrentIrql();
1325 ASSERT(OldIrql
<= APC_LEVEL
);
1326 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1329 // Grab a copy of the PTE
1331 TempPte
= *PointerPte
;
1333 /* Do we have a prototype PTE? */
1334 if (PointerProtoPte
)
1336 /* This should never happen */
1337 ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte
));
1339 /* Check if this is a kernel-mode address */
1340 SuperProtoPte
= MiAddressToPte(PointerProtoPte
);
1341 if (Address
>= MmSystemRangeStart
)
1343 /* Lock the PFN database */
1344 LockIrql
= MiAcquirePfnLock();
1346 /* Has the PTE been made valid yet? */
1347 if (!SuperProtoPte
->u
.Hard
.Valid
)
1351 else if (PointerPte
->u
.Hard
.Valid
== 1)
1356 /* Resolve the fault -- this will release the PFN lock */
1357 Status
= MiResolveProtoPteFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode
),
1367 ASSERT(Status
== STATUS_SUCCESS
);
1369 /* Complete this as a transition fault */
1370 ASSERT(OldIrql
== KeGetCurrentIrql());
1371 ASSERT(OldIrql
<= APC_LEVEL
);
1372 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1377 /* We only handle the lookup path */
1378 ASSERT(PointerPte
->u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
);
1380 /* Is there a non-image VAD? */
1382 (Vad
->u
.VadFlags
.VadType
!= VadImageMap
) &&
1383 !(Vad
->u2
.VadFlags2
.ExtendableFile
))
1385 /* One day, ReactOS will cluster faults */
1386 ASSERT(Address
<= MM_HIGHEST_USER_ADDRESS
);
1387 DPRINT("Should cluster fault, but won't\n");
1390 /* Only one PTE to handle for now */
1394 /* Lock the PFN database */
1395 LockIrql
= MiAcquirePfnLock();
1397 /* We only handle the valid path */
1398 ASSERT(SuperProtoPte
->u
.Hard
.Valid
== 1);
1400 /* Capture the PTE */
1401 TempPte
= *PointerProtoPte
;
1403 /* Loop to handle future case of clustered faults */
1406 /* For our current usage, this should be true */
1407 if (TempPte
.u
.Hard
.Valid
== 1)
1409 /* Bump the share count on the PTE */
1410 PageFrameIndex
= PFN_FROM_PTE(&TempPte
);
1411 Pfn1
= MI_PFN_ELEMENT(PageFrameIndex
);
1412 Pfn1
->u2
.ShareCount
++;
1414 else if ((TempPte
.u
.Soft
.Prototype
== 0) &&
1415 (TempPte
.u
.Soft
.Transition
== 1))
1417 /* This is a standby page, bring it back from the cache */
1418 PageFrameIndex
= TempPte
.u
.Trans
.PageFrameNumber
;
1419 DPRINT("oooh, shiny, a soft fault! 0x%lx\n", PageFrameIndex
);
1420 Pfn1
= MI_PFN_ELEMENT(PageFrameIndex
);
1421 ASSERT(Pfn1
->u3
.e1
.PageLocation
!= ActiveAndValid
);
1423 /* Should not yet happen in ReactOS */
1424 ASSERT(Pfn1
->u3
.e1
.ReadInProgress
== 0);
1425 ASSERT(Pfn1
->u4
.InPageError
== 0);
1428 MiUnlinkPageFromList(Pfn1
);
1430 /* Bump its reference count */
1431 ASSERT(Pfn1
->u2
.ShareCount
== 0);
1432 InterlockedIncrement16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
);
1433 Pfn1
->u2
.ShareCount
++;
1435 /* Make it valid again */
1436 /* This looks like another macro.... */
1437 Pfn1
->u3
.e1
.PageLocation
= ActiveAndValid
;
1438 ASSERT(PointerProtoPte
->u
.Hard
.Valid
== 0);
1439 ASSERT(PointerProtoPte
->u
.Trans
.Prototype
== 0);
1440 ASSERT(PointerProtoPte
->u
.Trans
.Transition
== 1);
1441 TempPte
.u
.Long
= (PointerProtoPte
->u
.Long
& ~0xFFF) |
1442 MmProtectToPteMask
[PointerProtoPte
->u
.Trans
.Protection
];
1443 TempPte
.u
.Hard
.Valid
= 1;
1444 MI_MAKE_ACCESSED_PAGE(&TempPte
);
1446 /* Is the PTE writeable? */
1447 if ((Pfn1
->u3
.e1
.Modified
) &&
1448 MI_IS_PAGE_WRITEABLE(&TempPte
) &&
1449 !MI_IS_PAGE_COPY_ON_WRITE(&TempPte
))
1452 MI_MAKE_DIRTY_PAGE(&TempPte
);
1457 MI_MAKE_CLEAN_PAGE(&TempPte
);
1460 /* Write the valid PTE */
1461 MI_WRITE_VALID_PTE(PointerProtoPte
, TempPte
);
1462 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1466 /* Page is invalid, get out of the loop */
1470 /* One more done, was it the last? */
1471 if (++ProcessedPtes
== PteCount
)
1473 /* Complete the fault */
1474 MiCompleteProtoPteFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode
),
1481 /* THIS RELEASES THE PFN LOCK! */
1485 /* No clustered faults yet */
1489 /* Did we resolve the fault? */
1492 /* Bump the transition count */
1493 InterlockedExchangeAddSizeT(&KeGetCurrentPrcb()->MmTransitionCount
, ProcessedPtes
);
1496 /* Loop all the processing we did */
1497 ASSERT(ProcessedPtes
== 0);
1499 /* Complete this as a transition fault */
1500 ASSERT(OldIrql
== KeGetCurrentIrql());
1501 ASSERT(OldIrql
<= APC_LEVEL
);
1502 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1503 return STATUS_PAGE_FAULT_TRANSITION
;
1506 /* We did not -- PFN lock is still held, prepare to resolve prototype PTE fault */
1507 OutPfn
= MI_PFN_ELEMENT(SuperProtoPte
->u
.Hard
.PageFrameNumber
);
1508 MiReferenceUsedPageAndBumpLockCount(OutPfn
);
1509 ASSERT(OutPfn
->u3
.e2
.ReferenceCount
> 1);
1510 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1512 /* Resolve the fault -- this will release the PFN lock */
1513 Status
= MiResolveProtoPteFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode
),
1523 //ASSERT(Status != STATUS_ISSUE_PAGING_IO);
1524 //ASSERT(Status != STATUS_REFAULT);
1525 //ASSERT(Status != STATUS_PTE_CHANGED);
1527 /* Did the routine clean out the PFN or should we? */
1530 /* We had a locked PFN, so acquire the PFN lock to dereference it */
1531 ASSERT(PointerProtoPte
!= NULL
);
1532 OldIrql
= MiAcquirePfnLock();
1534 /* Dereference the locked PFN */
1535 MiDereferencePfnAndDropLockCount(OutPfn
);
1536 ASSERT(OutPfn
->u3
.e2
.ReferenceCount
>= 1);
1538 /* And now release the lock */
1539 MiReleasePfnLock(OldIrql
);
1542 /* Complete this as a transition fault */
1543 ASSERT(OldIrql
== KeGetCurrentIrql());
1544 ASSERT(OldIrql
<= APC_LEVEL
);
1545 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1550 /* Is this a transition PTE */
1551 if (TempPte
.u
.Soft
.Transition
)
1553 PKEVENT
* InPageBlock
= NULL
;
1554 PKEVENT PreviousPageEvent
;
1555 KEVENT CurrentPageEvent
;
1557 /* Lock the PFN database */
1558 LockIrql
= MiAcquirePfnLock();
1561 Status
= MiResolveTransitionFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode
), Address
, PointerPte
, Process
, LockIrql
, &InPageBlock
);
1563 ASSERT(NT_SUCCESS(Status
));
1565 if (InPageBlock
!= NULL
)
1567 /* Another thread is reading or writing this page. Put us into the waiting queue. */
1568 KeInitializeEvent(&CurrentPageEvent
, NotificationEvent
, FALSE
);
1569 PreviousPageEvent
= *InPageBlock
;
1570 *InPageBlock
= &CurrentPageEvent
;
1573 /* And now release the lock and leave*/
1574 MiReleasePfnLock(LockIrql
);
1576 if (InPageBlock
!= NULL
)
1578 KeWaitForSingleObject(&CurrentPageEvent
, WrPageIn
, KernelMode
, FALSE
, NULL
);
1580 /* Let's the chain go on */
1581 if (PreviousPageEvent
)
1583 KeSetEvent(PreviousPageEvent
, IO_NO_INCREMENT
, FALSE
);
1587 ASSERT(OldIrql
== KeGetCurrentIrql());
1588 ASSERT(OldIrql
<= APC_LEVEL
);
1589 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1593 /* Should we page the data back in ? */
1594 if (TempPte
.u
.Soft
.PageFileHigh
!= 0)
1596 /* Lock the PFN database */
1597 LockIrql
= MiAcquirePfnLock();
1600 Status
= MiResolvePageFileFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode
), Address
, PointerPte
, Process
, &LockIrql
);
1602 /* And now release the lock and leave*/
1603 MiReleasePfnLock(LockIrql
);
1605 ASSERT(OldIrql
== KeGetCurrentIrql());
1606 ASSERT(OldIrql
<= APC_LEVEL
);
1607 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1612 // The PTE must be invalid but not completely empty. It must also not be a
1613 // prototype a transition or a paged-out PTE as those scenarii should've been handled above.
1614 // These are all Windows checks
1616 ASSERT(TempPte
.u
.Hard
.Valid
== 0);
1617 ASSERT(TempPte
.u
.Soft
.Prototype
== 0);
1618 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
1619 ASSERT(TempPte
.u
.Soft
.PageFileHigh
== 0);
1620 ASSERT(TempPte
.u
.Long
!= 0);
1623 // If we got this far, the PTE can only be a demand zero PTE, which is what
1624 // we want. Go handle it!
1626 Status
= MiResolveDemandZeroFault(Address
,
1628 (ULONG
)TempPte
.u
.Soft
.Protection
,
1631 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1632 if (NT_SUCCESS(Status
))
1635 // Make sure we're returning in a sane state and pass the status down
1637 ASSERT(OldIrql
== KeGetCurrentIrql());
1638 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1643 // Generate an access fault
1645 return STATUS_ACCESS_VIOLATION
;
1650 MmArmAccessFault(IN ULONG FaultCode
,
1652 IN KPROCESSOR_MODE Mode
,
1653 IN PVOID TrapInformation
)
1655 KIRQL OldIrql
= KeGetCurrentIrql(), LockIrql
;
1656 PMMPTE ProtoPte
= NULL
;
1657 PMMPTE PointerPte
= MiAddressToPte(Address
);
1658 PMMPDE PointerPde
= MiAddressToPde(Address
);
1659 #if (_MI_PAGING_LEVELS >= 3)
1660 PMMPDE PointerPpe
= MiAddressToPpe(Address
);
1661 #if (_MI_PAGING_LEVELS == 4)
1662 PMMPDE PointerPxe
= MiAddressToPxe(Address
);
1666 PETHREAD CurrentThread
;
1667 PEPROCESS CurrentProcess
;
1669 PMMSUPPORT WorkingSet
;
1670 ULONG ProtectionCode
;
1672 PFN_NUMBER PageFrameIndex
;
1674 BOOLEAN IsSessionAddress
;
1676 DPRINT("ARM3 FAULT AT: %p\n", Address
);
1678 /* Check for page fault on high IRQL */
1679 if (OldIrql
> APC_LEVEL
)
1681 #if (_MI_PAGING_LEVELS < 3)
1682 /* Could be a page table for paged pool, which we'll allow */
1683 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address
)) MiSynchronizeSystemPde((PMMPDE
)PointerPte
);
1684 MiCheckPdeForPagedPool(Address
);
1686 /* Check if any of the top-level pages are invalid */
1688 #if (_MI_PAGING_LEVELS == 4)
1689 (PointerPxe
->u
.Hard
.Valid
== 0) ||
1691 #if (_MI_PAGING_LEVELS >= 3)
1692 (PointerPpe
->u
.Hard
.Valid
== 0) ||
1694 (PointerPde
->u
.Hard
.Valid
== 0) ||
1695 (PointerPte
->u
.Hard
.Valid
== 0))
1697 /* This fault is not valid, print out some debugging help */
1698 DbgPrint("MM:***PAGE FAULT AT IRQL > 1 Va %p, IRQL %lx\n",
1701 if (TrapInformation
)
1703 PKTRAP_FRAME TrapFrame
= TrapInformation
;
1705 DbgPrint("MM:***EIP %p, EFL %p\n", TrapFrame
->Eip
, TrapFrame
->EFlags
);
1706 DbgPrint("MM:***EAX %p, ECX %p EDX %p\n", TrapFrame
->Eax
, TrapFrame
->Ecx
, TrapFrame
->Edx
);
1707 DbgPrint("MM:***EBX %p, ESI %p EDI %p\n", TrapFrame
->Ebx
, TrapFrame
->Esi
, TrapFrame
->Edi
);
1708 #elif defined(_M_AMD64)
1709 DbgPrint("MM:***RIP %p, EFL %p\n", TrapFrame
->Rip
, TrapFrame
->EFlags
);
1710 DbgPrint("MM:***RAX %p, RCX %p RDX %p\n", TrapFrame
->Rax
, TrapFrame
->Rcx
, TrapFrame
->Rdx
);
1711 DbgPrint("MM:***RBX %p, RSI %p RDI %p\n", TrapFrame
->Rbx
, TrapFrame
->Rsi
, TrapFrame
->Rdi
);
1712 #elif defined(_M_ARM)
1713 DbgPrint("MM:***PC %p\n", TrapFrame
->Pc
);
1714 DbgPrint("MM:***R0 %p, R1 %p R2 %p, R3 %p\n", TrapFrame
->R0
, TrapFrame
->R1
, TrapFrame
->R2
, TrapFrame
->R3
);
1715 DbgPrint("MM:***R11 %p, R12 %p SP %p, LR %p\n", TrapFrame
->R11
, TrapFrame
->R12
, TrapFrame
->Sp
, TrapFrame
->Lr
);
1719 /* Tell the trap handler to fail */
1720 return STATUS_IN_PAGE_ERROR
| 0x10000000;
1723 /* Not yet implemented in ReactOS */
1724 ASSERT(MI_IS_PAGE_LARGE(PointerPde
) == FALSE
);
1725 ASSERT((!MI_IS_NOT_PRESENT_FAULT(FaultCode
) && MI_IS_PAGE_COPY_ON_WRITE(PointerPte
)) == FALSE
);
1727 /* Check if this was a write */
1728 if (MI_IS_WRITE_ACCESS(FaultCode
))
1730 /* Was it to a read-only page? */
1731 Pfn1
= MI_PFN_ELEMENT(PointerPte
->u
.Hard
.PageFrameNumber
);
1732 if (!(PointerPte
->u
.Long
& PTE_READWRITE
) &&
1733 !(Pfn1
->OriginalPte
.u
.Soft
.Protection
& MM_READWRITE
))
1735 /* Crash with distinguished bugcheck code */
1736 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
1739 (ULONG_PTR
)TrapInformation
,
1744 /* Nothing is actually wrong */
1745 DPRINT1("Fault at IRQL %u is ok (%p)\n", OldIrql
, Address
);
1746 return STATUS_SUCCESS
;
1749 /* Check for kernel fault address */
1750 if (Address
>= MmSystemRangeStart
)
1752 /* Bail out, if the fault came from user mode */
1753 if (Mode
== UserMode
) return STATUS_ACCESS_VIOLATION
;
1755 #if (_MI_PAGING_LEVELS == 2)
1756 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address
)) MiSynchronizeSystemPde((PMMPDE
)PointerPte
);
1757 MiCheckPdeForPagedPool(Address
);
1760 /* Check if the higher page table entries are invalid */
1762 #if (_MI_PAGING_LEVELS == 4)
1763 /* AMD64 system, check if PXE is invalid */
1764 (PointerPxe
->u
.Hard
.Valid
== 0) ||
1766 #if (_MI_PAGING_LEVELS >= 3)
1767 /* PAE/AMD64 system, check if PPE is invalid */
1768 (PointerPpe
->u
.Hard
.Valid
== 0) ||
1770 /* Always check if the PDE is valid */
1771 (PointerPde
->u
.Hard
.Valid
== 0))
1773 /* PXE/PPE/PDE (still) not valid, kill the system */
1774 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
1777 (ULONG_PTR
)TrapInformation
,
1781 /* Not handling session faults yet */
1782 IsSessionAddress
= MI_IS_SESSION_ADDRESS(Address
);
1784 /* The PDE is valid, so read the PTE */
1785 TempPte
= *PointerPte
;
1786 if (TempPte
.u
.Hard
.Valid
== 1)
1788 /* Check if this was system space or session space */
1789 if (!IsSessionAddress
)
1791 /* Check if the PTE is still valid under PFN lock */
1792 OldIrql
= MiAcquirePfnLock();
1793 TempPte
= *PointerPte
;
1794 if (TempPte
.u
.Hard
.Valid
)
1796 /* Check if this was a write */
1797 if (MI_IS_WRITE_ACCESS(FaultCode
))
1799 /* Was it to a read-only page? */
1800 Pfn1
= MI_PFN_ELEMENT(PointerPte
->u
.Hard
.PageFrameNumber
);
1801 if (!(PointerPte
->u
.Long
& PTE_READWRITE
) &&
1802 !(Pfn1
->OriginalPte
.u
.Soft
.Protection
& MM_READWRITE
))
1804 /* Crash with distinguished bugcheck code */
1805 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
1808 (ULONG_PTR
)TrapInformation
,
1813 /* Check for execution of non-executable memory */
1814 if (MI_IS_INSTRUCTION_FETCH(FaultCode
) &&
1815 !MI_IS_PAGE_EXECUTABLE(&TempPte
))
1817 KeBugCheckEx(ATTEMPTED_EXECUTE_OF_NOEXECUTE_MEMORY
,
1819 (ULONG_PTR
)TempPte
.u
.Long
,
1820 (ULONG_PTR
)TrapInformation
,
1825 /* Release PFN lock and return all good */
1826 MiReleasePfnLock(OldIrql
);
1827 return STATUS_SUCCESS
;
1830 #if (_MI_PAGING_LEVELS == 2)
1831 /* Check if this was a session PTE that needs to remap the session PDE */
1832 if (MI_IS_SESSION_PTE(Address
))
1834 /* Do the remapping */
1835 Status
= MiCheckPdeForSessionSpace(Address
);
1836 if (!NT_SUCCESS(Status
))
1838 /* It failed, this address is invalid */
1839 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
1842 (ULONG_PTR
)TrapInformation
,
1848 _WARN("Session space stuff is not implemented yet!")
1852 /* Check for a fault on the page table or hyperspace */
1853 if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address
))
1855 #if (_MI_PAGING_LEVELS < 3)
1856 /* Windows does this check but I don't understand why -- it's done above! */
1857 ASSERT(MiCheckPdeForPagedPool(Address
) != STATUS_WAIT_1
);
1859 /* Handle this as a user mode fault */
1863 /* Get the current thread */
1864 CurrentThread
= PsGetCurrentThread();
1866 /* What kind of address is this */
1867 if (!IsSessionAddress
)
1869 /* Use the system working set */
1870 WorkingSet
= &MmSystemCacheWs
;
1871 CurrentProcess
= NULL
;
1873 /* Make sure we don't have a recursive working set lock */
1874 if ((CurrentThread
->OwnsProcessWorkingSetExclusive
) ||
1875 (CurrentThread
->OwnsProcessWorkingSetShared
) ||
1876 (CurrentThread
->OwnsSystemWorkingSetExclusive
) ||
1877 (CurrentThread
->OwnsSystemWorkingSetShared
) ||
1878 (CurrentThread
->OwnsSessionWorkingSetExclusive
) ||
1879 (CurrentThread
->OwnsSessionWorkingSetShared
))
1882 return STATUS_IN_PAGE_ERROR
| 0x10000000;
1887 /* Use the session process and working set */
1888 CurrentProcess
= HYDRA_PROCESS
;
1889 WorkingSet
= &MmSessionSpace
->GlobalVirtualAddress
->Vm
;
1891 /* Make sure we don't have a recursive working set lock */
1892 if ((CurrentThread
->OwnsSessionWorkingSetExclusive
) ||
1893 (CurrentThread
->OwnsSessionWorkingSetShared
))
1896 return STATUS_IN_PAGE_ERROR
| 0x10000000;
1900 /* Acquire the working set lock */
1901 KeRaiseIrql(APC_LEVEL
, &LockIrql
);
1902 MiLockWorkingSet(CurrentThread
, WorkingSet
);
1904 /* Re-read PTE now that we own the lock */
1905 TempPte
= *PointerPte
;
1906 if (TempPte
.u
.Hard
.Valid
== 1)
1908 /* Check if this was a write */
1909 if (MI_IS_WRITE_ACCESS(FaultCode
))
1911 /* Was it to a read-only page that is not copy on write? */
1912 Pfn1
= MI_PFN_ELEMENT(PointerPte
->u
.Hard
.PageFrameNumber
);
1913 if (!(TempPte
.u
.Long
& PTE_READWRITE
) &&
1914 !(Pfn1
->OriginalPte
.u
.Soft
.Protection
& MM_READWRITE
) &&
1915 !MI_IS_PAGE_COPY_ON_WRITE(&TempPte
))
1917 /* Case not yet handled */
1918 ASSERT(!IsSessionAddress
);
1920 /* Crash with distinguished bugcheck code */
1921 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
1924 (ULONG_PTR
)TrapInformation
,
1929 /* Check for execution of non-executable memory */
1930 if (MI_IS_INSTRUCTION_FETCH(FaultCode
) &&
1931 !MI_IS_PAGE_EXECUTABLE(&TempPte
))
1933 KeBugCheckEx(ATTEMPTED_EXECUTE_OF_NOEXECUTE_MEMORY
,
1935 (ULONG_PTR
)TempPte
.u
.Long
,
1936 (ULONG_PTR
)TrapInformation
,
1940 /* Check for read-only write in session space */
1941 if ((IsSessionAddress
) &&
1942 MI_IS_WRITE_ACCESS(FaultCode
) &&
1943 !MI_IS_PAGE_WRITEABLE(&TempPte
))
1946 ASSERT(MI_IS_SESSION_IMAGE_ADDRESS(Address
));
1949 if (!MI_IS_PAGE_COPY_ON_WRITE(&TempPte
))
1951 /* Then this is not allowed */
1952 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
1954 (ULONG_PTR
)TempPte
.u
.Long
,
1955 (ULONG_PTR
)TrapInformation
,
1959 /* Otherwise, handle COW */
1963 /* Release the working set */
1964 MiUnlockWorkingSet(CurrentThread
, WorkingSet
);
1965 KeLowerIrql(LockIrql
);
1967 /* Otherwise, the PDE was probably invalid, and all is good now */
1968 return STATUS_SUCCESS
;
1971 /* Check one kind of prototype PTE */
1972 if (TempPte
.u
.Soft
.Prototype
)
1974 /* Make sure protected pool is on, and that this is a pool address */
1975 if ((MmProtectFreedNonPagedPool
) &&
1976 (((Address
>= MmNonPagedPoolStart
) &&
1977 (Address
< (PVOID
)((ULONG_PTR
)MmNonPagedPoolStart
+
1978 MmSizeOfNonPagedPoolInBytes
))) ||
1979 ((Address
>= MmNonPagedPoolExpansionStart
) &&
1980 (Address
< MmNonPagedPoolEnd
))))
1982 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
1983 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL
,
1990 /* Get the prototype PTE! */
1991 ProtoPte
= MiProtoPteToPte(&TempPte
);
1993 /* Do we need to locate the prototype PTE in session space? */
1994 if ((IsSessionAddress
) &&
1995 (TempPte
.u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
))
1997 /* Yep, go find it as well as the VAD for it */
1998 ProtoPte
= MiCheckVirtualAddress(Address
,
2001 ASSERT(ProtoPte
!= NULL
);
2006 /* We don't implement transition PTEs */
2007 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
2009 /* Check for no-access PTE */
2010 if (TempPte
.u
.Soft
.Protection
== MM_NOACCESS
)
2012 /* Bugcheck the system! */
2013 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
2016 (ULONG_PTR
)TrapInformation
,
2020 /* Check for no protecton at all */
2021 if (TempPte
.u
.Soft
.Protection
== MM_ZERO_ACCESS
)
2023 /* Bugcheck the system! */
2024 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
2027 (ULONG_PTR
)TrapInformation
,
2032 /* Check for demand page */
2033 if (MI_IS_WRITE_ACCESS(FaultCode
) &&
2035 !(IsSessionAddress
) &&
2036 !(TempPte
.u
.Hard
.Valid
))
2038 /* Get the protection code */
2039 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
2040 if (!(TempPte
.u
.Soft
.Protection
& MM_READWRITE
))
2042 /* Bugcheck the system! */
2043 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
2046 (ULONG_PTR
)TrapInformation
,
2051 /* Now do the real fault handling */
2052 Status
= MiDispatchFault(FaultCode
,
2061 /* Release the working set */
2062 ASSERT(KeAreAllApcsDisabled() == TRUE
);
2063 MiUnlockWorkingSet(CurrentThread
, WorkingSet
);
2064 KeLowerIrql(LockIrql
);
2067 DPRINT("Fault resolved with status: %lx\n", Status
);
2071 /* This is a user fault */
2073 CurrentThread
= PsGetCurrentThread();
2074 CurrentProcess
= (PEPROCESS
)CurrentThread
->Tcb
.ApcState
.Process
;
2076 /* Lock the working set */
2077 MiLockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2079 ProtectionCode
= MM_INVALID_PROTECTION
;
2081 #if (_MI_PAGING_LEVELS == 4)
2082 /* Check if the PXE is valid */
2083 if (PointerPxe
->u
.Hard
.Valid
== 0)
2085 /* Right now, we only handle scenarios where the PXE is totally empty */
2086 ASSERT(PointerPxe
->u
.Long
== 0);
2088 /* This is only possible for user mode addresses! */
2089 ASSERT(PointerPte
<= MiHighestUserPte
);
2091 /* Check if we have a VAD */
2092 MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
2093 if (ProtectionCode
== MM_NOACCESS
)
2095 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2096 return STATUS_ACCESS_VIOLATION
;
2099 /* Resolve a demand zero fault */
2100 MiResolveDemandZeroFault(PointerPpe
,
2106 /* We should come back with a valid PXE */
2107 ASSERT(PointerPxe
->u
.Hard
.Valid
== 1);
2111 #if (_MI_PAGING_LEVELS >= 3)
2112 /* Check if the PPE is valid */
2113 if (PointerPpe
->u
.Hard
.Valid
== 0)
2115 /* Right now, we only handle scenarios where the PPE is totally empty */
2116 ASSERT(PointerPpe
->u
.Long
== 0);
2118 /* This is only possible for user mode addresses! */
2119 ASSERT(PointerPte
<= MiHighestUserPte
);
2121 /* Check if we have a VAD, unless we did this already */
2122 if (ProtectionCode
== MM_INVALID_PROTECTION
)
2124 MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
2127 if (ProtectionCode
== MM_NOACCESS
)
2129 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2130 return STATUS_ACCESS_VIOLATION
;
2133 /* Resolve a demand zero fault */
2134 MiResolveDemandZeroFault(PointerPde
,
2140 /* We should come back with a valid PPE */
2141 ASSERT(PointerPpe
->u
.Hard
.Valid
== 1);
2145 /* Check if the PDE is invalid */
2146 if (PointerPde
->u
.Hard
.Valid
== 0)
2148 /* Right now, we only handle scenarios where the PDE is totally empty */
2149 ASSERT(PointerPde
->u
.Long
== 0);
2151 /* And go dispatch the fault on the PDE. This should handle the demand-zero */
2153 UserPdeFault
= TRUE
;
2155 /* Check if we have a VAD, unless we did this already */
2156 if (ProtectionCode
== MM_INVALID_PROTECTION
)
2158 MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
2161 if (ProtectionCode
== MM_NOACCESS
)
2163 #if (_MI_PAGING_LEVELS == 2)
2164 /* Could be a page table for paged pool */
2165 MiCheckPdeForPagedPool(Address
);
2167 /* Has the code above changed anything -- is this now a valid PTE? */
2168 Status
= (PointerPde
->u
.Hard
.Valid
== 1) ? STATUS_SUCCESS
: STATUS_ACCESS_VIOLATION
;
2170 /* Either this was a bogus VA or we've fixed up a paged pool PDE */
2171 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2175 /* Resolve a demand zero fault */
2176 MiResolveDemandZeroFault(PointerPte
,
2182 UserPdeFault
= FALSE
;
2184 /* We should come back with APCs enabled, and with a valid PDE */
2185 ASSERT(KeAreAllApcsDisabled() == TRUE
);
2186 ASSERT(PointerPde
->u
.Hard
.Valid
== 1);
2190 /* Not yet implemented in ReactOS */
2191 ASSERT(MI_IS_PAGE_LARGE(PointerPde
) == FALSE
);
2194 /* Now capture the PTE. */
2195 TempPte
= *PointerPte
;
2197 /* Check if the PTE is valid */
2198 if (TempPte
.u
.Hard
.Valid
)
2200 /* Check if this is a write on a readonly PTE */
2201 if (MI_IS_WRITE_ACCESS(FaultCode
))
2203 /* Is this a copy on write PTE? */
2204 if (MI_IS_PAGE_COPY_ON_WRITE(&TempPte
))
2206 PFN_NUMBER PageFrameIndex
, OldPageFrameIndex
;
2209 LockIrql
= MiAcquirePfnLock();
2211 ASSERT(MmAvailablePages
> 0);
2213 /* Allocate a new page and copy it */
2214 PageFrameIndex
= MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(CurrentProcess
));
2215 OldPageFrameIndex
= PFN_FROM_PTE(&TempPte
);
2217 MiCopyPfn(PageFrameIndex
, OldPageFrameIndex
);
2219 /* Dereference whatever this PTE is referencing */
2220 Pfn1
= MI_PFN_ELEMENT(OldPageFrameIndex
);
2221 ASSERT(Pfn1
->u3
.e1
.PrototypePte
== 1);
2222 ASSERT(!MI_IS_PFN_DELETED(Pfn1
));
2223 ProtoPte
= Pfn1
->PteAddress
;
2224 MiDeletePte(PointerPte
, Address
, CurrentProcess
, ProtoPte
);
2226 /* And make a new shiny one with our page */
2227 MiInitializePfn(PageFrameIndex
, PointerPte
, TRUE
);
2228 TempPte
.u
.Hard
.PageFrameNumber
= PageFrameIndex
;
2229 TempPte
.u
.Hard
.Write
= 1;
2230 TempPte
.u
.Hard
.CopyOnWrite
= 0;
2232 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
2234 MiReleasePfnLock(LockIrql
);
2236 /* Return the status */
2237 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2238 return STATUS_PAGE_FAULT_COPY_ON_WRITE
;
2241 /* Is this a read-only PTE? */
2242 if (!MI_IS_PAGE_WRITEABLE(&TempPte
))
2244 /* Return the status */
2245 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2246 return STATUS_ACCESS_VIOLATION
;
2250 /* Check for execution of non-executable memory */
2251 if (MI_IS_INSTRUCTION_FETCH(FaultCode
) &&
2252 !MI_IS_PAGE_EXECUTABLE(&TempPte
))
2254 /* Return the status */
2255 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2256 return STATUS_ACCESS_VIOLATION
;
2259 /* The fault has already been resolved by a different thread */
2260 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2261 return STATUS_SUCCESS
;
2264 /* Quick check for demand-zero */
2265 if (TempPte
.u
.Long
== (MM_READWRITE
<< MM_PTE_SOFTWARE_PROTECTION_BITS
))
2267 /* Resolve the fault */
2268 MiResolveDemandZeroFault(Address
,
2274 /* Return the status */
2275 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2276 return STATUS_PAGE_FAULT_DEMAND_ZERO
;
2279 /* Check for zero PTE */
2280 if (TempPte
.u
.Long
== 0)
2282 /* Check if this address range belongs to a valid allocation (VAD) */
2283 ProtoPte
= MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
2284 if (ProtectionCode
== MM_NOACCESS
)
2286 #if (_MI_PAGING_LEVELS == 2)
2287 /* Could be a page table for paged pool */
2288 MiCheckPdeForPagedPool(Address
);
2290 /* Has the code above changed anything -- is this now a valid PTE? */
2291 Status
= (PointerPte
->u
.Hard
.Valid
== 1) ? STATUS_SUCCESS
: STATUS_ACCESS_VIOLATION
;
2293 /* Either this was a bogus VA or we've fixed up a paged pool PDE */
2294 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2299 * Check if this is a real user-mode address or actually a kernel-mode
2300 * page table for a user mode address
2302 if (Address
<= MM_HIGHEST_USER_ADDRESS
)
2304 /* Add an additional page table reference */
2305 MiIncrementPageTableReferences(Address
);
2308 /* Is this a guard page? */
2309 if ((ProtectionCode
& MM_PROTECT_SPECIAL
) == MM_GUARDPAGE
)
2311 /* The VAD protection cannot be MM_DECOMMIT! */
2312 ASSERT(ProtectionCode
!= MM_DECOMMIT
);
2314 /* Remove the bit */
2315 TempPte
.u
.Soft
.Protection
= ProtectionCode
& ~MM_GUARDPAGE
;
2316 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
2319 ASSERT(ProtoPte
== NULL
);
2320 ASSERT(CurrentThread
->ApcNeeded
== 0);
2322 /* Drop the working set lock */
2323 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2324 ASSERT(KeGetCurrentIrql() == OldIrql
);
2326 /* Handle stack expansion */
2327 return MiCheckForUserStackOverflow(Address
, TrapInformation
);
2330 /* Did we get a prototype PTE back? */
2333 /* Is this PTE actually part of the PDE-PTE self-mapping directory? */
2334 if (PointerPde
== MiAddressToPde(PTE_BASE
))
2336 /* Then it's really a demand-zero PDE (on behalf of user-mode) */
2338 _WARN("This is probably completely broken!");
2339 MI_WRITE_INVALID_PDE((PMMPDE
)PointerPte
, DemandZeroPde
);
2341 MI_WRITE_INVALID_PTE(PointerPte
, DemandZeroPde
);
2346 /* No, create a new PTE. First, write the protection */
2347 TempPte
.u
.Soft
.Protection
= ProtectionCode
;
2348 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
2351 /* Lock the PFN database since we're going to grab a page */
2352 OldIrql
= MiAcquirePfnLock();
2354 /* Make sure we have enough pages */
2355 ASSERT(MmAvailablePages
>= 32);
2357 /* Try to get a zero page */
2358 MI_SET_USAGE(MI_USAGE_PEB_TEB
);
2359 MI_SET_PROCESS2(CurrentProcess
->ImageFileName
);
2360 Color
= MI_GET_NEXT_PROCESS_COLOR(CurrentProcess
);
2361 PageFrameIndex
= MiRemoveZeroPageSafe(Color
);
2362 if (!PageFrameIndex
)
2364 /* Grab a page out of there. Later we should grab a colored zero page */
2365 PageFrameIndex
= MiRemoveAnyPage(Color
);
2366 ASSERT(PageFrameIndex
);
2368 /* Release the lock since we need to do some zeroing */
2369 MiReleasePfnLock(OldIrql
);
2371 /* Zero out the page, since it's for user-mode */
2372 MiZeroPfn(PageFrameIndex
);
2374 /* Grab the lock again so we can initialize the PFN entry */
2375 OldIrql
= MiAcquirePfnLock();
2378 /* Initialize the PFN entry now */
2379 MiInitializePfn(PageFrameIndex
, PointerPte
, 1);
2381 /* Increment the count of pages in the process */
2382 CurrentProcess
->NumberOfPrivatePages
++;
2384 /* One more demand-zero fault */
2385 KeGetCurrentPrcb()->MmDemandZeroCount
++;
2387 /* And we're done with the lock */
2388 MiReleasePfnLock(OldIrql
);
2390 /* Fault on user PDE, or fault on user PTE? */
2391 if (PointerPte
<= MiHighestUserPte
)
2393 /* User fault, build a user PTE */
2394 MI_MAKE_HARDWARE_PTE_USER(&TempPte
,
2396 PointerPte
->u
.Soft
.Protection
,
2401 /* This is a user-mode PDE, create a kernel PTE for it */
2402 MI_MAKE_HARDWARE_PTE(&TempPte
,
2404 PointerPte
->u
.Soft
.Protection
,
2408 /* Write the dirty bit for writeable pages */
2409 if (MI_IS_PAGE_WRITEABLE(&TempPte
)) MI_MAKE_DIRTY_PAGE(&TempPte
);
2411 /* And now write down the PTE, making the address valid */
2412 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
2413 Pfn1
= MI_PFN_ELEMENT(PageFrameIndex
);
2414 ASSERT(Pfn1
->u1
.Event
== NULL
);
2417 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
2418 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2419 return STATUS_PAGE_FAULT_DEMAND_ZERO
;
2422 /* We should have a valid protection here */
2423 ASSERT(ProtectionCode
!= 0x100);
2425 /* Write the prototype PTE */
2426 TempPte
= PrototypePte
;
2427 TempPte
.u
.Soft
.Protection
= ProtectionCode
;
2428 ASSERT(TempPte
.u
.Long
!= 0);
2429 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
2433 /* Get the protection code and check if this is a proto PTE */
2434 ProtectionCode
= (ULONG
)TempPte
.u
.Soft
.Protection
;
2435 if (TempPte
.u
.Soft
.Prototype
)
2437 /* Do we need to go find the real PTE? */
2438 if (TempPte
.u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
)
2440 /* Get the prototype pte and VAD for it */
2441 ProtoPte
= MiCheckVirtualAddress(Address
,
2446 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
2447 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2448 return STATUS_ACCESS_VIOLATION
;
2453 /* Get the prototype PTE! */
2454 ProtoPte
= MiProtoPteToPte(&TempPte
);
2456 /* Is it read-only */
2457 if (TempPte
.u
.Proto
.ReadOnly
)
2459 /* Set read-only code */
2460 ProtectionCode
= MM_READONLY
;
2464 /* Set unknown protection */
2465 ProtectionCode
= 0x100;
2466 ASSERT(CurrentProcess
->CloneRoot
!= NULL
);
2472 /* Do we have a valid protection code? */
2473 if (ProtectionCode
!= 0x100)
2475 /* Run a software access check first, including to detect guard pages */
2476 Status
= MiAccessCheck(PointerPte
,
2477 !MI_IS_NOT_PRESENT_FAULT(FaultCode
),
2482 if (Status
!= STATUS_SUCCESS
)
2485 ASSERT(CurrentThread
->ApcNeeded
== 0);
2487 /* Drop the working set lock */
2488 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2489 ASSERT(KeGetCurrentIrql() == OldIrql
);
2491 /* Did we hit a guard page? */
2492 if (Status
== STATUS_GUARD_PAGE_VIOLATION
)
2494 /* Handle stack expansion */
2495 return MiCheckForUserStackOverflow(Address
, TrapInformation
);
2498 /* Otherwise, fail back to the caller directly */
2503 /* Dispatch the fault */
2504 Status
= MiDispatchFault(FaultCode
,
2513 /* Return the status */
2514 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
2515 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2521 MmGetExecuteOptions(IN PULONG ExecuteOptions
)
2523 PKPROCESS CurrentProcess
= &PsGetCurrentProcess()->Pcb
;
2524 ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL
);
2526 *ExecuteOptions
= 0;
2528 if (CurrentProcess
->Flags
.ExecuteDisable
)
2530 *ExecuteOptions
|= MEM_EXECUTE_OPTION_DISABLE
;
2533 if (CurrentProcess
->Flags
.ExecuteEnable
)
2535 *ExecuteOptions
|= MEM_EXECUTE_OPTION_ENABLE
;
2538 if (CurrentProcess
->Flags
.DisableThunkEmulation
)
2540 *ExecuteOptions
|= MEM_EXECUTE_OPTION_DISABLE_THUNK_EMULATION
;
2543 if (CurrentProcess
->Flags
.Permanent
)
2545 *ExecuteOptions
|= MEM_EXECUTE_OPTION_PERMANENT
;
2548 if (CurrentProcess
->Flags
.ExecuteDispatchEnable
)
2550 *ExecuteOptions
|= MEM_EXECUTE_OPTION_EXECUTE_DISPATCH_ENABLE
;
2553 if (CurrentProcess
->Flags
.ImageDispatchEnable
)
2555 *ExecuteOptions
|= MEM_EXECUTE_OPTION_IMAGE_DISPATCH_ENABLE
;
2558 return STATUS_SUCCESS
;
2563 MmSetExecuteOptions(IN ULONG ExecuteOptions
)
2565 PKPROCESS CurrentProcess
= &PsGetCurrentProcess()->Pcb
;
2566 KLOCK_QUEUE_HANDLE ProcessLock
;
2567 NTSTATUS Status
= STATUS_ACCESS_DENIED
;
2568 ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL
);
2570 /* Only accept valid flags */
2571 if (ExecuteOptions
& ~MEM_EXECUTE_OPTION_VALID_FLAGS
)
2574 DPRINT1("Invalid no-execute options\n");
2575 return STATUS_INVALID_PARAMETER
;
2578 /* Change the NX state in the process lock */
2579 KiAcquireProcessLock(CurrentProcess
, &ProcessLock
);
2581 /* Don't change anything if the permanent flag was set */
2582 if (!CurrentProcess
->Flags
.Permanent
)
2584 /* Start by assuming it's not disabled */
2585 CurrentProcess
->Flags
.ExecuteDisable
= FALSE
;
2587 /* Now process each flag and turn the equivalent bit on */
2588 if (ExecuteOptions
& MEM_EXECUTE_OPTION_DISABLE
)
2590 CurrentProcess
->Flags
.ExecuteDisable
= TRUE
;
2592 if (ExecuteOptions
& MEM_EXECUTE_OPTION_ENABLE
)
2594 CurrentProcess
->Flags
.ExecuteEnable
= TRUE
;
2596 if (ExecuteOptions
& MEM_EXECUTE_OPTION_DISABLE_THUNK_EMULATION
)
2598 CurrentProcess
->Flags
.DisableThunkEmulation
= TRUE
;
2600 if (ExecuteOptions
& MEM_EXECUTE_OPTION_PERMANENT
)
2602 CurrentProcess
->Flags
.Permanent
= TRUE
;
2604 if (ExecuteOptions
& MEM_EXECUTE_OPTION_EXECUTE_DISPATCH_ENABLE
)
2606 CurrentProcess
->Flags
.ExecuteDispatchEnable
= TRUE
;
2608 if (ExecuteOptions
& MEM_EXECUTE_OPTION_IMAGE_DISPATCH_ENABLE
)
2610 CurrentProcess
->Flags
.ImageDispatchEnable
= TRUE
;
2613 /* These are turned on by default if no-execution is also eanbled */
2614 if (CurrentProcess
->Flags
.ExecuteEnable
)
2616 CurrentProcess
->Flags
.ExecuteDispatchEnable
= TRUE
;
2617 CurrentProcess
->Flags
.ImageDispatchEnable
= TRUE
;
2621 Status
= STATUS_SUCCESS
;
2624 /* Release the lock and return status */
2625 KiReleaseProcessLock(&ProcessLock
);