2 * PROJECT: ReactOS Kernel
3 * LICENSE: BSD - See COPYING.ARM in the top level directory
4 * FILE: ntoskrnl/mm/ARM3/pagfault.c
5 * PURPOSE: ARM Memory Manager Page Fault Handling
6 * PROGRAMMERS: ReactOS Portable Systems Group
9 /* INCLUDES *******************************************************************/
15 #define MODULE_INVOLVED_IN_ARM3
16 #include "../ARM3/miarm.h"
18 /* GLOBALS ********************************************************************/
20 #define HYDRA_PROCESS (PEPROCESS)1
22 BOOLEAN UserPdeFault
= FALSE
;
25 /* PRIVATE FUNCTIONS **********************************************************/
29 MiCheckForUserStackOverflow(IN PVOID Address
,
30 IN PVOID TrapInformation
)
32 PETHREAD CurrentThread
= PsGetCurrentThread();
33 PTEB Teb
= CurrentThread
->Tcb
.Teb
;
34 PVOID StackBase
, DeallocationStack
, NextStackAddress
;
38 /* Do we own the address space lock? */
39 if (CurrentThread
->AddressSpaceOwner
== 1)
41 /* This isn't valid */
42 DPRINT1("Process owns address space lock\n");
43 ASSERT(KeAreAllApcsDisabled() == TRUE
);
44 return STATUS_GUARD_PAGE_VIOLATION
;
47 /* Are we attached? */
48 if (KeIsAttachedProcess())
50 /* This isn't valid */
51 DPRINT1("Process is attached\n");
52 return STATUS_GUARD_PAGE_VIOLATION
;
55 /* Read the current settings */
56 StackBase
= Teb
->NtTib
.StackBase
;
57 DeallocationStack
= Teb
->DeallocationStack
;
58 GuranteedSize
= Teb
->GuaranteedStackBytes
;
59 DPRINT("Handling guard page fault with Stacks Addresses 0x%p and 0x%p, guarantee: %lx\n",
60 StackBase
, DeallocationStack
, GuranteedSize
);
62 /* Guarantees make this code harder, for now, assume there aren't any */
63 ASSERT(GuranteedSize
== 0);
65 /* So allocate only the minimum guard page size */
66 GuranteedSize
= PAGE_SIZE
;
68 /* Does this faulting stack address actually exist in the stack? */
69 if ((Address
>= StackBase
) || (Address
< DeallocationStack
))
72 DPRINT1("Faulting address outside of stack bounds. Address=%p, StackBase=%p, DeallocationStack=%p\n",
73 Address
, StackBase
, DeallocationStack
);
74 return STATUS_GUARD_PAGE_VIOLATION
;
77 /* This is where the stack will start now */
78 NextStackAddress
= (PVOID
)((ULONG_PTR
)PAGE_ALIGN(Address
) - GuranteedSize
);
80 /* Do we have at least one page between here and the end of the stack? */
81 if (((ULONG_PTR
)NextStackAddress
- PAGE_SIZE
) <= (ULONG_PTR
)DeallocationStack
)
83 /* We don't -- Windows would try to make this guard page valid now */
84 DPRINT1("Close to our death...\n");
85 return STATUS_STACK_OVERFLOW
;
88 /* Don't handle this flag yet */
89 ASSERT((PsGetCurrentProcess()->Peb
->NtGlobalFlag
& FLG_DISABLE_STACK_EXTENSION
) == 0);
91 /* Update the stack limit */
92 Teb
->NtTib
.StackLimit
= (PVOID
)((ULONG_PTR
)NextStackAddress
+ GuranteedSize
);
94 /* Now move the guard page to the next page */
95 Status
= ZwAllocateVirtualMemory(NtCurrentProcess(),
100 PAGE_READWRITE
| PAGE_GUARD
);
101 if ((NT_SUCCESS(Status
) || (Status
== STATUS_ALREADY_COMMITTED
)))
104 DPRINT("Guard page handled successfully for %p\n", Address
);
105 return STATUS_PAGE_FAULT_GUARD_PAGE
;
108 /* Fail, we couldn't move the guard page */
109 DPRINT1("Guard page failure: %lx\n", Status
);
111 return STATUS_STACK_OVERFLOW
;
117 _In_ ULONG ProtectionMask
,
119 _In_ BOOLEAN Execute
)
121 #define _BYTE_MASK(Bit0, Bit1, Bit2, Bit3, Bit4, Bit5, Bit6, Bit7) \
122 (Bit0) | ((Bit1) << 1) | ((Bit2) << 2) | ((Bit3) << 3) | \
123 ((Bit4) << 4) | ((Bit5) << 5) | ((Bit6) << 6) | ((Bit7) << 7)
124 static const UCHAR AccessAllowedMask
[2][2] =
126 { // Protect 0 1 2 3 4 5 6 7
127 _BYTE_MASK(0, 1, 1, 1, 1, 1, 1, 1), // READ
128 _BYTE_MASK(0, 0, 1, 1, 0, 0, 1, 1), // EXECUTE READ
131 _BYTE_MASK(0, 0, 0, 0, 1, 1, 1, 1), // WRITE
132 _BYTE_MASK(0, 0, 0, 0, 0, 0, 1, 1), // EXECUTE WRITE
136 /* We want only the lower access bits */
137 ProtectionMask
&= MM_PROTECT_ACCESS
;
139 /* Look it up in the table */
140 return (AccessAllowedMask
[Write
!= 0][Execute
!= 0] >> ProtectionMask
) & 1;
145 MiAccessCheck(IN PMMPTE PointerPte
,
146 IN BOOLEAN StoreInstruction
,
147 IN KPROCESSOR_MODE PreviousMode
,
148 IN ULONG_PTR ProtectionMask
,
154 /* Check for invalid user-mode access */
155 if ((PreviousMode
== UserMode
) && (PointerPte
> MiHighestUserPte
))
157 return STATUS_ACCESS_VIOLATION
;
160 /* Capture the PTE -- is it valid? */
161 TempPte
= *PointerPte
;
162 if (TempPte
.u
.Hard
.Valid
)
164 /* Was someone trying to write to it? */
165 if (StoreInstruction
)
168 if ((TempPte
.u
.Hard
.Write
) || (TempPte
.u
.Hard
.CopyOnWrite
))
170 /* Then there's nothing to worry about */
171 return STATUS_SUCCESS
;
174 /* Oops! This isn't allowed */
175 return STATUS_ACCESS_VIOLATION
;
178 /* Someone was trying to read from a valid PTE, that's fine too */
179 return STATUS_SUCCESS
;
182 /* Check if the protection on the page allows what is being attempted */
183 if (!MiIsAccessAllowed(ProtectionMask
, StoreInstruction
, FALSE
))
185 return STATUS_ACCESS_VIOLATION
;
188 /* Check if this is a guard page */
189 if ((ProtectionMask
& MM_PROTECT_SPECIAL
) == MM_GUARDPAGE
)
191 NT_ASSERT(ProtectionMask
!= MM_DECOMMIT
);
193 /* Attached processes can't expand their stack */
194 if (KeIsAttachedProcess()) return STATUS_ACCESS_VIOLATION
;
196 /* No support for transition PTEs yet */
197 ASSERT(((TempPte
.u
.Soft
.Transition
== 1) &&
198 (TempPte
.u
.Soft
.Prototype
== 0)) == FALSE
);
200 /* Remove the guard page bit, and return a guard page violation */
201 TempPte
.u
.Soft
.Protection
= ProtectionMask
& ~MM_GUARDPAGE
;
202 NT_ASSERT(TempPte
.u
.Long
!= 0);
203 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
204 return STATUS_GUARD_PAGE_VIOLATION
;
208 return STATUS_SUCCESS
;
213 MiCheckVirtualAddress(IN PVOID VirtualAddress
,
214 OUT PULONG ProtectCode
,
215 OUT PMMVAD
*ProtoVad
)
220 /* No prototype/section support for now */
223 /* User or kernel fault? */
224 if (VirtualAddress
<= MM_HIGHEST_USER_ADDRESS
)
226 /* Special case for shared data */
227 if (PAGE_ALIGN(VirtualAddress
) == (PVOID
)MM_SHARED_USER_DATA_VA
)
229 /* It's a read-only page */
230 *ProtectCode
= MM_READONLY
;
231 return MmSharedUserDataPte
;
234 /* Find the VAD, it might not exist if the address is bogus */
235 Vad
= MiLocateAddress(VirtualAddress
);
238 /* Bogus virtual address */
239 *ProtectCode
= MM_NOACCESS
;
243 /* ReactOS does not handle physical memory VADs yet */
244 ASSERT(Vad
->u
.VadFlags
.VadType
!= VadDevicePhysicalMemory
);
246 /* Check if it's a section, or just an allocation */
247 if (Vad
->u
.VadFlags
.PrivateMemory
)
249 /* ReactOS does not handle AWE VADs yet */
250 ASSERT(Vad
->u
.VadFlags
.VadType
!= VadAwe
);
252 /* This must be a TEB/PEB VAD */
253 if (Vad
->u
.VadFlags
.MemCommit
)
255 /* It's committed, so return the VAD protection */
256 *ProtectCode
= (ULONG
)Vad
->u
.VadFlags
.Protection
;
260 /* It has not yet been committed, so return no access */
261 *ProtectCode
= MM_NOACCESS
;
264 /* In both cases, return no PTE */
269 /* ReactOS does not supoprt these VADs yet */
270 ASSERT(Vad
->u
.VadFlags
.VadType
!= VadImageMap
);
271 ASSERT(Vad
->u2
.VadFlags2
.ExtendableFile
== 0);
273 /* Return the proto VAD */
276 /* Get the prototype PTE for this page */
277 PointerPte
= (((ULONG_PTR
)VirtualAddress
>> PAGE_SHIFT
) - Vad
->StartingVpn
) + Vad
->FirstPrototypePte
;
278 ASSERT(PointerPte
!= NULL
);
279 ASSERT(PointerPte
<= Vad
->LastContiguousPte
);
281 /* Return the Prototype PTE and the protection for the page mapping */
282 *ProtectCode
= (ULONG
)Vad
->u
.VadFlags
.Protection
;
286 else if (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress
))
288 /* This should never happen, as these addresses are handled by the double-maping */
289 if (((PMMPTE
)VirtualAddress
>= MiAddressToPte(MmPagedPoolStart
)) &&
290 ((PMMPTE
)VirtualAddress
<= MmPagedPoolInfo
.LastPteForPagedPool
))
292 /* Fail such access */
293 *ProtectCode
= MM_NOACCESS
;
297 /* Return full access rights */
298 *ProtectCode
= MM_READWRITE
;
301 else if (MI_IS_SESSION_ADDRESS(VirtualAddress
))
303 /* ReactOS does not have an image list yet, so bail out to failure case */
304 ASSERT(IsListEmpty(&MmSessionSpace
->ImageList
));
307 /* Default case -- failure */
308 *ProtectCode
= MM_NOACCESS
;
312 #if (_MI_PAGING_LEVELS == 2)
315 MiSynchronizeSystemPde(PMMPDE PointerPde
)
320 /* Get the Index from the PDE */
321 Index
= ((ULONG_PTR
)PointerPde
& (SYSTEM_PD_SIZE
- 1)) / sizeof(MMPTE
);
323 /* Copy the PDE from the double-mapped system page directory */
324 SystemPde
= MmSystemPagePtes
[Index
];
325 *PointerPde
= SystemPde
;
327 /* Make sure we re-read the PDE and PTE */
328 KeMemoryBarrierWithoutFence();
330 /* Return, if we had success */
331 return (BOOLEAN
)SystemPde
.u
.Hard
.Valid
;
336 MiCheckPdeForSessionSpace(IN PVOID Address
)
340 PVOID SessionAddress
;
343 /* Is this a session PTE? */
344 if (MI_IS_SESSION_PTE(Address
))
346 /* Make sure the PDE for session space is valid */
347 PointerPde
= MiAddressToPde(MmSessionSpace
);
348 if (!PointerPde
->u
.Hard
.Valid
)
350 /* This means there's no valid session, bail out */
351 DbgPrint("MiCheckPdeForSessionSpace: No current session for PTE %p\n",
354 return STATUS_ACCESS_VIOLATION
;
357 /* Now get the session-specific page table for this address */
358 SessionAddress
= MiPteToAddress(Address
);
359 PointerPde
= MiAddressToPte(Address
);
360 if (PointerPde
->u
.Hard
.Valid
) return STATUS_WAIT_1
;
362 /* It's not valid, so find it in the page table array */
363 Index
= ((ULONG_PTR
)SessionAddress
- (ULONG_PTR
)MmSessionBase
) >> 22;
364 TempPde
.u
.Long
= MmSessionSpace
->PageTables
[Index
].u
.Long
;
365 if (TempPde
.u
.Hard
.Valid
)
367 /* The copy is valid, so swap it in */
368 InterlockedExchange((PLONG
)PointerPde
, TempPde
.u
.Long
);
369 return STATUS_WAIT_1
;
372 /* We don't seem to have allocated a page table for this address yet? */
373 DbgPrint("MiCheckPdeForSessionSpace: No Session PDE for PTE %p, %p\n",
374 PointerPde
->u
.Long
, SessionAddress
);
376 return STATUS_ACCESS_VIOLATION
;
379 /* Is the address also a session address? If not, we're done */
380 if (!MI_IS_SESSION_ADDRESS(Address
)) return STATUS_SUCCESS
;
382 /* It is, so again get the PDE for session space */
383 PointerPde
= MiAddressToPde(MmSessionSpace
);
384 if (!PointerPde
->u
.Hard
.Valid
)
386 /* This means there's no valid session, bail out */
387 DbgPrint("MiCheckPdeForSessionSpace: No current session for VA %p\n",
390 return STATUS_ACCESS_VIOLATION
;
393 /* Now get the PDE for the address itself */
394 PointerPde
= MiAddressToPde(Address
);
395 if (!PointerPde
->u
.Hard
.Valid
)
397 /* Do the swap, we should be good to go */
398 Index
= ((ULONG_PTR
)Address
- (ULONG_PTR
)MmSessionBase
) >> 22;
399 PointerPde
->u
.Long
= MmSessionSpace
->PageTables
[Index
].u
.Long
;
400 if (PointerPde
->u
.Hard
.Valid
) return STATUS_WAIT_1
;
402 /* We had not allocated a page table for this session address yet, fail! */
403 DbgPrint("MiCheckPdeForSessionSpace: No Session PDE for VA %p, %p\n",
404 PointerPde
->u
.Long
, Address
);
406 return STATUS_ACCESS_VIOLATION
;
409 /* It's valid, so there's nothing to do */
410 return STATUS_SUCCESS
;
415 MiCheckPdeForPagedPool(IN PVOID Address
)
418 NTSTATUS Status
= STATUS_SUCCESS
;
420 /* Check session PDE */
421 if (MI_IS_SESSION_ADDRESS(Address
)) return MiCheckPdeForSessionSpace(Address
);
422 if (MI_IS_SESSION_PTE(Address
)) return MiCheckPdeForSessionSpace(Address
);
425 // Check if this is a fault while trying to access the page table itself
427 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address
))
430 // Send a hint to the page fault handler that this is only a valid fault
431 // if we already detected this was access within the page table range
433 PointerPde
= (PMMPDE
)MiAddressToPte(Address
);
434 Status
= STATUS_WAIT_1
;
436 else if (Address
< MmSystemRangeStart
)
439 // This is totally illegal
441 return STATUS_ACCESS_VIOLATION
;
446 // Get the PDE for the address
448 PointerPde
= MiAddressToPde(Address
);
452 // Check if it's not valid
454 if (PointerPde
->u
.Hard
.Valid
== 0)
457 // Copy it from our double-mapped system page directory
459 InterlockedExchangePte(PointerPde
,
460 MmSystemPagePtes
[((ULONG_PTR
)PointerPde
& (SYSTEM_PD_SIZE
- 1)) / sizeof(MMPTE
)].u
.Long
);
471 MiCheckPdeForPagedPool(IN PVOID Address
)
473 return STATUS_ACCESS_VIOLATION
;
479 MiZeroPfn(IN PFN_NUMBER PageFrameNumber
)
486 /* Get the PFN for this page */
487 Pfn1
= MiGetPfnEntry(PageFrameNumber
);
490 /* Grab a system PTE we can use to zero the page */
491 ZeroPte
= MiReserveSystemPtes(1, SystemPteSpace
);
494 /* Initialize the PTE for it */
495 TempPte
= ValidKernelPte
;
496 TempPte
.u
.Hard
.PageFrameNumber
= PageFrameNumber
;
499 if (Pfn1
->u3
.e1
.CacheAttribute
== MiWriteCombined
)
501 /* Write combining, no caching */
502 MI_PAGE_DISABLE_CACHE(&TempPte
);
503 MI_PAGE_WRITE_COMBINED(&TempPte
);
505 else if (Pfn1
->u3
.e1
.CacheAttribute
== MiNonCached
)
507 /* Write through, no caching */
508 MI_PAGE_DISABLE_CACHE(&TempPte
);
509 MI_PAGE_WRITE_THROUGH(&TempPte
);
512 /* Make the system PTE valid with our PFN */
513 MI_WRITE_VALID_PTE(ZeroPte
, TempPte
);
515 /* Get the address it maps to, and zero it out */
516 ZeroAddress
= MiPteToAddress(ZeroPte
);
517 KeZeroPages(ZeroAddress
, PAGE_SIZE
);
519 /* Now get rid of it */
520 MiReleaseSystemPtes(ZeroPte
, 1, SystemPteSpace
);
525 MiResolveDemandZeroFault(IN PVOID Address
,
526 IN PMMPTE PointerPte
,
527 IN PEPROCESS Process
,
530 PFN_NUMBER PageFrameNumber
= 0;
532 BOOLEAN NeedZero
= FALSE
, HaveLock
= FALSE
;
535 DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
539 /* Must currently only be called by paging path */
540 if ((Process
> HYDRA_PROCESS
) && (OldIrql
== MM_NOIRQL
))
543 ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte
));
546 ASSERT(Process
->ForkInProgress
== NULL
);
548 /* Get process color */
549 Color
= MI_GET_NEXT_PROCESS_COLOR(Process
);
550 ASSERT(Color
!= 0xFFFFFFFF);
552 /* We'll need a zero page */
557 /* Check if we need a zero page */
558 NeedZero
= (OldIrql
!= MM_NOIRQL
);
560 /* Session-backed image views must be zeroed */
561 if ((Process
== HYDRA_PROCESS
) &&
562 ((MI_IS_SESSION_IMAGE_ADDRESS(Address
)) ||
563 ((Address
>= MiSessionViewStart
) && (Address
< MiSessionSpaceWs
))))
568 /* Hardcode unknown color */
572 /* Check if the PFN database should be acquired */
573 if (OldIrql
== MM_NOIRQL
)
575 /* Acquire it and remember we should release it after */
576 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
580 /* We either manually locked the PFN DB, or already came with it locked */
581 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
582 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
584 /* Assert we have enough pages */
585 ASSERT(MmAvailablePages
>= 32);
588 if (UserPdeFault
) MI_SET_USAGE(MI_USAGE_PAGE_TABLE
);
589 if (!UserPdeFault
) MI_SET_USAGE(MI_USAGE_DEMAND_ZERO
);
591 if (Process
) MI_SET_PROCESS2(Process
->ImageFileName
);
592 if (!Process
) MI_SET_PROCESS2("Kernel Demand 0");
594 /* Do we need a zero page? */
595 if (Color
!= 0xFFFFFFFF)
597 /* Try to get one, if we couldn't grab a free page and zero it */
598 PageFrameNumber
= MiRemoveZeroPageSafe(Color
);
599 if (!PageFrameNumber
)
601 /* We'll need a free page and zero it manually */
602 PageFrameNumber
= MiRemoveAnyPage(Color
);
608 /* Get a color, and see if we should grab a zero or non-zero page */
609 Color
= MI_GET_NEXT_COLOR();
612 /* Process or system doesn't want a zero page, grab anything */
613 PageFrameNumber
= MiRemoveAnyPage(Color
);
617 /* System wants a zero page, obtain one */
618 PageFrameNumber
= MiRemoveZeroPage(Color
);
623 MiInitializePfn(PageFrameNumber
, PointerPte
, TRUE
);
625 /* Do we have the lock? */
629 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
631 /* Update performance counters */
632 if (Process
> HYDRA_PROCESS
) Process
->NumberOfPrivatePages
++;
635 /* Increment demand zero faults */
636 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount
);
638 /* Zero the page if need be */
639 if (NeedZero
) MiZeroPfn(PageFrameNumber
);
641 /* Fault on user PDE, or fault on user PTE? */
642 if (PointerPte
<= MiHighestUserPte
)
644 /* User fault, build a user PTE */
645 MI_MAKE_HARDWARE_PTE_USER(&TempPte
,
647 PointerPte
->u
.Soft
.Protection
,
652 /* This is a user-mode PDE, create a kernel PTE for it */
653 MI_MAKE_HARDWARE_PTE(&TempPte
,
655 PointerPte
->u
.Soft
.Protection
,
659 /* Set it dirty if it's a writable page */
660 if (MI_IS_PAGE_WRITEABLE(&TempPte
)) MI_MAKE_DIRTY_PAGE(&TempPte
);
663 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
665 /* Did we manually acquire the lock */
668 /* Get the PFN entry */
669 Pfn1
= MI_PFN_ELEMENT(PageFrameNumber
);
671 /* Windows does these sanity checks */
672 ASSERT(Pfn1
->u1
.Event
== 0);
673 ASSERT(Pfn1
->u3
.e1
.PrototypePte
== 0);
679 DPRINT("Demand zero page has now been paged in\n");
680 return STATUS_PAGE_FAULT_DEMAND_ZERO
;
685 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction
,
687 IN PMMPTE PointerPte
,
688 IN PMMPTE PointerProtoPte
,
690 IN PMMPFN
* LockedProtoPfn
)
693 PMMPTE OriginalPte
, PageTablePte
;
694 ULONG_PTR Protection
;
695 PFN_NUMBER PageFrameIndex
;
697 BOOLEAN OriginalProtection
, DirtyPage
;
699 /* Must be called with an valid prototype PTE, with the PFN lock held */
700 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
701 ASSERT(PointerProtoPte
->u
.Hard
.Valid
== 1);
704 PageFrameIndex
= PFN_FROM_PTE(PointerProtoPte
);
706 /* Get the PFN entry and set it as a prototype PTE */
707 Pfn1
= MiGetPfnEntry(PageFrameIndex
);
708 Pfn1
->u3
.e1
.PrototypePte
= 1;
710 /* Increment the share count for the page table */
711 PageTablePte
= MiAddressToPte(PointerPte
);
712 Pfn2
= MiGetPfnEntry(PageTablePte
->u
.Hard
.PageFrameNumber
);
713 Pfn2
->u2
.ShareCount
++;
715 /* Check where we should be getting the protection information from */
716 if (PointerPte
->u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
)
718 /* Get the protection from the PTE, there's no real Proto PTE data */
719 Protection
= PointerPte
->u
.Soft
.Protection
;
721 /* Remember that we did not use the proto protection */
722 OriginalProtection
= FALSE
;
726 /* Get the protection from the original PTE link */
727 OriginalPte
= &Pfn1
->OriginalPte
;
728 Protection
= OriginalPte
->u
.Soft
.Protection
;
730 /* Remember that we used the original protection */
731 OriginalProtection
= TRUE
;
733 /* Check if this was a write on a read only proto */
734 if ((StoreInstruction
) && !(Protection
& MM_READWRITE
))
737 StoreInstruction
= 0;
741 /* Check if this was a write on a non-COW page */
743 if ((StoreInstruction
) && ((Protection
& MM_WRITECOPY
) != MM_WRITECOPY
))
745 /* Then the page should be marked dirty */
749 ASSERT(Pfn1
->OriginalPte
.u
.Soft
.Prototype
!= 0);
752 /* Did we get a locked incoming PFN? */
755 /* Drop a reference */
756 ASSERT((*LockedProtoPfn
)->u3
.e2
.ReferenceCount
>= 1);
757 MiDereferencePfnAndDropLockCount(*LockedProtoPfn
);
758 *LockedProtoPfn
= NULL
;
761 /* Release the PFN lock */
762 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
764 /* Remove special/caching bits */
765 Protection
&= ~MM_PROTECT_SPECIAL
;
768 if (Pfn1
->u3
.e1
.CacheAttribute
== MiWriteCombined
)
770 /* Write combining, no caching */
771 MI_PAGE_DISABLE_CACHE(&TempPte
);
772 MI_PAGE_WRITE_COMBINED(&TempPte
);
774 else if (Pfn1
->u3
.e1
.CacheAttribute
== MiNonCached
)
776 /* Write through, no caching */
777 MI_PAGE_DISABLE_CACHE(&TempPte
);
778 MI_PAGE_WRITE_THROUGH(&TempPte
);
781 /* Check if this is a kernel or user address */
782 if (Address
< MmSystemRangeStart
)
784 /* Build the user PTE */
785 MI_MAKE_HARDWARE_PTE_USER(&TempPte
, PointerPte
, Protection
, PageFrameIndex
);
789 /* Build the kernel PTE */
790 MI_MAKE_HARDWARE_PTE(&TempPte
, PointerPte
, Protection
, PageFrameIndex
);
793 /* Set the dirty flag if needed */
794 if (DirtyPage
) TempPte
.u
.Hard
.Dirty
= TRUE
;
797 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
799 /* Reset the protection if needed */
800 if (OriginalProtection
) Protection
= MM_ZERO_ACCESS
;
803 ASSERT(PointerPte
== MiAddressToPte(Address
));
804 return STATUS_SUCCESS
;
809 MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction
,
810 _In_ PVOID FaultingAddress
,
811 _In_ PMMPTE PointerPte
,
812 _In_ PEPROCESS CurrentProcess
,
813 _Inout_ KIRQL
*OldIrql
)
818 MMPTE TempPte
= *PointerPte
;
821 ULONG PageFileIndex
= TempPte
.u
.Soft
.PageFileLow
;
822 ULONG_PTR PageFileOffset
= TempPte
.u
.Soft
.PageFileHigh
;
824 /* Things we don't support yet */
825 ASSERT(CurrentProcess
> HYDRA_PROCESS
);
826 ASSERT(*OldIrql
!= MM_NOIRQL
);
828 /* We must hold the PFN lock */
829 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
831 /* Some sanity checks */
832 ASSERT(TempPte
.u
.Hard
.Valid
== 0);
833 ASSERT(TempPte
.u
.Soft
.PageFileHigh
!= 0);
834 ASSERT(TempPte
.u
.Soft
.PageFileHigh
!= MI_PTE_LOOKUP_NEEDED
);
836 /* Get any page, it will be overwritten */
837 Color
= MI_GET_NEXT_PROCESS_COLOR(CurrentProcess
);
838 Page
= MiRemoveAnyPage(Color
);
840 /* Initialize this PFN */
841 MiInitializePfn(Page
, PointerPte
, StoreInstruction
);
843 /* Sets the PFN as being in IO operation */
844 Pfn1
= MI_PFN_ELEMENT(Page
);
845 ASSERT(Pfn1
->u1
.Event
== NULL
);
846 ASSERT(Pfn1
->u3
.e1
.ReadInProgress
== 0);
847 ASSERT(Pfn1
->u3
.e1
.WriteInProgress
== 0);
849 KeInitializeEvent(&Event
, NotificationEvent
, FALSE
);
850 Pfn1
->u1
.Event
= &Event
;
851 Pfn1
->u3
.e1
.ReadInProgress
= 1;
853 /* We must write the PTE now as the PFN lock will be released while performing the IO operation */
854 TempPte
.u
.Soft
.Transition
= 1;
855 TempPte
.u
.Soft
.PageFileLow
= 0;
856 TempPte
.u
.Soft
.Prototype
= 0;
857 TempPte
.u
.Trans
.PageFrameNumber
= Page
;
859 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
861 /* Release the PFN lock while we proceed */
862 KeReleaseQueuedSpinLock(LockQueuePfnLock
, *OldIrql
);
864 /* Do the paging IO */
865 Status
= MiReadPageFile(Page
, PageFileIndex
, PageFileOffset
);
867 /* Lock the PFN database again */
868 *OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
870 /* Nobody should have changed that while we were not looking */
871 ASSERT(Pfn1
->u1
.Event
== &Event
);
872 ASSERT(Pfn1
->u3
.e1
.ReadInProgress
== 1);
873 ASSERT(Pfn1
->u3
.e1
.WriteInProgress
== 0);
875 if (!NT_SUCCESS(Status
))
879 Pfn1
->u4
.InPageError
= 1;
880 Pfn1
->u1
.ReadStatus
= Status
;
883 /* This is now a nice and normal PFN */
884 Pfn1
->u1
.Event
= NULL
;
885 Pfn1
->u3
.e1
.ReadInProgress
= 0;
887 /* And the PTE can finally be valid */
888 MI_MAKE_HARDWARE_PTE(&TempPte
, PointerPte
, TempPte
.u
.Trans
.Protection
, Page
);
889 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
891 /* Waiters gonna wait */
892 KeSetEvent(&Event
, IO_NO_INCREMENT
, FALSE
);
899 MiResolveTransitionFault(IN PVOID FaultingAddress
,
900 IN PMMPTE PointerPte
,
901 IN PEPROCESS CurrentProcess
,
903 OUT PVOID
*InPageBlock
)
905 PFN_NUMBER PageFrameIndex
;
908 PMMPTE PointerToPteForProtoPage
;
909 DPRINT("Transition fault on 0x%p with PTE 0x%p in process %s\n",
910 FaultingAddress
, PointerPte
, CurrentProcess
->ImageFileName
);
912 /* Windowss does this check */
913 ASSERT(*InPageBlock
== NULL
);
915 /* ARM3 doesn't support this path */
916 ASSERT(OldIrql
!= MM_NOIRQL
);
918 /* Capture the PTE and make sure it's in transition format */
919 TempPte
= *PointerPte
;
920 ASSERT((TempPte
.u
.Soft
.Valid
== 0) &&
921 (TempPte
.u
.Soft
.Prototype
== 0) &&
922 (TempPte
.u
.Soft
.Transition
== 1));
924 /* Get the PFN and the PFN entry */
925 PageFrameIndex
= TempPte
.u
.Trans
.PageFrameNumber
;
926 DPRINT("Transition PFN: %lx\n", PageFrameIndex
);
927 Pfn1
= MiGetPfnEntry(PageFrameIndex
);
929 /* One more transition fault! */
930 InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount
);
932 /* This is from ARM3 -- Windows normally handles this here */
933 ASSERT(Pfn1
->u4
.InPageError
== 0);
935 /* See if we should wait before terminating the fault */
936 if (Pfn1
->u3
.e1
.ReadInProgress
== 1)
938 DPRINT1("The page is currently being read!\n");
939 ASSERT(Pfn1
->u1
.Event
!= NULL
);
940 *InPageBlock
= Pfn1
->u1
.Event
;
941 if (PointerPte
== Pfn1
->PteAddress
)
943 DPRINT1("And this if for this particular PTE.\n");
944 /* The PTE will be made valid by the thread serving the fault */
945 return STATUS_SUCCESS
; // FIXME: Maybe something more descriptive
949 /* Windows checks there's some free pages and this isn't an in-page error */
950 ASSERT(MmAvailablePages
> 0);
951 ASSERT(Pfn1
->u4
.InPageError
== 0);
953 /* ReactOS checks for this */
954 ASSERT(MmAvailablePages
> 32);
956 /* Was this a transition page in the valid list, or free/zero list? */
957 if (Pfn1
->u3
.e1
.PageLocation
== ActiveAndValid
)
959 /* All Windows does here is a bunch of sanity checks */
960 DPRINT("Transition in active list\n");
961 ASSERT((Pfn1
->PteAddress
>= MiAddressToPte(MmPagedPoolStart
)) &&
962 (Pfn1
->PteAddress
<= MiAddressToPte(MmPagedPoolEnd
)));
963 ASSERT(Pfn1
->u2
.ShareCount
!= 0);
964 ASSERT(Pfn1
->u3
.e2
.ReferenceCount
!= 0);
968 /* Otherwise, the page is removed from its list */
969 DPRINT("Transition page in free/zero list\n");
970 MiUnlinkPageFromList(Pfn1
);
971 MiReferenceUnusedPageAndBumpLockCount(Pfn1
);
974 /* At this point, there should no longer be any in-page errors */
975 ASSERT(Pfn1
->u4
.InPageError
== 0);
977 /* Check if this was a PFN with no more share references */
978 if (Pfn1
->u2
.ShareCount
== 0) MiDropLockCount(Pfn1
);
980 /* Bump the share count and make the page valid */
981 Pfn1
->u2
.ShareCount
++;
982 Pfn1
->u3
.e1
.PageLocation
= ActiveAndValid
;
984 /* Prototype PTEs are in paged pool, which itself might be in transition */
985 if (FaultingAddress
>= MmSystemRangeStart
)
987 /* Check if this is a paged pool PTE in transition state */
988 PointerToPteForProtoPage
= MiAddressToPte(PointerPte
);
989 TempPte
= *PointerToPteForProtoPage
;
990 if ((TempPte
.u
.Hard
.Valid
== 0) && (TempPte
.u
.Soft
.Transition
== 1))
992 /* This isn't yet supported */
993 DPRINT1("Double transition fault not yet supported\n");
998 /* Build the transition PTE -- maybe a macro? */
999 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1000 ASSERT(PointerPte
->u
.Trans
.Prototype
== 0);
1001 ASSERT(PointerPte
->u
.Trans
.Transition
== 1);
1002 TempPte
.u
.Long
= (PointerPte
->u
.Long
& ~0xFFF) |
1003 (MmProtectToPteMask
[PointerPte
->u
.Trans
.Protection
]) |
1004 MiDetermineUserGlobalPteMask(PointerPte
);
1006 /* Is the PTE writeable? */
1007 if (((Pfn1
->u3
.e1
.Modified
) && (TempPte
.u
.Hard
.Write
)) &&
1008 (TempPte
.u
.Hard
.CopyOnWrite
== 0))
1011 TempPte
.u
.Hard
.Dirty
= TRUE
;
1016 TempPte
.u
.Hard
.Dirty
= FALSE
;
1019 /* Write the valid PTE */
1020 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
1022 /* Return success */
1023 return STATUS_PAGE_FAULT_TRANSITION
;
1028 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction
,
1030 IN PMMPTE PointerPte
,
1031 IN PMMPTE PointerProtoPte
,
1032 IN OUT PMMPFN
*OutPfn
,
1033 OUT PVOID
*PageFileData
,
1034 OUT PMMPTE PteValue
,
1035 IN PEPROCESS Process
,
1037 IN PVOID TrapInformation
)
1039 MMPTE TempPte
, PteContents
;
1041 PFN_NUMBER PageFrameIndex
;
1043 PVOID InPageBlock
= NULL
;
1045 /* Must be called with an invalid, prototype PTE, with the PFN lock held */
1046 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1047 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1048 ASSERT(PointerPte
->u
.Soft
.Prototype
== 1);
1050 /* Read the prototype PTE and check if it's valid */
1051 TempPte
= *PointerProtoPte
;
1052 if (TempPte
.u
.Hard
.Valid
== 1)
1054 /* One more user of this mapped page */
1055 PageFrameIndex
= PFN_FROM_PTE(&TempPte
);
1056 Pfn1
= MiGetPfnEntry(PageFrameIndex
);
1057 Pfn1
->u2
.ShareCount
++;
1059 /* Call it a transition */
1060 InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount
);
1062 /* Complete the prototype PTE fault -- this will release the PFN lock */
1063 return MiCompleteProtoPteFault(StoreInstruction
,
1071 /* Make sure there's some protection mask */
1072 if (TempPte
.u
.Long
== 0)
1074 /* Release the lock */
1075 DPRINT1("Access on reserved section?\n");
1076 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
1077 return STATUS_ACCESS_VIOLATION
;
1080 /* There is no such thing as a decommitted prototype PTE */
1081 NT_ASSERT(TempPte
.u
.Long
!= MmDecommittedPte
.u
.Long
);
1083 /* Check for access rights on the PTE proper */
1084 PteContents
= *PointerPte
;
1085 if (PteContents
.u
.Soft
.PageFileHigh
!= MI_PTE_LOOKUP_NEEDED
)
1087 if (!PteContents
.u
.Proto
.ReadOnly
)
1089 /* Check for page acess in software */
1090 Status
= MiAccessCheck(PointerProtoPte
,
1093 TempPte
.u
.Soft
.Protection
,
1096 ASSERT(Status
== STATUS_SUCCESS
);
1098 /* Check for copy on write page */
1099 if ((TempPte
.u
.Soft
.Protection
& MM_WRITECOPY
) == MM_WRITECOPY
)
1101 /* Not yet supported */
1108 /* Check for copy on write page */
1109 if ((PteContents
.u
.Soft
.Protection
& MM_WRITECOPY
) == MM_WRITECOPY
)
1111 /* Not yet supported */
1116 /* Check for clone PTEs */
1117 if (PointerPte
<= MiHighestUserPte
) ASSERT(Process
->CloneRoot
== NULL
);
1119 /* We don't support mapped files yet */
1120 ASSERT(TempPte
.u
.Soft
.Prototype
== 0);
1122 /* We might however have transition PTEs */
1123 if (TempPte
.u
.Soft
.Transition
== 1)
1125 /* Resolve the transition fault */
1126 ASSERT(OldIrql
!= MM_NOIRQL
);
1127 Status
= MiResolveTransitionFault(Address
,
1132 ASSERT(NT_SUCCESS(Status
));
1136 /* We also don't support paged out pages */
1137 ASSERT(TempPte
.u
.Soft
.PageFileHigh
== 0);
1139 /* Resolve the demand zero fault */
1140 Status
= MiResolveDemandZeroFault(Address
,
1144 ASSERT(NT_SUCCESS(Status
));
1147 /* Complete the prototype PTE fault -- this will release the PFN lock */
1148 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1149 return MiCompleteProtoPteFault(StoreInstruction
,
1159 MiDispatchFault(IN BOOLEAN StoreInstruction
,
1161 IN PMMPTE PointerPte
,
1162 IN PMMPTE PointerProtoPte
,
1163 IN BOOLEAN Recursive
,
1164 IN PEPROCESS Process
,
1165 IN PVOID TrapInformation
,
1169 KIRQL OldIrql
, LockIrql
;
1171 PMMPTE SuperProtoPte
;
1172 PMMPFN Pfn1
, OutPfn
= NULL
;
1173 PFN_NUMBER PageFrameIndex
;
1174 PFN_COUNT PteCount
, ProcessedPtes
;
1175 DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
1179 /* Make sure the addresses are ok */
1180 ASSERT(PointerPte
== MiAddressToPte(Address
));
1183 // Make sure APCs are off and we're not at dispatch
1185 OldIrql
= KeGetCurrentIrql();
1186 ASSERT(OldIrql
<= APC_LEVEL
);
1187 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1190 // Grab a copy of the PTE
1192 TempPte
= *PointerPte
;
1194 /* Do we have a prototype PTE? */
1195 if (PointerProtoPte
)
1197 /* This should never happen */
1198 ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte
));
1200 /* Check if this is a kernel-mode address */
1201 SuperProtoPte
= MiAddressToPte(PointerProtoPte
);
1202 if (Address
>= MmSystemRangeStart
)
1204 /* Lock the PFN database */
1205 LockIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1207 /* Has the PTE been made valid yet? */
1208 if (!SuperProtoPte
->u
.Hard
.Valid
)
1212 else if (PointerPte
->u
.Hard
.Valid
== 1)
1217 /* Resolve the fault -- this will release the PFN lock */
1218 Status
= MiResolveProtoPteFault(StoreInstruction
,
1228 ASSERT(Status
== STATUS_SUCCESS
);
1230 /* Complete this as a transition fault */
1231 ASSERT(OldIrql
== KeGetCurrentIrql());
1232 ASSERT(OldIrql
<= APC_LEVEL
);
1233 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1238 /* We only handle the lookup path */
1239 ASSERT(PointerPte
->u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
);
1241 /* Is there a non-image VAD? */
1243 (Vad
->u
.VadFlags
.VadType
!= VadImageMap
) &&
1244 !(Vad
->u2
.VadFlags2
.ExtendableFile
))
1246 /* One day, ReactOS will cluster faults */
1247 ASSERT(Address
<= MM_HIGHEST_USER_ADDRESS
);
1248 DPRINT("Should cluster fault, but won't\n");
1251 /* Only one PTE to handle for now */
1255 /* Lock the PFN database */
1256 LockIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1258 /* We only handle the valid path */
1259 ASSERT(SuperProtoPte
->u
.Hard
.Valid
== 1);
1261 /* Capture the PTE */
1262 TempPte
= *PointerProtoPte
;
1264 /* Loop to handle future case of clustered faults */
1267 /* For our current usage, this should be true */
1268 if (TempPte
.u
.Hard
.Valid
== 1)
1270 /* Bump the share count on the PTE */
1271 PageFrameIndex
= PFN_FROM_PTE(&TempPte
);
1272 Pfn1
= MI_PFN_ELEMENT(PageFrameIndex
);
1273 Pfn1
->u2
.ShareCount
++;
1275 else if ((TempPte
.u
.Soft
.Prototype
== 0) &&
1276 (TempPte
.u
.Soft
.Transition
== 1))
1278 /* This is a standby page, bring it back from the cache */
1279 PageFrameIndex
= TempPte
.u
.Trans
.PageFrameNumber
;
1280 DPRINT("oooh, shiny, a soft fault! 0x%lx\n", PageFrameIndex
);
1281 Pfn1
= MI_PFN_ELEMENT(PageFrameIndex
);
1282 ASSERT(Pfn1
->u3
.e1
.PageLocation
!= ActiveAndValid
);
1284 /* Should not yet happen in ReactOS */
1285 ASSERT(Pfn1
->u3
.e1
.ReadInProgress
== 0);
1286 ASSERT(Pfn1
->u4
.InPageError
== 0);
1289 MiUnlinkPageFromList(Pfn1
);
1291 /* Bump its reference count */
1292 ASSERT(Pfn1
->u2
.ShareCount
== 0);
1293 InterlockedIncrement16((PSHORT
)&Pfn1
->u3
.e2
.ReferenceCount
);
1294 Pfn1
->u2
.ShareCount
++;
1296 /* Make it valid again */
1297 /* This looks like another macro.... */
1298 Pfn1
->u3
.e1
.PageLocation
= ActiveAndValid
;
1299 ASSERT(PointerProtoPte
->u
.Hard
.Valid
== 0);
1300 ASSERT(PointerProtoPte
->u
.Trans
.Prototype
== 0);
1301 ASSERT(PointerProtoPte
->u
.Trans
.Transition
== 1);
1302 TempPte
.u
.Long
= (PointerProtoPte
->u
.Long
& ~0xFFF) |
1303 MmProtectToPteMask
[PointerProtoPte
->u
.Trans
.Protection
];
1304 TempPte
.u
.Hard
.Valid
= 1;
1305 TempPte
.u
.Hard
.Accessed
= 1;
1307 /* Is the PTE writeable? */
1308 if (((Pfn1
->u3
.e1
.Modified
) && (TempPte
.u
.Hard
.Write
)) &&
1309 (TempPte
.u
.Hard
.CopyOnWrite
== 0))
1312 TempPte
.u
.Hard
.Dirty
= TRUE
;
1317 TempPte
.u
.Hard
.Dirty
= FALSE
;
1320 /* Write the valid PTE */
1321 MI_WRITE_VALID_PTE(PointerProtoPte
, TempPte
);
1322 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1326 /* Page is invalid, get out of the loop */
1330 /* One more done, was it the last? */
1331 if (++ProcessedPtes
== PteCount
)
1333 /* Complete the fault */
1334 MiCompleteProtoPteFault(StoreInstruction
,
1341 /* THIS RELEASES THE PFN LOCK! */
1345 /* No clustered faults yet */
1349 /* Did we resolve the fault? */
1352 /* Bump the transition count */
1353 InterlockedExchangeAddSizeT(&KeGetCurrentPrcb()->MmTransitionCount
, ProcessedPtes
);
1356 /* Loop all the processing we did */
1357 ASSERT(ProcessedPtes
== 0);
1359 /* Complete this as a transition fault */
1360 ASSERT(OldIrql
== KeGetCurrentIrql());
1361 ASSERT(OldIrql
<= APC_LEVEL
);
1362 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1363 return STATUS_PAGE_FAULT_TRANSITION
;
1366 /* We did not -- PFN lock is still held, prepare to resolve prototype PTE fault */
1367 OutPfn
= MI_PFN_ELEMENT(SuperProtoPte
->u
.Hard
.PageFrameNumber
);
1368 MiReferenceUsedPageAndBumpLockCount(OutPfn
);
1369 ASSERT(OutPfn
->u3
.e2
.ReferenceCount
> 1);
1370 ASSERT(PointerPte
->u
.Hard
.Valid
== 0);
1372 /* Resolve the fault -- this will release the PFN lock */
1373 Status
= MiResolveProtoPteFault(StoreInstruction
,
1383 //ASSERT(Status != STATUS_ISSUE_PAGING_IO);
1384 //ASSERT(Status != STATUS_REFAULT);
1385 //ASSERT(Status != STATUS_PTE_CHANGED);
1387 /* Did the routine clean out the PFN or should we? */
1390 /* We had a locked PFN, so acquire the PFN lock to dereference it */
1391 ASSERT(PointerProtoPte
!= NULL
);
1392 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1394 /* Dereference the locked PFN */
1395 MiDereferencePfnAndDropLockCount(OutPfn
);
1396 ASSERT(OutPfn
->u3
.e2
.ReferenceCount
>= 1);
1398 /* And now release the lock */
1399 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
1402 /* Complete this as a transition fault */
1403 ASSERT(OldIrql
== KeGetCurrentIrql());
1404 ASSERT(OldIrql
<= APC_LEVEL
);
1405 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1410 /* Is this a transition PTE */
1411 if (TempPte
.u
.Soft
.Transition
)
1413 PVOID InPageBlock
= NULL
;
1414 /* Lock the PFN database */
1415 LockIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1418 Status
= MiResolveTransitionFault(Address
, PointerPte
, Process
, LockIrql
, &InPageBlock
);
1420 NT_ASSERT(NT_SUCCESS(Status
));
1422 /* And now release the lock and leave*/
1423 KeReleaseQueuedSpinLock(LockQueuePfnLock
, LockIrql
);
1425 if (InPageBlock
!= NULL
)
1427 /* The page is being paged in by another process */
1428 KeWaitForSingleObject(InPageBlock
, WrPageIn
, KernelMode
, FALSE
, NULL
);
1431 ASSERT(OldIrql
== KeGetCurrentIrql());
1432 ASSERT(OldIrql
<= APC_LEVEL
);
1433 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1437 /* Should we page the data back in ? */
1438 if (TempPte
.u
.Soft
.PageFileHigh
!= 0)
1440 /* Lock the PFN database */
1441 LockIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1444 Status
= MiResolvePageFileFault(StoreInstruction
, Address
, PointerPte
, Process
, &LockIrql
);
1446 /* And now release the lock and leave*/
1447 KeReleaseQueuedSpinLock(LockQueuePfnLock
, LockIrql
);
1449 ASSERT(OldIrql
== KeGetCurrentIrql());
1450 ASSERT(OldIrql
<= APC_LEVEL
);
1451 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1456 // The PTE must be invalid but not completely empty. It must also not be a
1457 // prototype a transition or a paged-out PTE as those scenarii should've been handled above.
1458 // These are all Windows checks
1460 ASSERT(TempPte
.u
.Hard
.Valid
== 0);
1461 ASSERT(TempPte
.u
.Soft
.Prototype
== 0);
1462 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
1463 ASSERT(TempPte
.u
.Soft
.PageFileHigh
== 0);
1464 ASSERT(TempPte
.u
.Long
!= 0);
1467 // If we got this far, the PTE can only be a demand zero PTE, which is what
1468 // we want. Go handle it!
1470 Status
= MiResolveDemandZeroFault(Address
,
1474 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1475 if (NT_SUCCESS(Status
))
1478 // Make sure we're returning in a sane state and pass the status down
1480 ASSERT(OldIrql
== KeGetCurrentIrql());
1481 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
1486 // Generate an access fault
1488 return STATUS_ACCESS_VIOLATION
;
1493 MmArmAccessFault(IN BOOLEAN StoreInstruction
,
1495 IN KPROCESSOR_MODE Mode
,
1496 IN PVOID TrapInformation
)
1498 KIRQL OldIrql
= KeGetCurrentIrql(), LockIrql
;
1499 PMMPTE ProtoPte
= NULL
;
1500 PMMPTE PointerPte
= MiAddressToPte(Address
);
1501 PMMPDE PointerPde
= MiAddressToPde(Address
);
1502 #if (_MI_PAGING_LEVELS >= 3)
1503 PMMPDE PointerPpe
= MiAddressToPpe(Address
);
1504 #if (_MI_PAGING_LEVELS == 4)
1505 PMMPDE PointerPxe
= MiAddressToPxe(Address
);
1509 PETHREAD CurrentThread
;
1510 PEPROCESS CurrentProcess
;
1512 PMMSUPPORT WorkingSet
;
1513 ULONG ProtectionCode
;
1515 PFN_NUMBER PageFrameIndex
;
1517 BOOLEAN IsSessionAddress
;
1519 DPRINT("ARM3 FAULT AT: %p\n", Address
);
1521 /* Check for page fault on high IRQL */
1522 if (OldIrql
> APC_LEVEL
)
1524 #if (_MI_PAGING_LEVELS < 3)
1525 /* Could be a page table for paged pool, which we'll allow */
1526 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address
)) MiSynchronizeSystemPde((PMMPDE
)PointerPte
);
1527 MiCheckPdeForPagedPool(Address
);
1529 /* Check if any of the top-level pages are invalid */
1531 #if (_MI_PAGING_LEVELS == 4)
1532 (PointerPxe
->u
.Hard
.Valid
== 0) ||
1534 #if (_MI_PAGING_LEVELS >= 3)
1535 (PointerPpe
->u
.Hard
.Valid
== 0) ||
1537 (PointerPde
->u
.Hard
.Valid
== 0) ||
1538 (PointerPte
->u
.Hard
.Valid
== 0))
1540 /* This fault is not valid, print out some debugging help */
1541 DbgPrint("MM:***PAGE FAULT AT IRQL > 1 Va %p, IRQL %lx\n",
1544 if (TrapInformation
)
1546 PKTRAP_FRAME TrapFrame
= TrapInformation
;
1548 DbgPrint("MM:***EIP %p, EFL %p\n", TrapFrame
->Eip
, TrapFrame
->EFlags
);
1549 DbgPrint("MM:***EAX %p, ECX %p EDX %p\n", TrapFrame
->Eax
, TrapFrame
->Ecx
, TrapFrame
->Edx
);
1550 DbgPrint("MM:***EBX %p, ESI %p EDI %p\n", TrapFrame
->Ebx
, TrapFrame
->Esi
, TrapFrame
->Edi
);
1551 #elif defined(_M_AMD64)
1552 DbgPrint("MM:***RIP %p, EFL %p\n", TrapFrame
->Rip
, TrapFrame
->EFlags
);
1553 DbgPrint("MM:***RAX %p, RCX %p RDX %p\n", TrapFrame
->Rax
, TrapFrame
->Rcx
, TrapFrame
->Rdx
);
1554 DbgPrint("MM:***RBX %p, RSI %p RDI %p\n", TrapFrame
->Rbx
, TrapFrame
->Rsi
, TrapFrame
->Rdi
);
1558 /* Tell the trap handler to fail */
1559 return STATUS_IN_PAGE_ERROR
| 0x10000000;
1562 /* Not yet implemented in ReactOS */
1563 ASSERT(MI_IS_PAGE_LARGE(PointerPde
) == FALSE
);
1564 ASSERT(((StoreInstruction
) && (PointerPte
->u
.Hard
.CopyOnWrite
)) == FALSE
);
1566 /* Check if this was a write */
1567 if (StoreInstruction
)
1569 /* Was it to a read-only page? */
1570 Pfn1
= MI_PFN_ELEMENT(PointerPte
->u
.Hard
.PageFrameNumber
);
1571 if (!(PointerPte
->u
.Long
& PTE_READWRITE
) &&
1572 !(Pfn1
->OriginalPte
.u
.Soft
.Protection
& MM_READWRITE
))
1574 /* Crash with distinguished bugcheck code */
1575 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
1578 (ULONG_PTR
)TrapInformation
,
1583 /* Nothing is actually wrong */
1584 DPRINT1("Fault at IRQL %u is ok (%p)\n", OldIrql
, Address
);
1585 return STATUS_SUCCESS
;
1588 /* Check for kernel fault address */
1589 if (Address
>= MmSystemRangeStart
)
1591 /* Bail out, if the fault came from user mode */
1592 if (Mode
== UserMode
) return STATUS_ACCESS_VIOLATION
;
1594 #if (_MI_PAGING_LEVELS == 2)
1595 if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address
)) MiSynchronizeSystemPde((PMMPDE
)PointerPte
);
1596 MiCheckPdeForPagedPool(Address
);
1599 /* Check if the higher page table entries are invalid */
1601 #if (_MI_PAGING_LEVELS == 4)
1602 /* AMD64 system, check if PXE is invalid */
1603 (PointerPxe
->u
.Hard
.Valid
== 0) ||
1605 #if (_MI_PAGING_LEVELS >= 3)
1606 /* PAE/AMD64 system, check if PPE is invalid */
1607 (PointerPpe
->u
.Hard
.Valid
== 0) ||
1609 /* Always check if the PDE is valid */
1610 (PointerPde
->u
.Hard
.Valid
== 0))
1612 /* PXE/PPE/PDE (still) not valid, kill the system */
1613 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
1616 (ULONG_PTR
)TrapInformation
,
1620 /* Not handling session faults yet */
1621 IsSessionAddress
= MI_IS_SESSION_ADDRESS(Address
);
1623 /* The PDE is valid, so read the PTE */
1624 TempPte
= *PointerPte
;
1625 if (TempPte
.u
.Hard
.Valid
== 1)
1627 /* Check if this was system space or session space */
1628 if (!IsSessionAddress
)
1630 /* Check if the PTE is still valid under PFN lock */
1631 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
1632 TempPte
= *PointerPte
;
1633 if (TempPte
.u
.Hard
.Valid
)
1635 /* Check if this was a write */
1636 if (StoreInstruction
)
1638 /* Was it to a read-only page? */
1639 Pfn1
= MI_PFN_ELEMENT(PointerPte
->u
.Hard
.PageFrameNumber
);
1640 if (!(PointerPte
->u
.Long
& PTE_READWRITE
) &&
1641 !(Pfn1
->OriginalPte
.u
.Soft
.Protection
& MM_READWRITE
))
1643 /* Crash with distinguished bugcheck code */
1644 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
1647 (ULONG_PTR
)TrapInformation
,
1653 /* Release PFN lock and return all good */
1654 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
1655 return STATUS_SUCCESS
;
1658 #if (_MI_PAGING_LEVELS == 2)
1659 /* Check if this was a session PTE that needs to remap the session PDE */
1660 if (MI_IS_SESSION_PTE(Address
))
1662 /* Do the remapping */
1663 Status
= MiCheckPdeForSessionSpace(Address
);
1664 if (!NT_SUCCESS(Status
))
1666 /* It failed, this address is invalid */
1667 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
1670 (ULONG_PTR
)TrapInformation
,
1676 _WARN("Session space stuff is not implemented yet!")
1680 /* Check for a fault on the page table or hyperspace */
1681 if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address
))
1683 #if (_MI_PAGING_LEVELS < 3)
1684 /* Windows does this check but I don't understand why -- it's done above! */
1685 ASSERT(MiCheckPdeForPagedPool(Address
) != STATUS_WAIT_1
);
1687 /* Handle this as a user mode fault */
1691 /* Get the current thread */
1692 CurrentThread
= PsGetCurrentThread();
1694 /* What kind of address is this */
1695 if (!IsSessionAddress
)
1697 /* Use the system working set */
1698 WorkingSet
= &MmSystemCacheWs
;
1699 CurrentProcess
= NULL
;
1701 /* Make sure we don't have a recursive working set lock */
1702 if ((CurrentThread
->OwnsProcessWorkingSetExclusive
) ||
1703 (CurrentThread
->OwnsProcessWorkingSetShared
) ||
1704 (CurrentThread
->OwnsSystemWorkingSetExclusive
) ||
1705 (CurrentThread
->OwnsSystemWorkingSetShared
) ||
1706 (CurrentThread
->OwnsSessionWorkingSetExclusive
) ||
1707 (CurrentThread
->OwnsSessionWorkingSetShared
))
1710 return STATUS_IN_PAGE_ERROR
| 0x10000000;
1715 /* Use the session process and working set */
1716 CurrentProcess
= HYDRA_PROCESS
;
1717 WorkingSet
= &MmSessionSpace
->GlobalVirtualAddress
->Vm
;
1719 /* Make sure we don't have a recursive working set lock */
1720 if ((CurrentThread
->OwnsSessionWorkingSetExclusive
) ||
1721 (CurrentThread
->OwnsSessionWorkingSetShared
))
1724 return STATUS_IN_PAGE_ERROR
| 0x10000000;
1728 /* Acquire the working set lock */
1729 KeRaiseIrql(APC_LEVEL
, &LockIrql
);
1730 MiLockWorkingSet(CurrentThread
, WorkingSet
);
1732 /* Re-read PTE now that we own the lock */
1733 TempPte
= *PointerPte
;
1734 if (TempPte
.u
.Hard
.Valid
== 1)
1736 /* Check if this was a write */
1737 if (StoreInstruction
)
1739 /* Was it to a read-only page that is not copy on write? */
1740 Pfn1
= MI_PFN_ELEMENT(PointerPte
->u
.Hard
.PageFrameNumber
);
1741 if (!(TempPte
.u
.Long
& PTE_READWRITE
) &&
1742 !(Pfn1
->OriginalPte
.u
.Soft
.Protection
& MM_READWRITE
) &&
1743 !(TempPte
.u
.Hard
.CopyOnWrite
))
1745 /* Case not yet handled */
1746 ASSERT(!IsSessionAddress
);
1748 /* Crash with distinguished bugcheck code */
1749 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
1752 (ULONG_PTR
)TrapInformation
,
1757 /* Check for read-only write in session space */
1758 if ((IsSessionAddress
) &&
1759 (StoreInstruction
) &&
1760 !(TempPte
.u
.Hard
.Write
))
1763 ASSERT(MI_IS_SESSION_IMAGE_ADDRESS(Address
));
1766 if (TempPte
.u
.Hard
.CopyOnWrite
== 0)
1768 /* Then this is not allowed */
1769 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
1771 (ULONG_PTR
)TempPte
.u
.Long
,
1772 (ULONG_PTR
)TrapInformation
,
1776 /* Otherwise, handle COW */
1780 /* Release the working set */
1781 MiUnlockWorkingSet(CurrentThread
, WorkingSet
);
1782 KeLowerIrql(LockIrql
);
1784 /* Otherwise, the PDE was probably invalid, and all is good now */
1785 return STATUS_SUCCESS
;
1788 /* Check one kind of prototype PTE */
1789 if (TempPte
.u
.Soft
.Prototype
)
1791 /* Make sure protected pool is on, and that this is a pool address */
1792 if ((MmProtectFreedNonPagedPool
) &&
1793 (((Address
>= MmNonPagedPoolStart
) &&
1794 (Address
< (PVOID
)((ULONG_PTR
)MmNonPagedPoolStart
+
1795 MmSizeOfNonPagedPoolInBytes
))) ||
1796 ((Address
>= MmNonPagedPoolExpansionStart
) &&
1797 (Address
< MmNonPagedPoolEnd
))))
1799 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
1800 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL
,
1807 /* Get the prototype PTE! */
1808 ProtoPte
= MiProtoPteToPte(&TempPte
);
1810 /* Do we need to locate the prototype PTE in session space? */
1811 if ((IsSessionAddress
) &&
1812 (TempPte
.u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
))
1814 /* Yep, go find it as well as the VAD for it */
1815 ProtoPte
= MiCheckVirtualAddress(Address
,
1818 ASSERT(ProtoPte
!= NULL
);
1823 /* We don't implement transition PTEs */
1824 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
1826 /* Check for no-access PTE */
1827 if (TempPte
.u
.Soft
.Protection
== MM_NOACCESS
)
1829 /* Bugcheck the system! */
1830 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
1833 (ULONG_PTR
)TrapInformation
,
1837 /* Check for no protecton at all */
1838 if (TempPte
.u
.Soft
.Protection
== MM_ZERO_ACCESS
)
1840 /* Bugcheck the system! */
1841 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA
,
1844 (ULONG_PTR
)TrapInformation
,
1849 /* Check for demand page */
1850 if ((StoreInstruction
) &&
1852 !(IsSessionAddress
) &&
1853 !(TempPte
.u
.Hard
.Valid
))
1855 /* Get the protection code */
1856 ASSERT(TempPte
.u
.Soft
.Transition
== 0);
1857 if (!(TempPte
.u
.Soft
.Protection
& MM_READWRITE
))
1859 /* Bugcheck the system! */
1860 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY
,
1863 (ULONG_PTR
)TrapInformation
,
1868 /* Now do the real fault handling */
1869 Status
= MiDispatchFault(StoreInstruction
,
1878 /* Release the working set */
1879 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1880 MiUnlockWorkingSet(CurrentThread
, WorkingSet
);
1881 KeLowerIrql(LockIrql
);
1884 DPRINT("Fault resolved with status: %lx\n", Status
);
1888 /* This is a user fault */
1890 CurrentThread
= PsGetCurrentThread();
1891 CurrentProcess
= (PEPROCESS
)CurrentThread
->Tcb
.ApcState
.Process
;
1893 /* Lock the working set */
1894 MiLockProcessWorkingSet(CurrentProcess
, CurrentThread
);
1896 #if (_MI_PAGING_LEVELS == 4)
1897 // Note to Timo: You should call MiCheckVirtualAddress and also check if it's zero pte
1898 // also this is missing the page count increment
1899 /* Check if the PXE is valid */
1900 if (PointerPxe
->u
.Hard
.Valid
== 0)
1902 /* Right now, we only handle scenarios where the PXE is totally empty */
1903 ASSERT(PointerPxe
->u
.Long
== 0);
1905 /* Resolve a demand zero fault */
1906 Status
= MiResolveDemandZeroFault(PointerPpe
,
1911 /* We should come back with a valid PXE */
1912 ASSERT(PointerPxe
->u
.Hard
.Valid
== 1);
1916 #if (_MI_PAGING_LEVELS >= 3)
1917 // Note to Timo: You should call MiCheckVirtualAddress and also check if it's zero pte
1918 // also this is missing the page count increment
1919 /* Check if the PPE is valid */
1920 if (PointerPpe
->u
.Hard
.Valid
== 0)
1922 /* Right now, we only handle scenarios where the PPE is totally empty */
1923 ASSERT(PointerPpe
->u
.Long
== 0);
1925 /* Resolve a demand zero fault */
1926 Status
= MiResolveDemandZeroFault(PointerPde
,
1931 /* We should come back with a valid PPE */
1932 ASSERT(PointerPpe
->u
.Hard
.Valid
== 1);
1936 /* Check if the PDE is valid */
1937 if (PointerPde
->u
.Hard
.Valid
== 0)
1939 /* Right now, we only handle scenarios where the PDE is totally empty */
1940 ASSERT(PointerPde
->u
.Long
== 0);
1942 /* And go dispatch the fault on the PDE. This should handle the demand-zero */
1944 UserPdeFault
= TRUE
;
1946 MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
1947 if (ProtectionCode
== MM_NOACCESS
)
1949 #if (_MI_PAGING_LEVELS == 2)
1950 /* Could be a page table for paged pool */
1951 MiCheckPdeForPagedPool(Address
);
1953 /* Has the code above changed anything -- is this now a valid PTE? */
1954 Status
= (PointerPde
->u
.Hard
.Valid
== 1) ? STATUS_SUCCESS
: STATUS_ACCESS_VIOLATION
;
1956 /* Either this was a bogus VA or we've fixed up a paged pool PDE */
1957 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
1961 /* Write a demand-zero PDE */
1962 MI_WRITE_INVALID_PTE(PointerPde
, DemandZeroPde
);
1964 /* Dispatch the fault */
1965 Status
= MiDispatchFault(TRUE
,
1970 PsGetCurrentProcess(),
1974 UserPdeFault
= FALSE
;
1976 /* We should come back with APCs enabled, and with a valid PDE */
1977 ASSERT(KeAreAllApcsDisabled() == TRUE
);
1978 ASSERT(PointerPde
->u
.Hard
.Valid
== 1);
1982 /* Not yet implemented in ReactOS */
1983 ASSERT(MI_IS_PAGE_LARGE(PointerPde
) == FALSE
);
1986 /* Now capture the PTE. */
1987 TempPte
= *PointerPte
;
1989 /* Check if the PTE is valid */
1990 if (TempPte
.u
.Hard
.Valid
)
1992 /* Check if this is a write on a readonly PTE */
1993 if (StoreInstruction
)
1995 /* Is this a copy on write PTE? */
1996 if (TempPte
.u
.Hard
.CopyOnWrite
)
1998 /* Not supported yet */
2002 /* Is this a read-only PTE? */
2003 if (!TempPte
.u
.Hard
.Write
)
2005 /* Return the status */
2006 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2007 return STATUS_ACCESS_VIOLATION
;
2011 /* FIXME: Execution is ignored for now, since we don't have no-execute pages yet */
2013 /* The fault has already been resolved by a different thread */
2014 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2015 return STATUS_SUCCESS
;
2018 /* Quick check for demand-zero */
2019 if (TempPte
.u
.Long
== (MM_READWRITE
<< MM_PTE_SOFTWARE_PROTECTION_BITS
))
2021 /* Resolve the fault */
2022 MiResolveDemandZeroFault(Address
,
2027 /* Return the status */
2028 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2029 return STATUS_PAGE_FAULT_DEMAND_ZERO
;
2032 /* Check for zero PTE */
2033 if (TempPte
.u
.Long
== 0)
2035 /* Check if this address range belongs to a valid allocation (VAD) */
2036 ProtoPte
= MiCheckVirtualAddress(Address
, &ProtectionCode
, &Vad
);
2037 if (ProtectionCode
== MM_NOACCESS
)
2039 #if (_MI_PAGING_LEVELS == 2)
2040 /* Could be a page table for paged pool */
2041 MiCheckPdeForPagedPool(Address
);
2043 /* Has the code above changed anything -- is this now a valid PTE? */
2044 Status
= (PointerPte
->u
.Hard
.Valid
== 1) ? STATUS_SUCCESS
: STATUS_ACCESS_VIOLATION
;
2046 /* Either this was a bogus VA or we've fixed up a paged pool PDE */
2047 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2052 * Check if this is a real user-mode address or actually a kernel-mode
2053 * page table for a user mode address
2055 if (Address
<= MM_HIGHEST_USER_ADDRESS
)
2057 /* Add an additional page table reference */
2058 MiIncrementPageTableReferences(Address
);
2061 /* Is this a guard page? */
2062 if ((ProtectionCode
& MM_PROTECT_SPECIAL
) == MM_GUARDPAGE
)
2064 /* The VAD protection cannot be MM_DECOMMIT! */
2065 NT_ASSERT(ProtectionCode
!= MM_DECOMMIT
);
2067 /* Remove the bit */
2068 TempPte
.u
.Soft
.Protection
= ProtectionCode
& ~MM_GUARDPAGE
;
2069 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
2072 ASSERT(ProtoPte
== NULL
);
2073 ASSERT(CurrentThread
->ApcNeeded
== 0);
2075 /* Drop the working set lock */
2076 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2077 ASSERT(KeGetCurrentIrql() == OldIrql
);
2079 /* Handle stack expansion */
2080 return MiCheckForUserStackOverflow(Address
, TrapInformation
);
2083 /* Did we get a prototype PTE back? */
2086 /* Is this PTE actually part of the PDE-PTE self-mapping directory? */
2087 if (PointerPde
== MiAddressToPde(PTE_BASE
))
2089 /* Then it's really a demand-zero PDE (on behalf of user-mode) */
2090 MI_WRITE_INVALID_PTE(PointerPte
, DemandZeroPde
);
2094 /* No, create a new PTE. First, write the protection */
2095 TempPte
.u
.Soft
.Protection
= ProtectionCode
;
2096 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
2099 /* Lock the PFN database since we're going to grab a page */
2100 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
2102 /* Make sure we have enough pages */
2103 ASSERT(MmAvailablePages
>= 32);
2105 /* Try to get a zero page */
2106 MI_SET_USAGE(MI_USAGE_PEB_TEB
);
2107 MI_SET_PROCESS2(CurrentProcess
->ImageFileName
);
2108 Color
= MI_GET_NEXT_PROCESS_COLOR(CurrentProcess
);
2109 PageFrameIndex
= MiRemoveZeroPageSafe(Color
);
2110 if (!PageFrameIndex
)
2112 /* Grab a page out of there. Later we should grab a colored zero page */
2113 PageFrameIndex
= MiRemoveAnyPage(Color
);
2114 ASSERT(PageFrameIndex
);
2116 /* Release the lock since we need to do some zeroing */
2117 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
2119 /* Zero out the page, since it's for user-mode */
2120 MiZeroPfn(PageFrameIndex
);
2122 /* Grab the lock again so we can initialize the PFN entry */
2123 OldIrql
= KeAcquireQueuedSpinLock(LockQueuePfnLock
);
2126 /* Initialize the PFN entry now */
2127 MiInitializePfn(PageFrameIndex
, PointerPte
, 1);
2129 /* And we're done with the lock */
2130 KeReleaseQueuedSpinLock(LockQueuePfnLock
, OldIrql
);
2132 /* Increment the count of pages in the process */
2133 CurrentProcess
->NumberOfPrivatePages
++;
2135 /* One more demand-zero fault */
2136 InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount
);
2138 /* Fault on user PDE, or fault on user PTE? */
2139 if (PointerPte
<= MiHighestUserPte
)
2141 /* User fault, build a user PTE */
2142 MI_MAKE_HARDWARE_PTE_USER(&TempPte
,
2144 PointerPte
->u
.Soft
.Protection
,
2149 /* This is a user-mode PDE, create a kernel PTE for it */
2150 MI_MAKE_HARDWARE_PTE(&TempPte
,
2152 PointerPte
->u
.Soft
.Protection
,
2156 /* Write the dirty bit for writeable pages */
2157 if (MI_IS_PAGE_WRITEABLE(&TempPte
)) MI_MAKE_DIRTY_PAGE(&TempPte
);
2159 /* And now write down the PTE, making the address valid */
2160 MI_WRITE_VALID_PTE(PointerPte
, TempPte
);
2161 Pfn1
= MI_PFN_ELEMENT(PageFrameIndex
);
2162 ASSERT(Pfn1
->u1
.Event
== NULL
);
2165 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
2166 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2167 return STATUS_PAGE_FAULT_DEMAND_ZERO
;
2170 /* We should have a valid protection here */
2171 ASSERT(ProtectionCode
!= 0x100);
2173 /* Write the prototype PTE */
2174 TempPte
= PrototypePte
;
2175 TempPte
.u
.Soft
.Protection
= ProtectionCode
;
2176 NT_ASSERT(TempPte
.u
.Long
!= 0);
2177 MI_WRITE_INVALID_PTE(PointerPte
, TempPte
);
2181 /* Get the protection code and check if this is a proto PTE */
2182 ProtectionCode
= (ULONG
)TempPte
.u
.Soft
.Protection
;
2183 if (TempPte
.u
.Soft
.Prototype
)
2185 /* Do we need to go find the real PTE? */
2186 if (TempPte
.u
.Soft
.PageFileHigh
== MI_PTE_LOOKUP_NEEDED
)
2188 /* Get the prototype pte and VAD for it */
2189 ProtoPte
= MiCheckVirtualAddress(Address
,
2194 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
2195 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2196 return STATUS_ACCESS_VIOLATION
;
2201 /* Get the prototype PTE! */
2202 ProtoPte
= MiProtoPteToPte(&TempPte
);
2204 /* Is it read-only */
2205 if (TempPte
.u
.Proto
.ReadOnly
)
2207 /* Set read-only code */
2208 ProtectionCode
= MM_READONLY
;
2212 /* Set unknown protection */
2213 ProtectionCode
= 0x100;
2214 ASSERT(CurrentProcess
->CloneRoot
!= NULL
);
2220 /* Do we have a valid protection code? */
2221 if (ProtectionCode
!= 0x100)
2223 /* Run a software access check first, including to detect guard pages */
2224 Status
= MiAccessCheck(PointerPte
,
2230 if (Status
!= STATUS_SUCCESS
)
2233 ASSERT(CurrentThread
->ApcNeeded
== 0);
2235 /* Drop the working set lock */
2236 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2237 ASSERT(KeGetCurrentIrql() == OldIrql
);
2239 /* Did we hit a guard page? */
2240 if (Status
== STATUS_GUARD_PAGE_VIOLATION
)
2242 /* Handle stack expansion */
2243 return MiCheckForUserStackOverflow(Address
, TrapInformation
);
2246 /* Otherwise, fail back to the caller directly */
2251 /* Dispatch the fault */
2252 Status
= MiDispatchFault(StoreInstruction
,
2261 /* Return the status */
2262 ASSERT(KeGetCurrentIrql() <= APC_LEVEL
);
2263 MiUnlockProcessWorkingSet(CurrentProcess
, CurrentThread
);
2269 MmGetExecuteOptions(IN PULONG ExecuteOptions
)
2271 PKPROCESS CurrentProcess
= &PsGetCurrentProcess()->Pcb
;
2272 ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL
);
2274 *ExecuteOptions
= 0;
2276 if (CurrentProcess
->Flags
.ExecuteDisable
)
2278 *ExecuteOptions
|= MEM_EXECUTE_OPTION_DISABLE
;
2281 if (CurrentProcess
->Flags
.ExecuteEnable
)
2283 *ExecuteOptions
|= MEM_EXECUTE_OPTION_ENABLE
;
2286 if (CurrentProcess
->Flags
.DisableThunkEmulation
)
2288 *ExecuteOptions
|= MEM_EXECUTE_OPTION_DISABLE_THUNK_EMULATION
;
2291 if (CurrentProcess
->Flags
.Permanent
)
2293 *ExecuteOptions
|= MEM_EXECUTE_OPTION_PERMANENT
;
2296 if (CurrentProcess
->Flags
.ExecuteDispatchEnable
)
2298 *ExecuteOptions
|= MEM_EXECUTE_OPTION_EXECUTE_DISPATCH_ENABLE
;
2301 if (CurrentProcess
->Flags
.ImageDispatchEnable
)
2303 *ExecuteOptions
|= MEM_EXECUTE_OPTION_IMAGE_DISPATCH_ENABLE
;
2306 return STATUS_SUCCESS
;
2311 MmSetExecuteOptions(IN ULONG ExecuteOptions
)
2313 PKPROCESS CurrentProcess
= &PsGetCurrentProcess()->Pcb
;
2314 KLOCK_QUEUE_HANDLE ProcessLock
;
2315 NTSTATUS Status
= STATUS_ACCESS_DENIED
;
2316 ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL
);
2318 /* Only accept valid flags */
2319 if (ExecuteOptions
& ~MEM_EXECUTE_OPTION_VALID_FLAGS
)
2322 DPRINT1("Invalid no-execute options\n");
2323 return STATUS_INVALID_PARAMETER
;
2326 /* Change the NX state in the process lock */
2327 KiAcquireProcessLock(CurrentProcess
, &ProcessLock
);
2329 /* Don't change anything if the permanent flag was set */
2330 if (!CurrentProcess
->Flags
.Permanent
)
2332 /* Start by assuming it's not disabled */
2333 CurrentProcess
->Flags
.ExecuteDisable
= FALSE
;
2335 /* Now process each flag and turn the equivalent bit on */
2336 if (ExecuteOptions
& MEM_EXECUTE_OPTION_DISABLE
)
2338 CurrentProcess
->Flags
.ExecuteDisable
= TRUE
;
2340 if (ExecuteOptions
& MEM_EXECUTE_OPTION_ENABLE
)
2342 CurrentProcess
->Flags
.ExecuteEnable
= TRUE
;
2344 if (ExecuteOptions
& MEM_EXECUTE_OPTION_DISABLE_THUNK_EMULATION
)
2346 CurrentProcess
->Flags
.DisableThunkEmulation
= TRUE
;
2348 if (ExecuteOptions
& MEM_EXECUTE_OPTION_PERMANENT
)
2350 CurrentProcess
->Flags
.Permanent
= TRUE
;
2352 if (ExecuteOptions
& MEM_EXECUTE_OPTION_EXECUTE_DISPATCH_ENABLE
)
2354 CurrentProcess
->Flags
.ExecuteDispatchEnable
= TRUE
;
2356 if (ExecuteOptions
& MEM_EXECUTE_OPTION_IMAGE_DISPATCH_ENABLE
)
2358 CurrentProcess
->Flags
.ImageDispatchEnable
= TRUE
;
2361 /* These are turned on by default if no-execution is also eanbled */
2362 if (CurrentProcess
->Flags
.ExecuteEnable
)
2364 CurrentProcess
->Flags
.ExecuteDispatchEnable
= TRUE
;
2365 CurrentProcess
->Flags
.ImageDispatchEnable
= TRUE
;
2369 Status
= STATUS_SUCCESS
;
2372 /* Release the lock and return status */
2373 KiReleaseProcessLock(&ProcessLock
);