[NTOS]: This is why you shouldn't let Antoine Dodson commit code.
[reactos.git] / reactos / ntoskrnl / mm / ARM3 / pagfault.c
index ea633a1..635458e 100644 (file)
 
 /* PRIVATE FUNCTIONS **********************************************************/
 
+PMMPTE
+NTAPI
+MiCheckVirtualAddress(IN PVOID VirtualAddress,
+                      OUT PULONG ProtectCode,
+                      OUT PMMVAD *ProtoVad)
+{
+    PMMVAD Vad;
+    
+    /* No prototype/section support for now */
+    *ProtoVad = NULL;
+    
+    /* Only valid for user VADs for now */
+    ASSERT(VirtualAddress <= MM_HIGHEST_USER_ADDRESS);
+    
+    /* Special case for shared data */
+    if (PAGE_ALIGN(VirtualAddress) == (PVOID)USER_SHARED_DATA)
+    {
+        /* It's a read-only page */
+        *ProtectCode = MM_READONLY;
+        return MmSharedUserDataPte;
+    }
+    
+    /* Find the VAD, it might not exist if the address is bogus */
+    Vad = MiLocateAddress(VirtualAddress);
+    if (!Vad)
+    {
+        /* Bogus virtual address */
+        *ProtectCode = MM_NOACCESS;
+        return NULL;
+    }
+    
+    /* This must be a TEB/PEB VAD */
+    ASSERT(Vad->u.VadFlags.PrivateMemory == TRUE);
+    ASSERT(Vad->u.VadFlags.MemCommit == TRUE);
+    ASSERT(Vad->u.VadFlags.VadType == VadNone);
+    
+    /* Return the protection on it */
+    *ProtectCode = Vad->u.VadFlags.Protection;
+    return NULL;
+}
 NTSTATUS
 FASTCALL
 MiCheckPdeForPagedPool(IN PVOID Address)
 {
-    PMMPTE PointerPde;
+    PMMPDE PointerPde;
     NTSTATUS Status = STATUS_SUCCESS;
     
+    /* No session support in ReactOS yet */
+    ASSERT(MI_IS_SESSION_ADDRESS(Address) == FALSE);
+    ASSERT(MI_IS_SESSION_PTE(Address) == FALSE);
+    
     //
     // Check if this is a fault while trying to access the page table itself
     //
-    if ((Address >= (PVOID)MiAddressToPte(MmSystemRangeStart)) &&
-        (Address < (PVOID)PTE_TOP))
+    if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
     {
         //
         // Send a hint to the page fault handler that this is only a valid fault
         // if we already detected this was access within the page table range
         //
-        PointerPde = MiAddressToPte(Address);
+        PointerPde = (PMMPDE)MiAddressToPte(Address);
         Status = STATUS_WAIT_1;
     }
     else if (Address < MmSystemRangeStart)
@@ -60,13 +104,15 @@ MiCheckPdeForPagedPool(IN PVOID Address)
     //
     if (PointerPde->u.Hard.Valid == 0)
     {
+#ifdef _M_AMD64
+        ASSERT(FALSE);
+#else
         //
         // Copy it from our double-mapped system page directory
         //
         InterlockedExchangePte(PointerPde,
-                               MmSystemPagePtes[((ULONG_PTR)PointerPde &
-                                                 (PAGE_SIZE - 1)) /
-                                                sizeof(MMPTE)].u.Long);
+                               MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)].u.Long);
+#endif
     }
     
     //
@@ -75,6 +121,52 @@ MiCheckPdeForPagedPool(IN PVOID Address)
     return Status;
 }
 
+VOID
+NTAPI
+MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
+{
+    PMMPTE ZeroPte;
+    MMPTE TempPte;
+    PMMPFN Pfn1;
+    PVOID ZeroAddress;
+    
+    /* Get the PFN for this page */
+    Pfn1 = MiGetPfnEntry(PageFrameNumber);
+    ASSERT(Pfn1);
+        
+    /* Grab a system PTE we can use to zero the page */
+    ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
+    ASSERT(ZeroPte);
+    
+    /* Initialize the PTE for it */
+    TempPte = ValidKernelPte;
+    TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
+
+    /* Setup caching */
+    if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
+    {
+        /* Write combining, no caching */
+        MI_PAGE_DISABLE_CACHE(&TempPte);
+        MI_PAGE_WRITE_COMBINED(&TempPte);
+    }
+    else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
+    {
+        /* Write through, no caching */
+        MI_PAGE_DISABLE_CACHE(&TempPte);
+        MI_PAGE_WRITE_THROUGH(&TempPte);
+    }
+
+    /* Make the system PTE valid with our PFN */
+    MI_WRITE_VALID_PTE(ZeroPte, TempPte);
+
+    /* Get the address it maps to, and zero it out */
+    ZeroAddress = MiPteToAddress(ZeroPte);
+    KeZeroPages(ZeroAddress, PAGE_SIZE);
+
+    /* Now get rid of it */
+    MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
+}
+
 NTSTATUS
 NTAPI
 MiResolveDemandZeroFault(IN PVOID Address,
@@ -82,23 +174,60 @@ MiResolveDemandZeroFault(IN PVOID Address,
                          IN PEPROCESS Process,
                          IN KIRQL OldIrql)
 {
-    PFN_NUMBER PageFrameNumber;
+    PFN_NUMBER PageFrameNumber = 0;
     MMPTE TempPte;
+    BOOLEAN NeedZero = FALSE;
+    ULONG Color;
     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
             Address,
             Process);
     
+    /* Must currently only be called by paging path */
+    ASSERT(OldIrql == MM_NOIRQL);
+    if (Process)
+    {
+        /* Sanity check */
+        ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
+
+        /* No forking yet */
+        ASSERT(Process->ForkInProgress == NULL);
+        
+        /* Get process color */
+        Color = MI_GET_NEXT_PROCESS_COLOR(Process);
+        
+        /* We'll need a zero page */
+        NeedZero = TRUE;
+    }
+    else
+    {
+        /* Get the next system page color */
+        Color = MI_GET_NEXT_COLOR();
+    }
+        
     //
     // Lock the PFN database
     //
     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
     ASSERT(PointerPte->u.Hard.Valid == 0);
     
-    //
-    // Get a page
-    //
-    PageFrameNumber = MmAllocPage(MC_PPOOL, 0);
-    DPRINT("New pool page: %lx\n", PageFrameNumber);
+    /* Do we need a zero page? */
+    if (NeedZero)
+    {
+        /* Try to get one, if we couldn't grab a free page and zero it */
+        PageFrameNumber = MiRemoveZeroPageSafe(Color);
+        if (PageFrameNumber) NeedZero = FALSE;
+    }
+    
+    /* Did we get a page? */
+    if (!PageFrameNumber)
+    {
+        /* We either failed to find a zero page, or this is a system request */
+        PageFrameNumber = MiRemoveAnyPage(Color);
+        DPRINT("New pool page: %lx\n", PageFrameNumber);
+    }
+    
+    /* Initialize it */
+    MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
     
     //
     // Release PFN lock
@@ -110,14 +239,33 @@ MiResolveDemandZeroFault(IN PVOID Address,
     //
     InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
     
-    //
-    // Build the PTE
-    //
-    TempPte = HyperTemplatePte;
-    TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
-    *PointerPte = TempPte;
-    ASSERT(PointerPte->u.Hard.Valid == 1);
+    /* Zero the page if need be */
+    if (NeedZero) MiZeroPfn(PageFrameNumber);
+    
+    /* Build the PTE */
+    if (PointerPte <= MiHighestUserPte)
+    {
+        /* For user mode */
+        MI_MAKE_HARDWARE_PTE_USER(&TempPte,
+                                  PointerPte,
+                                  PointerPte->u.Soft.Protection,
+                                  PageFrameNumber);
+    }
+    else
+    {
+        /* For kernel mode */
+        MI_MAKE_HARDWARE_PTE(&TempPte,
+                             PointerPte,
+                             PointerPte->u.Soft.Protection,
+                             PageFrameNumber);
+    }
     
+    /* Set it dirty if it's a writable page */
+    if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
+    
+    /* Write it */
+    MI_WRITE_VALID_PTE(PointerPte, TempPte);
+
     //
     // It's all good now
     //
@@ -125,36 +273,156 @@ MiResolveDemandZeroFault(IN PVOID Address,
     return STATUS_PAGE_FAULT_DEMAND_ZERO;
 }
 
+NTSTATUS
+NTAPI
+MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
+                        IN PVOID Address,
+                        IN PMMPTE PointerPte,
+                        IN PMMPTE PointerProtoPte,
+                        IN KIRQL OldIrql,
+                        IN PMMPFN Pfn1)
+{
+    MMPTE TempPte;
+    PFN_NUMBER PageFrameIndex;
+    
+    /* Must be called with an valid prototype PTE, with the PFN lock held */
+    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+    ASSERT(PointerProtoPte->u.Hard.Valid == 1);
+    
+    /* Quick-n-dirty */
+    ASSERT(PointerPte->u.Soft.PageFileHigh == 0xFFFFF);
+    
+    /* Get the page */
+    PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
+
+    /* Release the PFN lock */
+    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+    
+    /* Build the user PTE */
+    ASSERT(Address < MmSystemRangeStart);
+    MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, MM_READONLY, PageFrameIndex);
+    
+    /* Write the PTE */
+    MI_WRITE_VALID_PTE(PointerPte, TempPte);
+
+    /* Return success */
+    return STATUS_SUCCESS;
+}
+
+NTSTATUS
+NTAPI
+MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
+                       IN PVOID Address,
+                       IN PMMPTE PointerPte,
+                       IN PMMPTE PointerProtoPte,
+                       IN OUT PMMPFN *OutPfn,
+                       OUT PVOID *PageFileData,
+                       OUT PMMPTE PteValue,
+                       IN PEPROCESS Process,
+                       IN KIRQL OldIrql,
+                       IN PVOID TrapInformation)
+{
+    MMPTE TempPte;
+    PMMPFN Pfn1;
+    PFN_NUMBER PageFrameIndex;
+    
+    /* Must be called with an invalid, prototype PTE, with the PFN lock held */
+    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+    ASSERT(PointerPte->u.Hard.Valid == 0);
+    ASSERT(PointerPte->u.Soft.Prototype == 1);
+
+    /* Read the prototype PTE -- it must be valid since we only handle shared data */
+    TempPte = *PointerProtoPte;
+    ASSERT(TempPte.u.Hard.Valid == 1);
+    
+    /* One more user of this mapped page */
+    PageFrameIndex = PFN_FROM_PTE(&TempPte);
+    Pfn1 = MiGetPfnEntry(PageFrameIndex);
+    Pfn1->u2.ShareCount++;
+    
+    /* Call it a transition */
+    InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
+    
+    /* Complete the prototype PTE fault -- this will release the PFN lock */
+    return MiCompleteProtoPteFault(StoreInstruction,
+                                   Address,
+                                   PointerPte,
+                                   PointerProtoPte,
+                                   OldIrql,
+                                   NULL);
+}
+
 NTSTATUS
 NTAPI
 MiDispatchFault(IN BOOLEAN StoreInstruction,
                 IN PVOID Address,
                 IN PMMPTE PointerPte,
-                IN PMMPTE PrototypePte,
+                IN PMMPTE PointerProtoPte,
                 IN BOOLEAN Recursive,
                 IN PEPROCESS Process,
                 IN PVOID TrapInformation,
                 IN PVOID Vad)
 {
     MMPTE TempPte;
-    KIRQL OldIrql;
+    KIRQL OldIrql, LockIrql;
     NTSTATUS Status;
+    PMMPTE SuperProtoPte;
     DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
-            Address,
-            Process);
+             Address,
+             Process);
     
     //
     // Make sure APCs are off and we're not at dispatch
     //
-    OldIrql = KeGetCurrentIrql ();
+    OldIrql = KeGetCurrentIrql();
     ASSERT(OldIrql <= APC_LEVEL);
-    ASSERT(KeAreAllApcsDisabled () == TRUE);
+    ASSERT(KeAreAllApcsDisabled() == TRUE);
     
     //
     // Grab a copy of the PTE
     //
     TempPte = *PointerPte;
     
+    /* Do we have a prototype PTE? */
+    if (PointerProtoPte)
+    {
+        /* This should never happen */
+        ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
+            
+        /* We currently only handle the shared user data PTE path */
+        ASSERT(Address < MmSystemRangeStart);
+        ASSERT(PointerPte->u.Soft.Prototype == 1);
+        ASSERT(PointerPte->u.Soft.PageFileHigh == 0xFFFFF);
+        ASSERT(Vad == NULL);
+        
+        /* Lock the PFN database */
+        LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+        
+        /* For the shared data page, this should be true */
+        SuperProtoPte = MiAddressToPte(PointerProtoPte);
+        ASSERT(SuperProtoPte->u.Hard.Valid == 1);
+        ASSERT(TempPte.u.Hard.Valid == 0);
+
+        /* Resolve the fault -- this will release the PFN lock */
+        Status = MiResolveProtoPteFault(StoreInstruction,
+                                        Address,
+                                        PointerPte,
+                                        PointerProtoPte,
+                                        NULL,
+                                        NULL,
+                                        NULL,
+                                        Process,
+                                        LockIrql,
+                                        TrapInformation);
+        ASSERT(Status == STATUS_SUCCESS);
+
+        /* Complete this as a transition fault */
+        ASSERT(OldIrql == KeGetCurrentIrql());
+        ASSERT(OldIrql <= APC_LEVEL);
+        ASSERT(KeAreAllApcsDisabled() == TRUE);
+        return STATUS_PAGE_FAULT_TRANSITION;
+    }
+    
     //
     // The PTE must be invalid, but not totally blank
     //
@@ -175,7 +443,8 @@ MiDispatchFault(IN BOOLEAN StoreInstruction,
     Status = MiResolveDemandZeroFault(Address,
                                       PointerPte,
                                       Process,
-                                      -1);
+                                      MM_NOIRQL);
+    ASSERT(KeAreAllApcsDisabled () == TRUE);
     if (NT_SUCCESS(Status))
     {
         //
@@ -200,17 +469,29 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction,
                  IN PVOID TrapInformation)
 {
     KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
-    PMMPTE PointerPte;
+    PMMPTE PointerPte, ProtoPte;
+    PMMPDE PointerPde;
     MMPTE TempPte;
     PETHREAD CurrentThread;
+    PEPROCESS CurrentProcess;
     NTSTATUS Status;
+    PMMSUPPORT WorkingSet;
+    ULONG ProtectionCode;
+    PMMVAD Vad;
+    PFN_NUMBER PageFrameIndex;
+    ULONG Color;
     DPRINT("ARM3 FAULT AT: %p\n", Address);
     
     //
     // Get the PTE and PDE
     //
     PointerPte = MiAddressToPte(Address);
-    
+    PointerPde = MiAddressToPde(Address);
+#if (_MI_PAGING_LEVELS >= 3)
+    /* We need the PPE and PXE addresses */
+    ASSERT(FALSE);
+#endif
+
     //
     // Check for dispatch-level snafu
     //
@@ -235,25 +516,30 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction,
         //
         if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
         
+#if (_MI_PAGING_LEVELS >= 3)
+        /* Need to check PXE and PDE validity */
+        ASSERT(FALSE);
+#endif
+
         //
         // Is the PDE valid?
         //
-        if (!MiIsPdeForAddressValid(Address))
+        if (!PointerPde->u.Hard.Valid == 0)
         {
             //
             // Debug spew (eww!)
             //
             DPRINT("Invalid PDE\n");
-            
+#if (_MI_PAGING_LEVELS == 2) 
             //
             // Handle mapping in "Special" PDE directoreis
             //
             MiCheckPdeForPagedPool(Address);
-            
+#endif
             //
             // Now we SHOULD be good
             //
-            if (!MiIsPdeForAddressValid(Address))
+            if (PointerPde->u.Hard.Valid == 0)
             {
                 //
                 // FIXFIX: Do the S-LIST hack
@@ -297,26 +583,36 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction,
         //
         // Check for a fault on the page table or hyperspace itself
         //
-        if ((Address >= (PVOID)PTE_BASE) && (Address <= (PVOID)HYPER_SPACE_END))
+        if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
         {
             //
             // This might happen...not sure yet
             //
-            DPRINT1("FAULT ON PAGE TABLES!\n");
+            DPRINT1("FAULT ON PAGE TABLES: %p %lx %lx!\n", Address, *PointerPte, *PointerPde);
+#if (_MI_PAGING_LEVELS == 2) 
+            //
+            // Map in the page table
+            //
+            if (MiCheckPdeForPagedPool(Address) == STATUS_WAIT_1)
+            {
+                DPRINT1("PAGE TABLES FAULTED IN!\n");
+                return STATUS_SUCCESS;
+            }
+#endif
+            //
+            // Otherwise the page table doesn't actually exist
+            //
+            DPRINT1("FAILING\n");
             return STATUS_ACCESS_VIOLATION;
         }
         
-        //
-        // Now we must raise to APC_LEVEL and mark the thread as owner
-        // We don't actually implement a working set pushlock, so this is only
-        // for internal consistency (and blocking APCs)
-        //
-        KeRaiseIrql(APC_LEVEL, &LockIrql);
+        /* In this path, we are using the system working set */
         CurrentThread = PsGetCurrentThread();
-        KeEnterGuardedRegion();
-        ASSERT((CurrentThread->OwnsSystemWorkingSetExclusive == 0) &&
-               (CurrentThread->OwnsSystemWorkingSetShared == 0));
-        CurrentThread->OwnsSystemWorkingSetExclusive = 1; 
+        WorkingSet = &MmSystemCacheWs;
+        
+        /* Acquire it */
+        KeRaiseIrql(APC_LEVEL, &LockIrql);
+        MiLockWorkingSet(CurrentThread, WorkingSet);
         
         //
         // Re-read PTE now that the IRQL has been raised
@@ -336,16 +632,38 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction,
                 return STATUS_ACCESS_VIOLATION;
             }
             
+            /* Release the working set */
+            MiUnlockWorkingSet(CurrentThread, WorkingSet);
+            KeLowerIrql(LockIrql);
+            
             //
             // Otherwise, the PDE was probably invalid, and all is good now
             //
             return STATUS_SUCCESS;
         }
         
-        //
-        // We don't implement prototype PTEs
-        //
-        ASSERT(TempPte.u.Soft.Prototype == 0);
+        /* Check one kind of prototype PTE */
+        if (TempPte.u.Soft.Prototype)
+        {
+            /* The one used for protected pool... */
+            ASSERT(MmProtectFreedNonPagedPool == TRUE);
+            
+            /* Make sure protected pool is on, and that this is a pool address */
+            if ((MmProtectFreedNonPagedPool) &&
+                (((Address >= MmNonPagedPoolStart) &&
+                  (Address < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
+                                     MmSizeOfNonPagedPoolInBytes))) ||
+                 ((Address >= MmNonPagedPoolExpansionStart) &&
+                  (Address < MmNonPagedPoolEnd))))
+            {
+                /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
+                KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL,
+                             (ULONG_PTR)Address,
+                             StoreInstruction,
+                             Mode,
+                             4);
+            }
+        }
         
         //
         // We don't implement transition PTEs
@@ -363,13 +681,10 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction,
                                  NULL,
                                  TrapInformation,
                                  NULL);
-        
-        //
-        // Re-enable APCs
-        //
+
+        /* Release the working set */
         ASSERT(KeAreAllApcsDisabled() == TRUE);
-        CurrentThread->OwnsSystemWorkingSetExclusive = 0;
-        KeLeaveGuardedRegion();
+        MiUnlockWorkingSet(CurrentThread, WorkingSet);
         KeLowerIrql(LockIrql);
         
         //
@@ -379,11 +694,168 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction,
         return Status;
     }
     
-    //
-    // DIE DIE DIE
-    //
-    DPRINT1("WARNING: USER MODE FAULT IN ARM3???\n");
-    return STATUS_ACCESS_VIOLATION;
+    /* This is a user fault */
+    CurrentThread = PsGetCurrentThread();
+    CurrentProcess = PsGetCurrentProcess();
+    
+    /* Lock the working set */
+    MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
+    
+#if (_MI_PAGING_LEVELS >= 3)
+    /* Need to check/handle PPE and PXE validity too */
+    ASSERT(FALSE);
+#endif
+
+    /* First things first, is the PDE valid? */
+    ASSERT(PointerPde != MiAddressToPde(PTE_BASE));
+    ASSERT(PointerPde->u.Hard.LargePage == 0);
+    if (PointerPde->u.Hard.Valid == 0)
+    {
+        /* Right now, we only handle scenarios where the PDE is totally empty */
+        ASSERT(PointerPde->u.Long == 0);
+
+        /* Check if this address range belongs to a valid allocation (VAD) */
+        MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
+        
+        /* Right now, we expect a valid protection mask on the VAD */
+        ASSERT(ProtectionCode != MM_NOACCESS);
+
+        /* Make the PDE demand-zero */
+        MI_WRITE_INVALID_PTE(PointerPde, DemandZeroPde);
+
+        /* And go dispatch the fault on the PDE. This should handle the demand-zero */
+        Status = MiDispatchFault(TRUE,
+                                 PointerPte,
+                                 PointerPde,
+                                 NULL,
+                                 FALSE,
+                                 PsGetCurrentProcess(),
+                                 TrapInformation,
+                                 NULL);
+
+        /* We should come back with APCs enabled, and with a valid PDE */
+        ASSERT(KeAreAllApcsDisabled() == TRUE);
+#if (_MI_PAGING_LEVELS >= 3)
+        /* Need to check/handle PPE and PXE validity too */
+        ASSERT(FALSE);
+#endif
+        ASSERT(PointerPde->u.Hard.Valid == 1);
+    }
+
+    /* Now capture the PTE. We only handle cases where it's totally empty */
+    TempPte = *PointerPte;
+    ASSERT(TempPte.u.Long == 0);
+
+    /* Check if this address range belongs to a valid allocation (VAD) */
+    ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
+    if (ProtectionCode == MM_NOACCESS)
+    {
+        /* This is a bogus VA */
+        Status = STATUS_ACCESS_VIOLATION;
+        
+        /* Could be a not-yet-mapped paged pool page table */
+#if (_MI_PAGING_LEVELS == 2) 
+        MiCheckPdeForPagedPool(Address);
+#endif
+        /* See if that fixed it */
+        if (PointerPte->u.Hard.Valid == 1) Status = STATUS_SUCCESS;
+        
+        /* Return the status */
+        MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
+        return Status;
+    }
+    
+    /* Did we get a prototype PTE back? */
+    if (!ProtoPte)
+    {
+        /* No, create a new PTE. First, write the protection */
+        PointerPte->u.Soft.Protection = ProtectionCode;
+
+        /* Lock the PFN database since we're going to grab a page */
+        OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+        
+        /* Try to get a zero page */
+        Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
+        PageFrameIndex = MiRemoveZeroPageSafe(Color);
+        if (!PageFrameIndex)
+        {
+            /* Grab a page out of there. Later we should grab a colored zero page */
+            PageFrameIndex = MiRemoveAnyPage(Color);
+            ASSERT(PageFrameIndex);
+
+            /* Release the lock since we need to do some zeroing */
+            KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+
+            /* Zero out the page, since it's for user-mode */
+            MiZeroPfn(PageFrameIndex);
+
+            /* Grab the lock again so we can initialize the PFN entry */
+            OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
+        }
+
+        /* Initialize the PFN entry now */
+        MiInitializePfn(PageFrameIndex, PointerPte, 1);
+
+        /* And we're done with the lock */
+        KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
+
+        /* One more demand-zero fault */
+        InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
+
+        /* Was the fault on an actual user page, or a kernel page for the user? */
+        if (PointerPte <= MiHighestUserPte)
+        {
+            /* User fault, build a user PTE */
+            MI_MAKE_HARDWARE_PTE_USER(&TempPte,
+                                      PointerPte,
+                                      PointerPte->u.Soft.Protection,
+                                      PageFrameIndex);
+        }
+        else
+        {
+            /* Session, kernel, or user PTE, figure it out and build it */
+            MI_MAKE_HARDWARE_PTE(&TempPte,
+                                 PointerPte,
+                                 PointerPte->u.Soft.Protection,
+                                 PageFrameIndex);
+        }
+
+        /* Write the dirty bit for writeable pages */
+        if (TempPte.u.Hard.Write) TempPte.u.Hard.Dirty = TRUE;
+
+        /* And now write down the PTE, making the address valid */
+        MI_WRITE_VALID_PTE(PointerPte, TempPte);
+        
+        /* Demand zero */
+        Status = STATUS_PAGE_FAULT_DEMAND_ZERO;
+    }
+    else
+    {
+        /* The only "prototype PTE" we support is the shared user data path */
+        ASSERT(ProtectionCode == MM_READONLY);
+        
+        /* Write the prototype PTE */
+        TempPte = PrototypePte;
+        TempPte.u.Soft.Protection = ProtectionCode;
+        MI_WRITE_INVALID_PTE(PointerPte, TempPte);
+        
+        /* Handle the fault */
+        Status = MiDispatchFault(StoreInstruction,
+                                 Address,
+                                 PointerPte,
+                                 ProtoPte,
+                                 FALSE,
+                                 CurrentProcess,
+                                 TrapInformation,
+                                 Vad);
+        ASSERT(Status == STATUS_PAGE_FAULT_TRANSITION);
+        ASSERT(PointerPte->u.Hard.Valid == 1);
+        ASSERT(PointerPte->u.Hard.PageFrameNumber == MmSharedUserDataPte->u.Hard.PageFrameNumber);
+    }
+    
+    /* Release the working set */
+    MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
+    return Status;
 }
 
 /* EOF */