ntoskrnl/mm/ARM3/pagfault.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pagfault.c
   5  * PURPOSE:         ARM Memory Manager Page Fault Handling
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include "../ARM3/miarm.h"
  17
  18 /* GLOBALS ********************************************************************/
  19
  20 #define HYDRA_PROCESS (PEPROCESS)1
  21 #if MI_TRACE_PFNS
  22 BOOLEAN UserPdeFault = FALSE;
  23 #endif
  24
  25 /* PRIVATE FUNCTIONS **********************************************************/
  26
  27 NTSTATUS
  28 NTAPI
  29 MiCheckForUserStackOverflow(IN PVOID Address,
  30                             IN PVOID TrapInformation)
  31 {
  32     PETHREAD CurrentThread = PsGetCurrentThread();
  33     PTEB Teb = CurrentThread->Tcb.Teb;
  34     PVOID StackBase, DeallocationStack, NextStackAddress;
  35     SIZE_T GuranteedSize;
  36     NTSTATUS Status;
  37
  38     /* Do we own the address space lock? */
  39     if (CurrentThread->AddressSpaceOwner == 1)
  40     {
  41         /* This isn't valid */
  42         DPRINT1("Process owns address space lock\n");
  43         ASSERT(KeAreAllApcsDisabled() == TRUE);
  44         return STATUS_GUARD_PAGE_VIOLATION;
  45     }
  46
  47     /* Are we attached? */
  48     if (KeIsAttachedProcess())
  49     {
  50         /* This isn't valid */
  51         DPRINT1("Process is attached\n");
  52         return STATUS_GUARD_PAGE_VIOLATION;
  53     }
  54
  55     /* Read the current settings */
  56     StackBase = Teb->NtTib.StackBase;
  57     DeallocationStack = Teb->DeallocationStack;
  58     GuranteedSize = Teb->GuaranteedStackBytes;
  59     DPRINT("Handling guard page fault with Stacks Addresses 0x%p and 0x%p, guarantee: %lx\n",
  60             StackBase, DeallocationStack, GuranteedSize);
  61
  62     /* Guarantees make this code harder, for now, assume there aren't any */
  63     ASSERT(GuranteedSize == 0);
  64
  65     /* So allocate only the minimum guard page size */
  66     GuranteedSize = PAGE_SIZE;
  67
  68     /* Does this faulting stack address actually exist in the stack? */
  69     if ((Address >= StackBase) || (Address < DeallocationStack))
  70     {
  71         /* That's odd... */
  72         DPRINT1("Faulting address outside of stack bounds. Address=%p, StackBase=%p, DeallocationStack=%p\n",
  73                 Address, StackBase, DeallocationStack);
  74         return STATUS_GUARD_PAGE_VIOLATION;
  75     }
  76
  77     /* This is where the stack will start now */
  78     NextStackAddress = (PVOID)((ULONG_PTR)PAGE_ALIGN(Address) - GuranteedSize);
  79
  80     /* Do we have at least one page between here and the end of the stack? */
  81     if (((ULONG_PTR)NextStackAddress - PAGE_SIZE) <= (ULONG_PTR)DeallocationStack)
  82     {
  83         /* We don't -- Windows would try to make this guard page valid now */
  84         DPRINT1("Close to our death...\n");
  85         ASSERT(FALSE);
  86         return STATUS_STACK_OVERFLOW;
  87     }
  88
  89     /* Don't handle this flag yet */
  90     ASSERT((PsGetCurrentProcess()->Peb->NtGlobalFlag & FLG_DISABLE_STACK_EXTENSION) == 0);
  91
  92     /* Update the stack limit */
  93     Teb->NtTib.StackLimit = (PVOID)((ULONG_PTR)NextStackAddress + GuranteedSize);
  94
  95     /* Now move the guard page to the next page */
  96     Status = ZwAllocateVirtualMemory(NtCurrentProcess(),
  97                                      &NextStackAddress,
  98                                      0,
  99                                      &GuranteedSize,
 100                                      MEM_COMMIT,
 101                                      PAGE_READWRITE | PAGE_GUARD);
 102     if ((NT_SUCCESS(Status) || (Status == STATUS_ALREADY_COMMITTED)))
 103     {
 104         /* We did it! */
 105         DPRINT("Guard page handled successfully for %p\n", Address);
 106         return STATUS_PAGE_FAULT_GUARD_PAGE;
 107     }
 108
 109     /* Fail, we couldn't move the guard page */
 110     DPRINT1("Guard page failure: %lx\n", Status);
 111     ASSERT(FALSE);
 112     return STATUS_STACK_OVERFLOW;
 113 }
 114
 115 FORCEINLINE
 116 BOOLEAN
 117 MiIsAccessAllowed(
 118     _In_ ULONG ProtectionMask,
 119     _In_ BOOLEAN Write,
 120     _In_ BOOLEAN Execute)
 121 {
 122     #define _BYTE_MASK(Bit0, Bit1, Bit2, Bit3, Bit4, Bit5, Bit6, Bit7) \
 123         (Bit0) | ((Bit1) << 1) | ((Bit2) << 2) | ((Bit3) << 3) | \
 124         ((Bit4) << 4) | ((Bit5) << 5) | ((Bit6) << 6) | ((Bit7) << 7)
 125     static const UCHAR AccessAllowedMask[2][2] =
 126     {
 127         {   // Protect 0  1  2  3  4  5  6  7
 128             _BYTE_MASK(0, 1, 1, 1, 1, 1, 1, 1), // READ
 129             _BYTE_MASK(0, 0, 1, 1, 0, 0, 1, 1), // EXECUTE READ
 130         },
 131         {
 132             _BYTE_MASK(0, 0, 0, 0, 1, 1, 1, 1), // WRITE
 133             _BYTE_MASK(0, 0, 0, 0, 0, 0, 1, 1), // EXECUTE WRITE
 134         }
 135     };
 136
 137     /* We want only the lower access bits */
 138     ProtectionMask &= MM_PROTECT_ACCESS;
 139
 140     /* Look it up in the table */
 141     return (AccessAllowedMask[Write != 0][Execute != 0] >> ProtectionMask) & 1;
 142 }
 143
 144 NTSTATUS
 145 NTAPI
 146 MiAccessCheck(IN PMMPTE PointerPte,
 147               IN BOOLEAN StoreInstruction,
 148               IN KPROCESSOR_MODE PreviousMode,
 149               IN ULONG_PTR ProtectionMask,
 150               IN PVOID TrapFrame,
 151               IN BOOLEAN LockHeld)
 152 {
 153     MMPTE TempPte;
 154
 155     /* Check for invalid user-mode access */
 156     if ((PreviousMode == UserMode) && (PointerPte > MiHighestUserPte))
 157     {
 158         return STATUS_ACCESS_VIOLATION;
 159     }
 160
 161     /* Capture the PTE -- is it valid? */
 162     TempPte = *PointerPte;
 163     if (TempPte.u.Hard.Valid)
 164     {
 165         /* Was someone trying to write to it? */
 166         if (StoreInstruction)
 167         {
 168             /* Is it writable?*/
 169             if ((TempPte.u.Hard.Write) || (TempPte.u.Hard.CopyOnWrite))
 170             {
 171                 /* Then there's nothing to worry about */
 172                 return STATUS_SUCCESS;
 173             }
 174
 175             /* Oops! This isn't allowed */
 176             return STATUS_ACCESS_VIOLATION;
 177         }
 178
 179         /* Someone was trying to read from a valid PTE, that's fine too */
 180         return STATUS_SUCCESS;
 181     }
 182
 183     /* Check if the protection on the page allows what is being attempted */
 184     if (!MiIsAccessAllowed(ProtectionMask, StoreInstruction, FALSE))
 185     {
 186         return STATUS_ACCESS_VIOLATION;
 187     }
 188
 189     /* Check if this is a guard page */
 190     if ((ProtectionMask & MM_PROTECT_SPECIAL) == MM_GUARDPAGE)
 191     {
 192         NT_ASSERT(ProtectionMask != MM_DECOMMIT);
 193
 194         /* Attached processes can't expand their stack */
 195         if (KeIsAttachedProcess()) return STATUS_ACCESS_VIOLATION;
 196
 197         /* No support for transition PTEs yet */
 198         ASSERT(((TempPte.u.Soft.Transition == 1) &&
 199                 (TempPte.u.Soft.Prototype == 0)) == FALSE);
 200
 201         /* Remove the guard page bit, and return a guard page violation */
 202         TempPte.u.Soft.Protection = ProtectionMask & ~MM_GUARDPAGE;
 203         NT_ASSERT(TempPte.u.Long != 0);
 204         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 205         return STATUS_GUARD_PAGE_VIOLATION;
 206     }
 207
 208     /* Nothing to do */
 209     return STATUS_SUCCESS;
 210 }
 211
 212 PMMPTE
 213 NTAPI
 214 MiCheckVirtualAddress(IN PVOID VirtualAddress,
 215                       OUT PULONG ProtectCode,
 216                       OUT PMMVAD *ProtoVad)
 217 {
 218     PMMVAD Vad;
 219     PMMPTE PointerPte;
 220
 221     /* No prototype/section support for now */
 222     *ProtoVad = NULL;
 223
 224     /* User or kernel fault? */
 225     if (VirtualAddress <= MM_HIGHEST_USER_ADDRESS)
 226     {
 227         /* Special case for shared data */
 228         if (PAGE_ALIGN(VirtualAddress) == (PVOID)MM_SHARED_USER_DATA_VA)
 229         {
 230             /* It's a read-only page */
 231             *ProtectCode = MM_READONLY;
 232             return MmSharedUserDataPte;
 233         }
 234
 235         /* Find the VAD, it might not exist if the address is bogus */
 236         Vad = MiLocateAddress(VirtualAddress);
 237         if (!Vad)
 238         {
 239             /* Bogus virtual address */
 240             *ProtectCode = MM_NOACCESS;
 241             return NULL;
 242         }
 243
 244         /* ReactOS does not handle physical memory VADs yet */
 245         ASSERT(Vad->u.VadFlags.VadType != VadDevicePhysicalMemory);
 246
 247         /* Check if it's a section, or just an allocation */
 248         if (Vad->u.VadFlags.PrivateMemory)
 249         {
 250             /* ReactOS does not handle AWE VADs yet */
 251             ASSERT(Vad->u.VadFlags.VadType != VadAwe);
 252
 253             /* This must be a TEB/PEB VAD */
 254             if (Vad->u.VadFlags.MemCommit)
 255             {
 256                 /* It's committed, so return the VAD protection */
 257                 *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
 258             }
 259             else
 260             {
 261                 /* It has not yet been committed, so return no access */
 262                 *ProtectCode = MM_NOACCESS;
 263             }
 264
 265             /* In both cases, return no PTE */
 266             return NULL;
 267         }
 268         else
 269         {
 270             /* ReactOS does not supoprt these VADs yet */
 271             ASSERT(Vad->u.VadFlags.VadType != VadImageMap);
 272             ASSERT(Vad->u2.VadFlags2.ExtendableFile == 0);
 273
 274             /* Return the proto VAD */
 275             *ProtoVad = Vad;
 276
 277             /* Get the prototype PTE for this page */
 278             PointerPte = (((ULONG_PTR)VirtualAddress >> PAGE_SHIFT) - Vad->StartingVpn) + Vad->FirstPrototypePte;
 279             ASSERT(PointerPte != NULL);
 280             ASSERT(PointerPte <= Vad->LastContiguousPte);
 281
 282             /* Return the Prototype PTE and the protection for the page mapping */
 283             *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
 284             return PointerPte;
 285         }
 286     }
 287     else if (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress))
 288     {
 289         /* This should never happen, as these addresses are handled by the double-maping */
 290         if (((PMMPTE)VirtualAddress >= MiAddressToPte(MmPagedPoolStart)) &&
 291             ((PMMPTE)VirtualAddress <= MmPagedPoolInfo.LastPteForPagedPool))
 292         {
 293             /* Fail such access */
 294             *ProtectCode = MM_NOACCESS;
 295             return NULL;
 296         }
 297
 298         /* Return full access rights */
 299         *ProtectCode = MM_READWRITE;
 300         return NULL;
 301     }
 302     else if (MI_IS_SESSION_ADDRESS(VirtualAddress))
 303     {
 304         /* ReactOS does not have an image list yet, so bail out to failure case */
 305         ASSERT(IsListEmpty(&MmSessionSpace->ImageList));
 306     }
 307
 308     /* Default case -- failure */
 309     *ProtectCode = MM_NOACCESS;
 310     return NULL;
 311 }
 312
 313 #if (_MI_PAGING_LEVELS == 2)
 314 FORCEINLINE
 315 BOOLEAN
 316 MiSynchronizeSystemPde(PMMPDE PointerPde)
 317 {
 318     MMPDE SystemPde;
 319     ULONG Index;
 320
 321     /* Get the Index from the PDE */
 322     Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
 323
 324     /* Copy the PDE from the double-mapped system page directory */
 325     SystemPde = MmSystemPagePtes[Index];
 326     *PointerPde = SystemPde;
 327
 328     /* Make sure we re-read the PDE and PTE */
 329     KeMemoryBarrierWithoutFence();
 330
 331     /* Return, if we had success */
 332     return (BOOLEAN)SystemPde.u.Hard.Valid;
 333 }
 334
 335 NTSTATUS
 336 FASTCALL
 337 MiCheckPdeForSessionSpace(IN PVOID Address)
 338 {
 339     MMPTE TempPde;
 340     PMMPTE PointerPde;
 341     PVOID SessionAddress;
 342     ULONG Index;
 343
 344     /* Is this a session PTE? */
 345     if (MI_IS_SESSION_PTE(Address))
 346     {
 347         /* Make sure the PDE for session space is valid */
 348         PointerPde = MiAddressToPde(MmSessionSpace);
 349         if (!PointerPde->u.Hard.Valid)
 350         {
 351             /* This means there's no valid session, bail out */
 352             DbgPrint("MiCheckPdeForSessionSpace: No current session for PTE %p\n",
 353                      Address);
 354             DbgBreakPoint();
 355             return STATUS_ACCESS_VIOLATION;
 356         }
 357
 358         /* Now get the session-specific page table for this address */
 359         SessionAddress = MiPteToAddress(Address);
 360         PointerPde = MiAddressToPte(Address);
 361         if (PointerPde->u.Hard.Valid) return STATUS_WAIT_1;
 362
 363         /* It's not valid, so find it in the page table array */
 364         Index = ((ULONG_PTR)SessionAddress - (ULONG_PTR)MmSessionBase) >> 22;
 365         TempPde.u.Long = MmSessionSpace->PageTables[Index].u.Long;
 366         if (TempPde.u.Hard.Valid)
 367         {
 368             /* The copy is valid, so swap it in */
 369             InterlockedExchange((PLONG)PointerPde, TempPde.u.Long);
 370             return STATUS_WAIT_1;
 371         }
 372
 373         /* We don't seem to have allocated a page table for this address yet? */
 374         DbgPrint("MiCheckPdeForSessionSpace: No Session PDE for PTE %p, %p\n",
 375                  PointerPde->u.Long, SessionAddress);
 376         DbgBreakPoint();
 377         return STATUS_ACCESS_VIOLATION;
 378     }
 379
 380     /* Is the address also a session address? If not, we're done */
 381     if (!MI_IS_SESSION_ADDRESS(Address)) return STATUS_SUCCESS;
 382
 383     /* It is, so again get the PDE for session space */
 384     PointerPde = MiAddressToPde(MmSessionSpace);
 385     if (!PointerPde->u.Hard.Valid)
 386     {
 387         /* This means there's no valid session, bail out */
 388         DbgPrint("MiCheckPdeForSessionSpace: No current session for VA %p\n",
 389                     Address);
 390         DbgBreakPoint();
 391         return STATUS_ACCESS_VIOLATION;
 392     }
 393
 394     /* Now get the PDE for the address itself */
 395     PointerPde = MiAddressToPde(Address);
 396     if (!PointerPde->u.Hard.Valid)
 397     {
 398         /* Do the swap, we should be good to go */
 399         Index = ((ULONG_PTR)Address - (ULONG_PTR)MmSessionBase) >> 22;
 400         PointerPde->u.Long = MmSessionSpace->PageTables[Index].u.Long;
 401         if (PointerPde->u.Hard.Valid) return STATUS_WAIT_1;
 402
 403         /* We had not allocated a page table for this session address yet, fail! */
 404         DbgPrint("MiCheckPdeForSessionSpace: No Session PDE for VA %p, %p\n",
 405                  PointerPde->u.Long, Address);
 406         DbgBreakPoint();
 407         return STATUS_ACCESS_VIOLATION;
 408     }
 409
 410     /* It's valid, so there's nothing to do */
 411     return STATUS_SUCCESS;
 412 }
 413
 414 NTSTATUS
 415 FASTCALL
 416 MiCheckPdeForPagedPool(IN PVOID Address)
 417 {
 418     PMMPDE PointerPde;
 419     NTSTATUS Status = STATUS_SUCCESS;
 420
 421     /* Check session PDE */
 422     if (MI_IS_SESSION_ADDRESS(Address)) return MiCheckPdeForSessionSpace(Address);
 423     if (MI_IS_SESSION_PTE(Address)) return MiCheckPdeForSessionSpace(Address);
 424
 425     //
 426     // Check if this is a fault while trying to access the page table itself
 427     //
 428     if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
 429     {
 430         //
 431         // Send a hint to the page fault handler that this is only a valid fault
 432         // if we already detected this was access within the page table range
 433         //
 434         PointerPde = (PMMPDE)MiAddressToPte(Address);
 435         Status = STATUS_WAIT_1;
 436     }
 437     else if (Address < MmSystemRangeStart)
 438     {
 439         //
 440         // This is totally illegal
 441         //
 442         return STATUS_ACCESS_VIOLATION;
 443     }
 444     else
 445     {
 446         //
 447         // Get the PDE for the address
 448         //
 449         PointerPde = MiAddressToPde(Address);
 450     }
 451
 452     //
 453     // Check if it's not valid
 454     //
 455     if (PointerPde->u.Hard.Valid == 0)
 456     {
 457         //
 458         // Copy it from our double-mapped system page directory
 459         //
 460         InterlockedExchangePte(PointerPde,
 461                                MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)].u.Long);
 462     }
 463
 464     //
 465     // Return status
 466     //
 467     return Status;
 468 }
 469 #else
 470 NTSTATUS
 471 FASTCALL
 472 MiCheckPdeForPagedPool(IN PVOID Address)
 473 {
 474     return STATUS_ACCESS_VIOLATION;
 475 }
 476 #endif
 477
 478 VOID
 479 NTAPI
 480 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
 481 {
 482     PMMPTE ZeroPte;
 483     MMPTE TempPte;
 484     PMMPFN Pfn1;
 485     PVOID ZeroAddress;
 486
 487     /* Get the PFN for this page */
 488     Pfn1 = MiGetPfnEntry(PageFrameNumber);
 489     ASSERT(Pfn1);
 490
 491     /* Grab a system PTE we can use to zero the page */
 492     ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
 493     ASSERT(ZeroPte);
 494
 495     /* Initialize the PTE for it */
 496     TempPte = ValidKernelPte;
 497     TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
 498
 499     /* Setup caching */
 500     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 501     {
 502         /* Write combining, no caching */
 503         MI_PAGE_DISABLE_CACHE(&TempPte);
 504         MI_PAGE_WRITE_COMBINED(&TempPte);
 505     }
 506     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 507     {
 508         /* Write through, no caching */
 509         MI_PAGE_DISABLE_CACHE(&TempPte);
 510         MI_PAGE_WRITE_THROUGH(&TempPte);
 511     }
 512
 513     /* Make the system PTE valid with our PFN */
 514     MI_WRITE_VALID_PTE(ZeroPte, TempPte);
 515
 516     /* Get the address it maps to, and zero it out */
 517     ZeroAddress = MiPteToAddress(ZeroPte);
 518     KeZeroPages(ZeroAddress, PAGE_SIZE);
 519
 520     /* Now get rid of it */
 521     MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
 522 }
 523
 524 NTSTATUS
 525 NTAPI
 526 MiResolveDemandZeroFault(IN PVOID Address,
 527                          IN PMMPTE PointerPte,
 528                          IN PEPROCESS Process,
 529                          IN KIRQL OldIrql)
 530 {
 531     PFN_NUMBER PageFrameNumber = 0;
 532     MMPTE TempPte;
 533     BOOLEAN NeedZero = FALSE, HaveLock = FALSE;
 534     ULONG Color;
 535     PMMPFN Pfn1;
 536     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
 537             Address,
 538             Process);
 539
 540     /* Must currently only be called by paging path */
 541     if ((Process > HYDRA_PROCESS) && (OldIrql == MM_NOIRQL))
 542     {
 543         /* Sanity check */
 544         ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
 545
 546         /* No forking yet */
 547         ASSERT(Process->ForkInProgress == NULL);
 548
 549         /* Get process color */
 550         Color = MI_GET_NEXT_PROCESS_COLOR(Process);
 551         ASSERT(Color != 0xFFFFFFFF);
 552
 553         /* We'll need a zero page */
 554         NeedZero = TRUE;
 555     }
 556     else
 557     {
 558         /* Check if we need a zero page */
 559         NeedZero = (OldIrql != MM_NOIRQL);
 560
 561         /* Session-backed image views must be zeroed */
 562         if ((Process == HYDRA_PROCESS) &&
 563             ((MI_IS_SESSION_IMAGE_ADDRESS(Address)) ||
 564              ((Address >= MiSessionViewStart) && (Address < MiSessionSpaceWs))))
 565         {
 566             NeedZero = TRUE;
 567         }
 568
 569         /* Hardcode unknown color */
 570         Color = 0xFFFFFFFF;
 571     }
 572
 573     /* Check if the PFN database should be acquired */
 574     if (OldIrql == MM_NOIRQL)
 575     {
 576         /* Acquire it and remember we should release it after */
 577         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 578         HaveLock = TRUE;
 579     }
 580
 581     /* We either manually locked the PFN DB, or already came with it locked */
 582     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 583     ASSERT(PointerPte->u.Hard.Valid == 0);
 584
 585     /* Assert we have enough pages */
 586     ASSERT(MmAvailablePages >= 32);
 587
 588 #if MI_TRACE_PFNS
 589     if (UserPdeFault) MI_SET_USAGE(MI_USAGE_PAGE_TABLE);
 590     if (!UserPdeFault) MI_SET_USAGE(MI_USAGE_DEMAND_ZERO);
 591 #endif
 592     if (Process) MI_SET_PROCESS2(Process->ImageFileName);
 593     if (!Process) MI_SET_PROCESS2("Kernel Demand 0");
 594
 595     /* Do we need a zero page? */
 596     if (Color != 0xFFFFFFFF)
 597     {
 598         /* Try to get one, if we couldn't grab a free page and zero it */
 599         PageFrameNumber = MiRemoveZeroPageSafe(Color);
 600         if (!PageFrameNumber)
 601         {
 602             /* We'll need a free page and zero it manually */
 603             PageFrameNumber = MiRemoveAnyPage(Color);
 604             NeedZero = TRUE;
 605         }
 606     }
 607     else
 608     {
 609         /* Get a color, and see if we should grab a zero or non-zero page */
 610         Color = MI_GET_NEXT_COLOR();
 611         if (!NeedZero)
 612         {
 613             /* Process or system doesn't want a zero page, grab anything */
 614             PageFrameNumber = MiRemoveAnyPage(Color);
 615         }
 616         else
 617         {
 618             /* System wants a zero page, obtain one */
 619             PageFrameNumber = MiRemoveZeroPage(Color);
 620         }
 621     }
 622
 623     /* Initialize it */
 624     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
 625
 626     /* Do we have the lock? */
 627     if (HaveLock)
 628     {
 629         /* Release it */
 630         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 631
 632         /* Update performance counters */
 633         if (Process > HYDRA_PROCESS) Process->NumberOfPrivatePages++;
 634     }
 635
 636     /* Increment demand zero faults */
 637     InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
 638
 639     /* Zero the page if need be */
 640     if (NeedZero) MiZeroPfn(PageFrameNumber);
 641
 642     /* Fault on user PDE, or fault on user PTE? */
 643     if (PointerPte <= MiHighestUserPte)
 644     {
 645         /* User fault, build a user PTE */
 646         MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 647                                   PointerPte,
 648                                   PointerPte->u.Soft.Protection,
 649                                   PageFrameNumber);
 650     }
 651     else
 652     {
 653         /* This is a user-mode PDE, create a kernel PTE for it */
 654         MI_MAKE_HARDWARE_PTE(&TempPte,
 655                              PointerPte,
 656                              PointerPte->u.Soft.Protection,
 657                              PageFrameNumber);
 658     }
 659
 660     /* Set it dirty if it's a writable page */
 661     if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
 662
 663     /* Write it */
 664     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 665
 666     /* Did we manually acquire the lock */
 667     if (HaveLock)
 668     {
 669         /* Get the PFN entry */
 670         Pfn1 = MI_PFN_ELEMENT(PageFrameNumber);
 671
 672         /* Windows does these sanity checks */
 673         ASSERT(Pfn1->u1.Event == 0);
 674         ASSERT(Pfn1->u3.e1.PrototypePte == 0);
 675     }
 676
 677     //
 678     // It's all good now
 679     //
 680     DPRINT("Demand zero page has now been paged in\n");
 681     return STATUS_PAGE_FAULT_DEMAND_ZERO;
 682 }
 683
 684 NTSTATUS
 685 NTAPI
 686 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
 687                         IN PVOID Address,
 688                         IN PMMPTE PointerPte,
 689                         IN PMMPTE PointerProtoPte,
 690                         IN KIRQL OldIrql,
 691                         IN PMMPFN* LockedProtoPfn)
 692 {
 693     MMPTE TempPte;
 694     PMMPTE OriginalPte, PageTablePte;
 695     ULONG_PTR Protection;
 696     PFN_NUMBER PageFrameIndex;
 697     PMMPFN Pfn1, Pfn2;
 698     BOOLEAN OriginalProtection, DirtyPage;
 699
 700     /* Must be called with an valid prototype PTE, with the PFN lock held */
 701     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 702     ASSERT(PointerProtoPte->u.Hard.Valid == 1);
 703
 704     /* Get the page */
 705     PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
 706
 707     /* Get the PFN entry and set it as a prototype PTE */
 708     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 709     Pfn1->u3.e1.PrototypePte = 1;
 710
 711     /* Increment the share count for the page table */
 712     // FIXME: This doesn't work because we seem to bump the sharecount to two, and MiDeletePte gets annoyed and ASSERTs.
 713     // This could be beause MiDeletePte is now being called from strange code in Rosmm
 714     PageTablePte = MiAddressToPte(PointerPte);
 715     Pfn2 = MiGetPfnEntry(PageTablePte->u.Hard.PageFrameNumber);
 716     //Pfn2->u2.ShareCount++;
 717     DBG_UNREFERENCED_LOCAL_VARIABLE(Pfn2);
 718
 719     /* Check where we should be getting the protection information from */
 720     if (PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
 721     {
 722         /* Get the protection from the PTE, there's no real Proto PTE data */
 723         Protection = PointerPte->u.Soft.Protection;
 724
 725         /* Remember that we did not use the proto protection */
 726         OriginalProtection = FALSE;
 727     }
 728     else
 729     {
 730         /* Get the protection from the original PTE link */
 731         OriginalPte = &Pfn1->OriginalPte;
 732         Protection = OriginalPte->u.Soft.Protection;
 733
 734         /* Remember that we used the original protection */
 735         OriginalProtection = TRUE;
 736
 737         /* Check if this was a write on a read only proto */
 738         if ((StoreInstruction) && !(Protection & MM_READWRITE))
 739         {
 740             /* Clear the flag */
 741             StoreInstruction = 0;
 742         }
 743     }
 744
 745     /* Check if this was a write on a non-COW page */
 746     DirtyPage = FALSE;
 747     if ((StoreInstruction) && ((Protection & MM_WRITECOPY) != MM_WRITECOPY))
 748     {
 749         /* Then the page should be marked dirty */
 750         DirtyPage = TRUE;
 751
 752         /* ReactOS check */
 753         ASSERT(Pfn1->OriginalPte.u.Soft.Prototype != 0);
 754     }
 755
 756     /* Did we get a locked incoming PFN? */
 757     if (*LockedProtoPfn)
 758     {
 759         /* Drop a reference */
 760         ASSERT((*LockedProtoPfn)->u3.e2.ReferenceCount >= 1);
 761         MiDereferencePfnAndDropLockCount(*LockedProtoPfn);
 762         *LockedProtoPfn = NULL;
 763     }
 764
 765     /* Release the PFN lock */
 766     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 767
 768     /* Remove special/caching bits */
 769     Protection &= ~MM_PROTECT_SPECIAL;
 770
 771     /* Setup caching */
 772     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 773     {
 774         /* Write combining, no caching */
 775         MI_PAGE_DISABLE_CACHE(&TempPte);
 776         MI_PAGE_WRITE_COMBINED(&TempPte);
 777     }
 778     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 779     {
 780         /* Write through, no caching */
 781         MI_PAGE_DISABLE_CACHE(&TempPte);
 782         MI_PAGE_WRITE_THROUGH(&TempPte);
 783     }
 784
 785     /* Check if this is a kernel or user address */
 786     if (Address < MmSystemRangeStart)
 787     {
 788         /* Build the user PTE */
 789         MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, Protection, PageFrameIndex);
 790     }
 791     else
 792     {
 793         /* Build the kernel PTE */
 794         MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, Protection, PageFrameIndex);
 795     }
 796
 797     /* Set the dirty flag if needed */
 798     if (DirtyPage) TempPte.u.Hard.Dirty = TRUE;
 799
 800     /* Write the PTE */
 801     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 802
 803     /* Reset the protection if needed */
 804     if (OriginalProtection) Protection = MM_ZERO_ACCESS;
 805
 806     /* Return success */
 807     ASSERT(PointerPte == MiAddressToPte(Address));
 808     return STATUS_SUCCESS;
 809 }
 810
 811 NTSTATUS
 812 NTAPI
 813 MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction,
 814                        _In_ PVOID FaultingAddress,
 815                        _In_ PMMPTE PointerPte,
 816                        _In_ PEPROCESS CurrentProcess,
 817                        _Inout_ KIRQL *OldIrql)
 818 {
 819     ULONG Color;
 820     PFN_NUMBER Page;
 821     NTSTATUS Status;
 822     MMPTE TempPte = *PointerPte;
 823     KEVENT Event;
 824     PMMPFN Pfn1;
 825     ULONG PageFileIndex = TempPte.u.Soft.PageFileLow;
 826     ULONG_PTR PageFileOffset = TempPte.u.Soft.PageFileHigh;
 827
 828     /* Things we don't support yet */
 829     ASSERT(CurrentProcess > HYDRA_PROCESS);
 830     ASSERT(*OldIrql != MM_NOIRQL);
 831
 832     /* We must hold the PFN lock */
 833     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 834
 835     /* Some sanity checks */
 836     ASSERT(TempPte.u.Hard.Valid == 0);
 837     ASSERT(TempPte.u.Soft.PageFileHigh != 0);
 838     ASSERT(TempPte.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED);
 839
 840     /* Get any page, it will be overwritten */
 841     Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
 842     Page = MiRemoveAnyPage(Color);
 843
 844     /* Initialize this PFN */
 845     MiInitializePfn(Page, PointerPte, StoreInstruction);
 846
 847     /* Sets the PFN as being in IO operation */
 848     Pfn1 = MI_PFN_ELEMENT(Page);
 849     ASSERT(Pfn1->u1.Event == NULL);
 850     ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
 851     ASSERT(Pfn1->u3.e1.WriteInProgress == 0);
 852
 853     KeInitializeEvent(&Event, NotificationEvent, FALSE);
 854     Pfn1->u1.Event = &Event;
 855     Pfn1->u3.e1.ReadInProgress = 1;
 856
 857     /* We must write the PTE now as the PFN lock will be released while performing the IO operation */
 858     TempPte.u.Soft.Transition = 1;
 859     TempPte.u.Soft.PageFileLow = 0;
 860     TempPte.u.Soft.Prototype = 0;
 861     TempPte.u.Trans.PageFrameNumber = Page;
 862
 863     MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 864
 865     /* Release the PFN lock while we proceed */
 866     KeReleaseQueuedSpinLock(LockQueuePfnLock, *OldIrql);
 867
 868     /* Do the paging IO */
 869     Status = MiReadPageFile(Page, PageFileIndex, PageFileOffset);
 870
 871     /* Lock the PFN database again */
 872     *OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 873
 874     /* Nobody should have changed that while we were not looking */
 875     ASSERT(Pfn1->u1.Event == &Event);
 876     ASSERT(Pfn1->u3.e1.ReadInProgress == 1);
 877     ASSERT(Pfn1->u3.e1.WriteInProgress == 0);
 878
 879     if (!NT_SUCCESS(Status))
 880     {
 881         /* Malheur! */
 882         ASSERT(FALSE);
 883         Pfn1->u4.InPageError = 1;
 884         Pfn1->u1.ReadStatus = Status;
 885     }
 886
 887     /* This is now a nice and normal PFN */
 888     Pfn1->u1.Event = NULL;
 889     Pfn1->u3.e1.ReadInProgress = 0;
 890
 891     /* And the PTE can finally be valid */
 892     MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, TempPte.u.Trans.Protection, Page);
 893     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 894
 895     /* Waiters gonna wait */
 896     KeSetEvent(&Event, IO_NO_INCREMENT, FALSE);
 897
 898     return Status;
 899 }
 900
 901 NTSTATUS
 902 NTAPI
 903 MiResolveTransitionFault(IN PVOID FaultingAddress,
 904                          IN PMMPTE PointerPte,
 905                          IN PEPROCESS CurrentProcess,
 906                          IN KIRQL OldIrql,
 907                          OUT PVOID *InPageBlock)
 908 {
 909     PFN_NUMBER PageFrameIndex;
 910     PMMPFN Pfn1;
 911     MMPTE TempPte;
 912     PMMPTE PointerToPteForProtoPage;
 913     DPRINT1("Transition fault on 0x%p with PTE 0x%p in process %s\n",
 914             FaultingAddress, PointerPte, CurrentProcess->ImageFileName);
 915
 916     /* Windowss does this check */
 917     ASSERT(*InPageBlock == NULL);
 918
 919     /* ARM3 doesn't support this path */
 920     ASSERT(OldIrql != MM_NOIRQL);
 921
 922     /* Capture the PTE and make sure it's in transition format */
 923     TempPte = *PointerPte;
 924     ASSERT((TempPte.u.Soft.Valid == 0) &&
 925            (TempPte.u.Soft.Prototype == 0) &&
 926            (TempPte.u.Soft.Transition == 1));
 927
 928     /* Get the PFN and the PFN entry */
 929     PageFrameIndex = TempPte.u.Trans.PageFrameNumber;
 930     DPRINT1("Transition PFN: %lx\n", PageFrameIndex);
 931     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 932
 933     /* One more transition fault! */
 934     InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
 935
 936     /* This is from ARM3 -- Windows normally handles this here */
 937     ASSERT(Pfn1->u4.InPageError == 0);
 938
 939     /* See if we should wait before terminating the fault */
 940     if (Pfn1->u3.e1.ReadInProgress == 1)
 941     {
 942         DPRINT1("The page is currently being read!\n");
 943         ASSERT(Pfn1->u1.Event != NULL);
 944         *InPageBlock = Pfn1->u1.Event;
 945         if (PointerPte == Pfn1->PteAddress)
 946         {
 947             DPRINT1("And this if for this particular PTE.\n");
 948             /* The PTE will be made valid by the thread serving the fault */
 949             return STATUS_SUCCESS; // FIXME: Maybe something more descriptive
 950         }
 951     }
 952
 953     /* Windows checks there's some free pages and this isn't an in-page error */
 954     ASSERT(MmAvailablePages > 0);
 955     ASSERT(Pfn1->u4.InPageError == 0);
 956
 957     /* ReactOS checks for this */
 958     ASSERT(MmAvailablePages > 32);
 959
 960     /* Was this a transition page in the valid list, or free/zero list? */
 961     if (Pfn1->u3.e1.PageLocation == ActiveAndValid)
 962     {
 963         /* All Windows does here is a bunch of sanity checks */
 964         DPRINT1("Transition in active list\n");
 965         ASSERT((Pfn1->PteAddress >= MiAddressToPte(MmPagedPoolStart)) &&
 966                (Pfn1->PteAddress <= MiAddressToPte(MmPagedPoolEnd)));
 967         ASSERT(Pfn1->u2.ShareCount != 0);
 968         ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 969     }
 970     else
 971     {
 972         /* Otherwise, the page is removed from its list */
 973         DPRINT1("Transition page in free/zero list\n");
 974         MiUnlinkPageFromList(Pfn1);
 975         MiReferenceUnusedPageAndBumpLockCount(Pfn1);
 976     }
 977
 978     /* At this point, there should no longer be any in-page errors */
 979     ASSERT(Pfn1->u4.InPageError == 0);
 980
 981     /* Check if this was a PFN with no more share references */
 982     if (Pfn1->u2.ShareCount == 0) MiDropLockCount(Pfn1);
 983
 984     /* Bump the share count and make the page valid */
 985     Pfn1->u2.ShareCount++;
 986     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 987
 988     /* Prototype PTEs are in paged pool, which itself might be in transition */
 989     if (FaultingAddress >= MmSystemRangeStart)
 990     {
 991         /* Check if this is a paged pool PTE in transition state */
 992         PointerToPteForProtoPage = MiAddressToPte(PointerPte);
 993         TempPte = *PointerToPteForProtoPage;
 994         if ((TempPte.u.Hard.Valid == 0) && (TempPte.u.Soft.Transition == 1))
 995         {
 996             /* This isn't yet supported */
 997             DPRINT1("Double transition fault not yet supported\n");
 998             ASSERT(FALSE);
 999         }
1000     }
1001
1002     /* Build the transition PTE -- maybe a macro? */
1003     ASSERT(PointerPte->u.Hard.Valid == 0);
1004     ASSERT(PointerPte->u.Trans.Prototype == 0);
1005     ASSERT(PointerPte->u.Trans.Transition == 1);
1006     TempPte.u.Long = (PointerPte->u.Long & ~0xFFF) |
1007                      (MmProtectToPteMask[PointerPte->u.Trans.Protection]) |
1008                      MiDetermineUserGlobalPteMask(PointerPte);
1009
1010     /* Is the PTE writeable? */
1011     if (((Pfn1->u3.e1.Modified) && (TempPte.u.Hard.Write)) &&
1012         (TempPte.u.Hard.CopyOnWrite == 0))
1013     {
1014         /* Make it dirty */
1015         TempPte.u.Hard.Dirty = TRUE;
1016     }
1017     else
1018     {
1019         /* Make it clean */
1020         TempPte.u.Hard.Dirty = FALSE;
1021     }
1022
1023     /* Write the valid PTE */
1024     MI_WRITE_VALID_PTE(PointerPte, TempPte);
1025
1026     /* Return success */
1027     return STATUS_PAGE_FAULT_TRANSITION;
1028 }
1029
1030 NTSTATUS
1031 NTAPI
1032 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
1033                        IN PVOID Address,
1034                        IN PMMPTE PointerPte,
1035                        IN PMMPTE PointerProtoPte,
1036                        IN OUT PMMPFN *OutPfn,
1037                        OUT PVOID *PageFileData,
1038                        OUT PMMPTE PteValue,
1039                        IN PEPROCESS Process,
1040                        IN KIRQL OldIrql,
1041                        IN PVOID TrapInformation)
1042 {
1043     MMPTE TempPte, PteContents;
1044     PMMPFN Pfn1;
1045     PFN_NUMBER PageFrameIndex;
1046     NTSTATUS Status;
1047     PVOID InPageBlock = NULL;
1048
1049     /* Must be called with an invalid, prototype PTE, with the PFN lock held */
1050     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1051     ASSERT(PointerPte->u.Hard.Valid == 0);
1052     ASSERT(PointerPte->u.Soft.Prototype == 1);
1053
1054     /* Read the prototype PTE and check if it's valid */
1055     TempPte = *PointerProtoPte;
1056     if (TempPte.u.Hard.Valid == 1)
1057     {
1058         /* One more user of this mapped page */
1059         PageFrameIndex = PFN_FROM_PTE(&TempPte);
1060         Pfn1 = MiGetPfnEntry(PageFrameIndex);
1061         Pfn1->u2.ShareCount++;
1062
1063         /* Call it a transition */
1064         InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
1065
1066         /* Complete the prototype PTE fault -- this will release the PFN lock */
1067         return MiCompleteProtoPteFault(StoreInstruction,
1068                                        Address,
1069                                        PointerPte,
1070                                        PointerProtoPte,
1071                                        OldIrql,
1072                                        OutPfn);
1073     }
1074
1075     /* Make sure there's some protection mask */
1076     if (TempPte.u.Long == 0)
1077     {
1078         /* Release the lock */
1079         DPRINT1("Access on reserved section?\n");
1080         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1081         return STATUS_ACCESS_VIOLATION;
1082     }
1083
1084     /* There is no such thing as a decommitted prototype PTE */
1085     NT_ASSERT(TempPte.u.Long != MmDecommittedPte.u.Long);
1086
1087     /* Check for access rights on the PTE proper */
1088     PteContents = *PointerPte;
1089     if (PteContents.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED)
1090     {
1091         if (!PteContents.u.Proto.ReadOnly)
1092         {
1093             /* Check for page acess in software */
1094             Status = MiAccessCheck(PointerProtoPte,
1095                                    StoreInstruction,
1096                                    KernelMode,
1097                                    TempPte.u.Soft.Protection,
1098                                    TrapInformation,
1099                                    TRUE);
1100             ASSERT(Status == STATUS_SUCCESS);
1101
1102             /* Check for copy on write page */
1103             if ((TempPte.u.Soft.Protection & MM_WRITECOPY) == MM_WRITECOPY)
1104             {
1105                 /* Not yet supported */
1106                 ASSERT(FALSE);
1107             }
1108         }
1109     }
1110     else
1111     {
1112         /* Check for copy on write page */
1113         if ((PteContents.u.Soft.Protection & MM_WRITECOPY) == MM_WRITECOPY)
1114         {
1115             /* Not yet supported */
1116             ASSERT(FALSE);
1117         }
1118     }
1119
1120     /* Check for clone PTEs */
1121     if (PointerPte <= MiHighestUserPte) ASSERT(Process->CloneRoot == NULL);
1122
1123     /* We don't support mapped files yet */
1124     ASSERT(TempPte.u.Soft.Prototype == 0);
1125
1126     /* We might however have transition PTEs */
1127     if (TempPte.u.Soft.Transition == 1)
1128     {
1129         /* Resolve the transition fault */
1130         ASSERT(OldIrql != MM_NOIRQL);
1131         Status = MiResolveTransitionFault(Address,
1132                                           PointerProtoPte,
1133                                           Process,
1134                                           OldIrql,
1135                                           &InPageBlock);
1136         ASSERT(NT_SUCCESS(Status));
1137     }
1138     else
1139     {
1140         /* We also don't support paged out pages */
1141         ASSERT(TempPte.u.Soft.PageFileHigh == 0);
1142
1143         /* Resolve the demand zero fault */
1144         Status = MiResolveDemandZeroFault(Address,
1145                                           PointerProtoPte,
1146                                           Process,
1147                                           OldIrql);
1148         ASSERT(NT_SUCCESS(Status));
1149     }
1150
1151     /* Complete the prototype PTE fault -- this will release the PFN lock */
1152     ASSERT(PointerPte->u.Hard.Valid == 0);
1153     return MiCompleteProtoPteFault(StoreInstruction,
1154                                    Address,
1155                                    PointerPte,
1156                                    PointerProtoPte,
1157                                    OldIrql,
1158                                    OutPfn);
1159 }
1160
1161 NTSTATUS
1162 NTAPI
1163 MiDispatchFault(IN BOOLEAN StoreInstruction,
1164                 IN PVOID Address,
1165                 IN PMMPTE PointerPte,
1166                 IN PMMPTE PointerProtoPte,
1167                 IN BOOLEAN Recursive,
1168                 IN PEPROCESS Process,
1169                 IN PVOID TrapInformation,
1170                 IN PMMVAD Vad)
1171 {
1172     MMPTE TempPte;
1173     KIRQL OldIrql, LockIrql;
1174     NTSTATUS Status;
1175     PMMPTE SuperProtoPte;
1176     PMMPFN Pfn1, OutPfn = NULL;
1177     PFN_NUMBER PageFrameIndex;
1178     PFN_COUNT PteCount, ProcessedPtes;
1179     DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
1180              Address,
1181              Process);
1182
1183     /* Make sure the addresses are ok */
1184     ASSERT(PointerPte == MiAddressToPte(Address));
1185
1186     //
1187     // Make sure APCs are off and we're not at dispatch
1188     //
1189     OldIrql = KeGetCurrentIrql();
1190     ASSERT(OldIrql <= APC_LEVEL);
1191     ASSERT(KeAreAllApcsDisabled() == TRUE);
1192
1193     //
1194     // Grab a copy of the PTE
1195     //
1196     TempPte = *PointerPte;
1197
1198     /* Do we have a prototype PTE? */
1199     if (PointerProtoPte)
1200     {
1201         /* This should never happen */
1202         ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
1203
1204         /* Check if this is a kernel-mode address */
1205         SuperProtoPte = MiAddressToPte(PointerProtoPte);
1206         if (Address >= MmSystemRangeStart)
1207         {
1208             /* Lock the PFN database */
1209             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1210
1211             /* Has the PTE been made valid yet? */
1212             if (!SuperProtoPte->u.Hard.Valid)
1213             {
1214                 ASSERT(FALSE);
1215             }
1216             else if (PointerPte->u.Hard.Valid == 1)
1217             {
1218                 ASSERT(FALSE);
1219             }
1220
1221             /* Resolve the fault -- this will release the PFN lock */
1222             Status = MiResolveProtoPteFault(StoreInstruction,
1223                                             Address,
1224                                             PointerPte,
1225                                             PointerProtoPte,
1226                                             &OutPfn,
1227                                             NULL,
1228                                             NULL,
1229                                             Process,
1230                                             LockIrql,
1231                                             TrapInformation);
1232             ASSERT(Status == STATUS_SUCCESS);
1233
1234             /* Complete this as a transition fault */
1235             ASSERT(OldIrql == KeGetCurrentIrql());
1236             ASSERT(OldIrql <= APC_LEVEL);
1237             ASSERT(KeAreAllApcsDisabled() == TRUE);
1238             return Status;
1239         }
1240         else
1241         {
1242             /* We only handle the lookup path */
1243             ASSERT(PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED);
1244
1245             /* Is there a non-image VAD? */
1246             if ((Vad) &&
1247                 (Vad->u.VadFlags.VadType != VadImageMap) &&
1248                 !(Vad->u2.VadFlags2.ExtendableFile))
1249             {
1250                 /* One day, ReactOS will cluster faults */
1251                 ASSERT(Address <= MM_HIGHEST_USER_ADDRESS);
1252                 DPRINT("Should cluster fault, but won't\n");
1253             }
1254
1255             /* Only one PTE to handle for now */
1256             PteCount = 1;
1257             ProcessedPtes = 0;
1258
1259             /* Lock the PFN database */
1260             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1261
1262             /* We only handle the valid path */
1263             ASSERT(SuperProtoPte->u.Hard.Valid == 1);
1264
1265             /* Capture the PTE */
1266             TempPte = *PointerProtoPte;
1267
1268             /* Loop to handle future case of clustered faults */
1269             while (TRUE)
1270             {
1271                 /* For our current usage, this should be true */
1272                 if (TempPte.u.Hard.Valid == 1)
1273                 {
1274                     /* Bump the share count on the PTE */
1275                     PageFrameIndex = PFN_FROM_PTE(&TempPte);
1276                     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1277                     Pfn1->u2.ShareCount++;
1278                 }
1279                 else if ((TempPte.u.Soft.Prototype == 0) &&
1280                          (TempPte.u.Soft.Transition == 1))
1281                 {
1282                     /* This is a standby page, bring it back from the cache */
1283                     PageFrameIndex = TempPte.u.Trans.PageFrameNumber;
1284                     DPRINT("oooh, shiny, a soft fault! 0x%lx\n", PageFrameIndex);
1285                     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1286                     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1287
1288                     /* Should not yet happen in ReactOS */
1289                     ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
1290                     ASSERT(Pfn1->u4.InPageError == 0);
1291
1292                     /* Get the page */
1293                     MiUnlinkPageFromList(Pfn1);
1294
1295                     /* Bump its reference count */
1296                     ASSERT(Pfn1->u2.ShareCount == 0);
1297                     InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1298                     Pfn1->u2.ShareCount++;
1299
1300                     /* Make it valid again */
1301                     /* This looks like another macro.... */
1302                     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1303                     ASSERT(PointerProtoPte->u.Hard.Valid == 0);
1304                     ASSERT(PointerProtoPte->u.Trans.Prototype == 0);
1305                     ASSERT(PointerProtoPte->u.Trans.Transition == 1);
1306                     TempPte.u.Long = (PointerProtoPte->u.Long & ~0xFFF) |
1307                                      MmProtectToPteMask[PointerProtoPte->u.Trans.Protection];
1308                     TempPte.u.Hard.Valid = 1;
1309                     TempPte.u.Hard.Accessed = 1;
1310
1311                     /* Is the PTE writeable? */
1312                     if (((Pfn1->u3.e1.Modified) && (TempPte.u.Hard.Write)) &&
1313                         (TempPte.u.Hard.CopyOnWrite == 0))
1314                     {
1315                         /* Make it dirty */
1316                         TempPte.u.Hard.Dirty = TRUE;
1317                     }
1318                     else
1319                     {
1320                         /* Make it clean */
1321                         TempPte.u.Hard.Dirty = FALSE;
1322                     }
1323
1324                     /* Write the valid PTE */
1325                     MI_WRITE_VALID_PTE(PointerProtoPte, TempPte);
1326                     ASSERT(PointerPte->u.Hard.Valid == 0);
1327                 }
1328                 else
1329                 {
1330                     /* Page is invalid, get out of the loop */
1331                     break;
1332                 }
1333
1334                 /* One more done, was it the last? */
1335                 if (++ProcessedPtes == PteCount)
1336                 {
1337                     /* Complete the fault */
1338                     MiCompleteProtoPteFault(StoreInstruction,
1339                                             Address,
1340                                             PointerPte,
1341                                             PointerProtoPte,
1342                                             LockIrql,
1343                                             &OutPfn);
1344
1345                     /* THIS RELEASES THE PFN LOCK! */
1346                     break;
1347                 }
1348
1349                 /* No clustered faults yet */
1350                 ASSERT(FALSE);
1351             }
1352
1353             /* Did we resolve the fault? */
1354             if (ProcessedPtes)
1355             {
1356                 /* Bump the transition count */
1357                 InterlockedExchangeAddSizeT(&KeGetCurrentPrcb()->MmTransitionCount, ProcessedPtes);
1358                 ProcessedPtes--;
1359
1360                 /* Loop all the processing we did */
1361                 ASSERT(ProcessedPtes == 0);
1362
1363                 /* Complete this as a transition fault */
1364                 ASSERT(OldIrql == KeGetCurrentIrql());
1365                 ASSERT(OldIrql <= APC_LEVEL);
1366                 ASSERT(KeAreAllApcsDisabled() == TRUE);
1367                 return STATUS_PAGE_FAULT_TRANSITION;
1368             }
1369
1370             /* We did not -- PFN lock is still held, prepare to resolve prototype PTE fault */
1371             OutPfn = MI_PFN_ELEMENT(SuperProtoPte->u.Hard.PageFrameNumber);
1372             MiReferenceUsedPageAndBumpLockCount(OutPfn);
1373             ASSERT(OutPfn->u3.e2.ReferenceCount > 1);
1374             ASSERT(PointerPte->u.Hard.Valid == 0);
1375
1376             /* Resolve the fault -- this will release the PFN lock */
1377             Status = MiResolveProtoPteFault(StoreInstruction,
1378                                             Address,
1379                                             PointerPte,
1380                                             PointerProtoPte,
1381                                             &OutPfn,
1382                                             NULL,
1383                                             NULL,
1384                                             Process,
1385                                             LockIrql,
1386                                             TrapInformation);
1387             //ASSERT(Status != STATUS_ISSUE_PAGING_IO);
1388             //ASSERT(Status != STATUS_REFAULT);
1389             //ASSERT(Status != STATUS_PTE_CHANGED);
1390
1391             /* Did the routine clean out the PFN or should we? */
1392             if (OutPfn)
1393             {
1394                 /* We had a locked PFN, so acquire the PFN lock to dereference it */
1395                 ASSERT(PointerProtoPte != NULL);
1396                 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1397
1398                 /* Dereference the locked PFN */
1399                 MiDereferencePfnAndDropLockCount(OutPfn);
1400                 ASSERT(OutPfn->u3.e2.ReferenceCount >= 1);
1401
1402                 /* And now release the lock */
1403                 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1404             }
1405
1406             /* Complete this as a transition fault */
1407             ASSERT(OldIrql == KeGetCurrentIrql());
1408             ASSERT(OldIrql <= APC_LEVEL);
1409             ASSERT(KeAreAllApcsDisabled() == TRUE);
1410             return Status;
1411         }
1412     }
1413
1414     /* Is this a transition PTE */
1415     if (TempPte.u.Soft.Transition)
1416     {
1417         PVOID InPageBlock = NULL;
1418         /* Lock the PFN database */
1419         LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1420
1421         /* Resolve */
1422         Status = MiResolveTransitionFault(Address, PointerPte, Process, LockIrql, &InPageBlock);
1423
1424         NT_ASSERT(NT_SUCCESS(Status));
1425
1426         /* And now release the lock and leave*/
1427         KeReleaseQueuedSpinLock(LockQueuePfnLock, LockIrql);
1428
1429         if (InPageBlock != NULL)
1430         {
1431                 /* The page is being paged in by another process */
1432                 KeWaitForSingleObject(InPageBlock, WrPageIn, KernelMode, FALSE, NULL);
1433         }
1434
1435         ASSERT(OldIrql == KeGetCurrentIrql());
1436         ASSERT(OldIrql <= APC_LEVEL);
1437         ASSERT(KeAreAllApcsDisabled() == TRUE);
1438         return Status;
1439     }
1440
1441     /* Should we page the data back in ? */
1442     if (TempPte.u.Soft.PageFileHigh != 0)
1443     {
1444         /* Lock the PFN database */
1445         LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1446
1447         /* Resolve */
1448         Status = MiResolvePageFileFault(StoreInstruction, Address, PointerPte, Process, &LockIrql);
1449
1450         /* And now release the lock and leave*/
1451         KeReleaseQueuedSpinLock(LockQueuePfnLock, LockIrql);
1452
1453         ASSERT(OldIrql == KeGetCurrentIrql());
1454         ASSERT(OldIrql <= APC_LEVEL);
1455         ASSERT(KeAreAllApcsDisabled() == TRUE);
1456         return Status;
1457     }
1458
1459     //
1460     // The PTE must be invalid but not completely empty. It must also not be a
1461     // prototype a transition or a paged-out PTE as those scenarii should've been handled above.
1462     // These are all Windows checks
1463     //
1464     ASSERT(TempPte.u.Hard.Valid == 0);
1465     ASSERT(TempPte.u.Soft.Prototype == 0);
1466     ASSERT(TempPte.u.Soft.Transition == 0);
1467     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
1468     ASSERT(TempPte.u.Long != 0);
1469
1470     //
1471     // If we got this far, the PTE can only be a demand zero PTE, which is what
1472     // we want. Go handle it!
1473     //
1474     Status = MiResolveDemandZeroFault(Address,
1475                                       PointerPte,
1476                                       Process,
1477                                       MM_NOIRQL);
1478     ASSERT(KeAreAllApcsDisabled() == TRUE);
1479     if (NT_SUCCESS(Status))
1480     {
1481         //
1482         // Make sure we're returning in a sane state and pass the status down
1483         //
1484         ASSERT(OldIrql == KeGetCurrentIrql());
1485         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1486         return Status;
1487     }
1488
1489     //
1490     // Generate an access fault
1491     //
1492     return STATUS_ACCESS_VIOLATION;
1493 }
1494
1495 NTSTATUS
1496 NTAPI
1497 MmArmAccessFault(IN BOOLEAN StoreInstruction,
1498                  IN PVOID Address,
1499                  IN KPROCESSOR_MODE Mode,
1500                  IN PVOID TrapInformation)
1501 {
1502     KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
1503     PMMPTE ProtoPte = NULL;
1504     PMMPTE PointerPte = MiAddressToPte(Address);
1505     PMMPDE PointerPde = MiAddressToPde(Address);
1506 #if (_MI_PAGING_LEVELS >= 3)
1507     PMMPDE PointerPpe = MiAddressToPpe(Address);
1508 #if (_MI_PAGING_LEVELS == 4)
1509     PMMPDE PointerPxe = MiAddressToPxe(Address);
1510 #endif
1511 #endif
1512     MMPTE TempPte;
1513     PETHREAD CurrentThread;
1514     PEPROCESS CurrentProcess;
1515     NTSTATUS Status;
1516     PMMSUPPORT WorkingSet;
1517     ULONG ProtectionCode;
1518     PMMVAD Vad;
1519     PFN_NUMBER PageFrameIndex;
1520     ULONG Color;
1521     BOOLEAN IsSessionAddress;
1522     PMMPFN Pfn1;
1523     DPRINT("ARM3 FAULT AT: %p\n", Address);
1524
1525     /* Check for page fault on high IRQL */
1526     if (OldIrql > APC_LEVEL)
1527     {
1528 #if (_MI_PAGING_LEVELS < 3)
1529         /* Could be a page table for paged pool, which we'll allow */
1530         if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address)) MiSynchronizeSystemPde((PMMPDE)PointerPte);
1531         MiCheckPdeForPagedPool(Address);
1532 #endif
1533         /* Check if any of the top-level pages are invalid */
1534         if (
1535 #if (_MI_PAGING_LEVELS == 4)
1536             (PointerPxe->u.Hard.Valid == 0) ||
1537 #endif
1538 #if (_MI_PAGING_LEVELS >= 3)
1539             (PointerPpe->u.Hard.Valid == 0) ||
1540 #endif
1541             (PointerPde->u.Hard.Valid == 0) ||
1542             (PointerPte->u.Hard.Valid == 0))
1543         {
1544             /* This fault is not valid, print out some debugging help */
1545             DbgPrint("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
1546                      Address,
1547                      OldIrql);
1548             if (TrapInformation)
1549             {
1550                 PKTRAP_FRAME TrapFrame = TrapInformation;
1551 #ifdef _M_IX86
1552                 DbgPrint("MM:***EIP %p, EFL %p\n", TrapFrame->Eip, TrapFrame->EFlags);
1553                 DbgPrint("MM:***EAX %p, ECX %p EDX %p\n", TrapFrame->Eax, TrapFrame->Ecx, TrapFrame->Edx);
1554                 DbgPrint("MM:***EBX %p, ESI %p EDI %p\n", TrapFrame->Ebx, TrapFrame->Esi, TrapFrame->Edi);
1555 #elif defined(_M_AMD64)
1556                 DbgPrint("MM:***RIP %p, EFL %p\n", TrapFrame->Rip, TrapFrame->EFlags);
1557                 DbgPrint("MM:***RAX %p, RCX %p RDX %p\n", TrapFrame->Rax, TrapFrame->Rcx, TrapFrame->Rdx);
1558                 DbgPrint("MM:***RBX %p, RSI %p RDI %p\n", TrapFrame->Rbx, TrapFrame->Rsi, TrapFrame->Rdi);
1559 #endif
1560             }
1561
1562             /* Tell the trap handler to fail */
1563             return STATUS_IN_PAGE_ERROR | 0x10000000;
1564         }
1565
1566         /* Not yet implemented in ReactOS */
1567         ASSERT(MI_IS_PAGE_LARGE(PointerPde) == FALSE);
1568         ASSERT(((StoreInstruction) && (PointerPte->u.Hard.CopyOnWrite)) == FALSE);
1569
1570         /* Check if this was a write */
1571         if (StoreInstruction)
1572         {
1573             /* Was it to a read-only page? */
1574             Pfn1 = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
1575             if (!(PointerPte->u.Long & PTE_READWRITE) &&
1576                 !(Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE))
1577             {
1578                 /* Crash with distinguished bugcheck code */
1579                 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
1580                              (ULONG_PTR)Address,
1581                              PointerPte->u.Long,
1582                              (ULONG_PTR)TrapInformation,
1583                              10);
1584             }
1585         }
1586
1587         /* Nothing is actually wrong */
1588         DPRINT1("Fault at IRQL %u is ok (%p)\n", OldIrql, Address);
1589         return STATUS_SUCCESS;
1590     }
1591
1592     /* Check for kernel fault address */
1593     if (Address >= MmSystemRangeStart)
1594     {
1595         /* Bail out, if the fault came from user mode */
1596         if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
1597
1598 #if (_MI_PAGING_LEVELS == 2)
1599         if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address)) MiSynchronizeSystemPde((PMMPDE)PointerPte);
1600         MiCheckPdeForPagedPool(Address);
1601 #endif
1602
1603         /* Check if the higher page table entries are invalid */
1604         if (
1605 #if (_MI_PAGING_LEVELS == 4)
1606             /* AMD64 system, check if PXE is invalid */
1607             (PointerPxe->u.Hard.Valid == 0) ||
1608 #endif
1609 #if (_MI_PAGING_LEVELS >= 3)
1610             /* PAE/AMD64 system, check if PPE is invalid */
1611             (PointerPpe->u.Hard.Valid == 0) ||
1612 #endif
1613             /* Always check if the PDE is valid */
1614             (PointerPde->u.Hard.Valid == 0))
1615         {
1616             /* PXE/PPE/PDE (still) not valid, kill the system */
1617             KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
1618                          (ULONG_PTR)Address,
1619                          StoreInstruction,
1620                          (ULONG_PTR)TrapInformation,
1621                          2);
1622         }
1623
1624         /* Not handling session faults yet */
1625         IsSessionAddress = MI_IS_SESSION_ADDRESS(Address);
1626
1627         /* The PDE is valid, so read the PTE */
1628         TempPte = *PointerPte;
1629         if (TempPte.u.Hard.Valid == 1)
1630         {
1631             /* Check if this was system space or session space */
1632             if (!IsSessionAddress)
1633             {
1634                 /* Check if the PTE is still valid under PFN lock */
1635                 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1636                 TempPte = *PointerPte;
1637                 if (TempPte.u.Hard.Valid)
1638                 {
1639                     /* Check if this was a write */
1640                     if (StoreInstruction)
1641                     {
1642                         /* Was it to a read-only page? */
1643                         Pfn1 = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
1644                         if (!(PointerPte->u.Long & PTE_READWRITE) &&
1645                             !(Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE))
1646                         {
1647                             /* Crash with distinguished bugcheck code */
1648                             KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
1649                                          (ULONG_PTR)Address,
1650                                          PointerPte->u.Long,
1651                                          (ULONG_PTR)TrapInformation,
1652                                          11);
1653                         }
1654                     }
1655                 }
1656
1657                 /* Release PFN lock and return all good */
1658                 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1659                 return STATUS_SUCCESS;
1660             }
1661         }
1662 #if (_MI_PAGING_LEVELS == 2)
1663         /* Check if this was a session PTE that needs to remap the session PDE */
1664         if (MI_IS_SESSION_PTE(Address))
1665         {
1666             /* Do the remapping */
1667             Status = MiCheckPdeForSessionSpace(Address);
1668             if (!NT_SUCCESS(Status))
1669             {
1670                 /* It failed, this address is invalid */
1671                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
1672                              (ULONG_PTR)Address,
1673                              StoreInstruction,
1674                              (ULONG_PTR)TrapInformation,
1675                              6);
1676             }
1677         }
1678 #else
1679
1680 _WARN("Session space stuff is not implemented yet!")
1681
1682 #endif
1683
1684         /* Check for a fault on the page table or hyperspace */
1685         if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
1686         {
1687 #if (_MI_PAGING_LEVELS < 3)
1688             /* Windows does this check but I don't understand why -- it's done above! */
1689             ASSERT(MiCheckPdeForPagedPool(Address) != STATUS_WAIT_1);
1690 #endif
1691             /* Handle this as a user mode fault */
1692             goto UserFault;
1693         }
1694
1695         /* Get the current thread */
1696         CurrentThread = PsGetCurrentThread();
1697
1698         /* What kind of address is this */
1699         if (!IsSessionAddress)
1700         {
1701             /* Use the system working set */
1702             WorkingSet = &MmSystemCacheWs;
1703             CurrentProcess = NULL;
1704
1705             /* Make sure we don't have a recursive working set lock */
1706             if ((CurrentThread->OwnsProcessWorkingSetExclusive) ||
1707                 (CurrentThread->OwnsProcessWorkingSetShared) ||
1708                 (CurrentThread->OwnsSystemWorkingSetExclusive) ||
1709                 (CurrentThread->OwnsSystemWorkingSetShared) ||
1710                 (CurrentThread->OwnsSessionWorkingSetExclusive) ||
1711                 (CurrentThread->OwnsSessionWorkingSetShared))
1712             {
1713                 /* Fail */
1714                 return STATUS_IN_PAGE_ERROR | 0x10000000;
1715             }
1716         }
1717         else
1718         {
1719             /* Use the session process and working set */
1720             CurrentProcess = HYDRA_PROCESS;
1721             WorkingSet = &MmSessionSpace->GlobalVirtualAddress->Vm;
1722
1723             /* Make sure we don't have a recursive working set lock */
1724             if ((CurrentThread->OwnsSessionWorkingSetExclusive) ||
1725                 (CurrentThread->OwnsSessionWorkingSetShared))
1726             {
1727                 /* Fail */
1728                 return STATUS_IN_PAGE_ERROR | 0x10000000;
1729             }
1730         }
1731
1732         /* Acquire the working set lock */
1733         KeRaiseIrql(APC_LEVEL, &LockIrql);
1734         MiLockWorkingSet(CurrentThread, WorkingSet);
1735
1736         /* Re-read PTE now that we own the lock */
1737         TempPte = *PointerPte;
1738         if (TempPte.u.Hard.Valid == 1)
1739         {
1740             /* Check if this was a write */
1741             if (StoreInstruction)
1742             {
1743                 /* Was it to a read-only page that is not copy on write? */
1744                 Pfn1 = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
1745                 if (!(TempPte.u.Long & PTE_READWRITE) &&
1746                     !(Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE) &&
1747                     !(TempPte.u.Hard.CopyOnWrite))
1748                 {
1749                     /* Case not yet handled */
1750                     ASSERT(!IsSessionAddress);
1751
1752                     /* Crash with distinguished bugcheck code */
1753                     KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
1754                                  (ULONG_PTR)Address,
1755                                  TempPte.u.Long,
1756                                  (ULONG_PTR)TrapInformation,
1757                                  12);
1758                 }
1759             }
1760
1761             /* Check for read-only write in session space */
1762             if ((IsSessionAddress) &&
1763                 (StoreInstruction) &&
1764                 !(TempPte.u.Hard.Write))
1765             {
1766                 /* Sanity check */
1767                 ASSERT(MI_IS_SESSION_IMAGE_ADDRESS(Address));
1768
1769                 /* Was this COW? */
1770                 if (TempPte.u.Hard.CopyOnWrite == 0)
1771                 {
1772                     /* Then this is not allowed */
1773                     KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
1774                                  (ULONG_PTR)Address,
1775                                  (ULONG_PTR)TempPte.u.Long,
1776                                  (ULONG_PTR)TrapInformation,
1777                                  13);
1778                 }
1779
1780                 /* Otherwise, handle COW */
1781                 ASSERT(FALSE);
1782             }
1783
1784             /* Release the working set */
1785             MiUnlockWorkingSet(CurrentThread, WorkingSet);
1786             KeLowerIrql(LockIrql);
1787
1788             /* Otherwise, the PDE was probably invalid, and all is good now */
1789             return STATUS_SUCCESS;
1790         }
1791
1792         /* Check one kind of prototype PTE */
1793         if (TempPte.u.Soft.Prototype)
1794         {
1795             /* Make sure protected pool is on, and that this is a pool address */
1796             if ((MmProtectFreedNonPagedPool) &&
1797                 (((Address >= MmNonPagedPoolStart) &&
1798                   (Address < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
1799                                      MmSizeOfNonPagedPoolInBytes))) ||
1800                  ((Address >= MmNonPagedPoolExpansionStart) &&
1801                   (Address < MmNonPagedPoolEnd))))
1802             {
1803                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
1804                 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL,
1805                              (ULONG_PTR)Address,
1806                              StoreInstruction,
1807                              Mode,
1808                              4);
1809             }
1810
1811             /* Get the prototype PTE! */
1812             ProtoPte = MiProtoPteToPte(&TempPte);
1813
1814             /* Do we need to locate the prototype PTE in session space? */
1815             if ((IsSessionAddress) &&
1816                 (TempPte.u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED))
1817             {
1818                 /* Yep, go find it as well as the VAD for it */
1819                 ProtoPte = MiCheckVirtualAddress(Address,
1820                                                  &ProtectionCode,
1821                                                  &Vad);
1822                 ASSERT(ProtoPte != NULL);
1823             }
1824         }
1825         else
1826         {
1827             /* We don't implement transition PTEs */
1828             ASSERT(TempPte.u.Soft.Transition == 0);
1829
1830             /* Check for no-access PTE */
1831             if (TempPte.u.Soft.Protection == MM_NOACCESS)
1832             {
1833                 /* Bugcheck the system! */
1834                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
1835                              (ULONG_PTR)Address,
1836                              StoreInstruction,
1837                              (ULONG_PTR)TrapInformation,
1838                              1);
1839             }
1840         }
1841
1842         /* Check for demand page */
1843         if ((StoreInstruction) &&
1844             !(ProtoPte) &&
1845             !(IsSessionAddress) &&
1846             !(TempPte.u.Hard.Valid))
1847         {
1848             /* Get the protection code */
1849             ASSERT(TempPte.u.Soft.Transition == 0);
1850             if (!(TempPte.u.Soft.Protection & MM_READWRITE))
1851             {
1852                 /* Bugcheck the system! */
1853                 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
1854                              (ULONG_PTR)Address,
1855                              TempPte.u.Long,
1856                              (ULONG_PTR)TrapInformation,
1857                              14);
1858             }
1859         }
1860
1861         /* Now do the real fault handling */
1862         Status = MiDispatchFault(StoreInstruction,
1863                                  Address,
1864                                  PointerPte,
1865                                  ProtoPte,
1866                                  FALSE,
1867                                  CurrentProcess,
1868                                  TrapInformation,
1869                                  NULL);
1870
1871         /* Release the working set */
1872         ASSERT(KeAreAllApcsDisabled() == TRUE);
1873         MiUnlockWorkingSet(CurrentThread, WorkingSet);
1874         KeLowerIrql(LockIrql);
1875
1876         /* We are done! */
1877         DPRINT("Fault resolved with status: %lx\n", Status);
1878         return Status;
1879     }
1880
1881     /* This is a user fault */
1882 UserFault:
1883     CurrentThread = PsGetCurrentThread();
1884     CurrentProcess = (PEPROCESS)CurrentThread->Tcb.ApcState.Process;
1885
1886     /* Lock the working set */
1887     MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
1888
1889 #if (_MI_PAGING_LEVELS == 4)
1890 // Note to Timo: You should call MiCheckVirtualAddress and also check if it's zero pte
1891 // also this is missing the page count increment
1892     /* Check if the PXE is valid */
1893     if (PointerPxe->u.Hard.Valid == 0)
1894     {
1895         /* Right now, we only handle scenarios where the PXE is totally empty */
1896         ASSERT(PointerPxe->u.Long == 0);
1897 #if 0
1898         /* Resolve a demand zero fault */
1899         Status = MiResolveDemandZeroFault(PointerPpe,
1900                                           MM_READWRITE,
1901                                           CurrentProcess,
1902                                           MM_NOIRQL);
1903 #endif
1904         /* We should come back with a valid PXE */
1905         ASSERT(PointerPxe->u.Hard.Valid == 1);
1906     }
1907 #endif
1908
1909 #if (_MI_PAGING_LEVELS >= 3)
1910 // Note to Timo: You should call MiCheckVirtualAddress and also check if it's zero pte
1911 // also this is missing the page count increment
1912     /* Check if the PPE is valid */
1913     if (PointerPpe->u.Hard.Valid == 0)
1914     {
1915         /* Right now, we only handle scenarios where the PPE is totally empty */
1916         ASSERT(PointerPpe->u.Long == 0);
1917 #if 0
1918         /* Resolve a demand zero fault */
1919         Status = MiResolveDemandZeroFault(PointerPde,
1920                                           MM_READWRITE,
1921                                           CurrentProcess,
1922                                           MM_NOIRQL);
1923 #endif
1924         /* We should come back with a valid PPE */
1925         ASSERT(PointerPpe->u.Hard.Valid == 1);
1926     }
1927 #endif
1928
1929     /* Check if the PDE is valid */
1930     if (PointerPde->u.Hard.Valid == 0)
1931     {
1932         /* Right now, we only handle scenarios where the PDE is totally empty */
1933         ASSERT(PointerPde->u.Long == 0);
1934
1935         /* And go dispatch the fault on the PDE. This should handle the demand-zero */
1936 #if MI_TRACE_PFNS
1937         UserPdeFault = TRUE;
1938 #endif
1939         MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
1940         if (ProtectionCode == MM_NOACCESS)
1941         {
1942 #if (_MI_PAGING_LEVELS == 2)
1943             /* Could be a page table for paged pool */
1944             MiCheckPdeForPagedPool(Address);
1945 #endif
1946             /* Has the code above changed anything -- is this now a valid PTE? */
1947             Status = (PointerPde->u.Hard.Valid == 1) ? STATUS_SUCCESS : STATUS_ACCESS_VIOLATION;
1948
1949             /* Either this was a bogus VA or we've fixed up a paged pool PDE */
1950             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
1951             return Status;
1952         }
1953
1954         /* Write a demand-zero PDE */
1955         MI_WRITE_INVALID_PTE(PointerPde, DemandZeroPde);
1956
1957         /* Dispatch the fault */
1958         Status = MiDispatchFault(TRUE,
1959                                  PointerPte,
1960                                  PointerPde,
1961                                  NULL,
1962                                  FALSE,
1963                                  PsGetCurrentProcess(),
1964                                  TrapInformation,
1965                                  NULL);
1966 #if MI_TRACE_PFNS
1967         UserPdeFault = FALSE;
1968 #endif
1969         /* We should come back with APCs enabled, and with a valid PDE */
1970         ASSERT(KeAreAllApcsDisabled() == TRUE);
1971         ASSERT(PointerPde->u.Hard.Valid == 1);
1972     }
1973     else
1974     {
1975         /* Not yet implemented in ReactOS */
1976         ASSERT(MI_IS_PAGE_LARGE(PointerPde) == FALSE);
1977     }
1978
1979     /* Now capture the PTE. */
1980     TempPte = *PointerPte;
1981
1982     /* Check if the PTE is valid */
1983     if (TempPte.u.Hard.Valid)
1984     {
1985         /* Check if this is a write on a readonly PTE */
1986         if (StoreInstruction)
1987         {
1988             /* Is this a copy on write PTE? */
1989             if (TempPte.u.Hard.CopyOnWrite)
1990             {
1991                 /* Not supported yet */
1992                 ASSERT(FALSE);
1993             }
1994
1995             /* Is this a read-only PTE? */
1996             if (!TempPte.u.Hard.Write)
1997             {
1998                 /* Return the status */
1999                 MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2000                 return STATUS_ACCESS_VIOLATION;
2001             }
2002         }
2003
2004         /* FIXME: Execution is ignored for now, since we don't have no-execute pages yet */
2005
2006         /* The fault has already been resolved by a different thread */
2007         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2008         return STATUS_SUCCESS;
2009     }
2010
2011     /* Quick check for demand-zero */
2012     if (TempPte.u.Long == (MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS))
2013     {
2014         /* Resolve the fault */
2015         MiResolveDemandZeroFault(Address,
2016                                  PointerPte,
2017                                  CurrentProcess,
2018                                  MM_NOIRQL);
2019
2020         /* Return the status */
2021         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2022         return STATUS_PAGE_FAULT_DEMAND_ZERO;
2023     }
2024
2025     /* Check for zero PTE */
2026     if (TempPte.u.Long == 0)
2027     {
2028         /* Check if this address range belongs to a valid allocation (VAD) */
2029         ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
2030         if (ProtectionCode == MM_NOACCESS)
2031         {
2032 #if (_MI_PAGING_LEVELS == 2)
2033             /* Could be a page table for paged pool */
2034             MiCheckPdeForPagedPool(Address);
2035 #endif
2036             /* Has the code above changed anything -- is this now a valid PTE? */
2037             Status = (PointerPte->u.Hard.Valid == 1) ? STATUS_SUCCESS : STATUS_ACCESS_VIOLATION;
2038
2039             /* Either this was a bogus VA or we've fixed up a paged pool PDE */
2040             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2041             return Status;
2042         }
2043
2044         /*
2045          * Check if this is a real user-mode address or actually a kernel-mode
2046          * page table for a user mode address
2047          */
2048         if (Address <= MM_HIGHEST_USER_ADDRESS)
2049         {
2050             /* Add an additional page table reference */
2051             MiIncrementPageTableReferences(Address);
2052         }
2053
2054         /* Is this a guard page? */
2055         if ((ProtectionCode & MM_PROTECT_SPECIAL) == MM_GUARDPAGE)
2056         {
2057             /* The VAD protection cannot be MM_DECOMMIT! */
2058             NT_ASSERT(ProtectionCode != MM_DECOMMIT);
2059
2060             /* Remove the bit */
2061             TempPte.u.Soft.Protection = ProtectionCode & ~MM_GUARDPAGE;
2062             MI_WRITE_INVALID_PTE(PointerPte, TempPte);
2063
2064             /* Not supported */
2065             ASSERT(ProtoPte == NULL);
2066             ASSERT(CurrentThread->ApcNeeded == 0);
2067
2068             /* Drop the working set lock */
2069             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2070             ASSERT(KeGetCurrentIrql() == OldIrql);
2071
2072             /* Handle stack expansion */
2073             return MiCheckForUserStackOverflow(Address, TrapInformation);
2074         }
2075
2076         /* Did we get a prototype PTE back? */
2077         if (!ProtoPte)
2078         {
2079             /* Is this PTE actually part of the PDE-PTE self-mapping directory? */
2080             if (PointerPde == MiAddressToPde(PTE_BASE))
2081             {
2082                 /* Then it's really a demand-zero PDE (on behalf of user-mode) */
2083                 MI_WRITE_INVALID_PTE(PointerPte, DemandZeroPde);
2084             }
2085             else
2086             {
2087                 /* No, create a new PTE. First, write the protection */
2088                 TempPte.u.Soft.Protection = ProtectionCode;
2089                 MI_WRITE_INVALID_PTE(PointerPte, TempPte);
2090             }
2091
2092             /* Lock the PFN database since we're going to grab a page */
2093             OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
2094
2095             /* Make sure we have enough pages */
2096             ASSERT(MmAvailablePages >= 32);
2097
2098             /* Try to get a zero page */
2099             MI_SET_USAGE(MI_USAGE_PEB_TEB);
2100             MI_SET_PROCESS2(CurrentProcess->ImageFileName);
2101             Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
2102             PageFrameIndex = MiRemoveZeroPageSafe(Color);
2103             if (!PageFrameIndex)
2104             {
2105                 /* Grab a page out of there. Later we should grab a colored zero page */
2106                 PageFrameIndex = MiRemoveAnyPage(Color);
2107                 ASSERT(PageFrameIndex);
2108
2109                 /* Release the lock since we need to do some zeroing */
2110                 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
2111
2112                 /* Zero out the page, since it's for user-mode */
2113                 MiZeroPfn(PageFrameIndex);
2114
2115                 /* Grab the lock again so we can initialize the PFN entry */
2116                 OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
2117             }
2118
2119             /* Initialize the PFN entry now */
2120             MiInitializePfn(PageFrameIndex, PointerPte, 1);
2121
2122             /* And we're done with the lock */
2123             KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
2124
2125             /* Increment the count of pages in the process */
2126             CurrentProcess->NumberOfPrivatePages++;
2127
2128             /* One more demand-zero fault */
2129             InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
2130
2131             /* Fault on user PDE, or fault on user PTE? */
2132             if (PointerPte <= MiHighestUserPte)
2133             {
2134                 /* User fault, build a user PTE */
2135                 MI_MAKE_HARDWARE_PTE_USER(&TempPte,
2136                                           PointerPte,
2137                                           PointerPte->u.Soft.Protection,
2138                                           PageFrameIndex);
2139             }
2140             else
2141             {
2142                 /* This is a user-mode PDE, create a kernel PTE for it */
2143                 MI_MAKE_HARDWARE_PTE(&TempPte,
2144                                      PointerPte,
2145                                      PointerPte->u.Soft.Protection,
2146                                      PageFrameIndex);
2147             }
2148
2149             /* Write the dirty bit for writeable pages */
2150             if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
2151
2152             /* And now write down the PTE, making the address valid */
2153             MI_WRITE_VALID_PTE(PointerPte, TempPte);
2154             Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
2155             ASSERT(Pfn1->u1.Event == NULL);
2156
2157             /* Demand zero */
2158             ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
2159             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2160             return STATUS_PAGE_FAULT_DEMAND_ZERO;
2161         }
2162
2163         /* We should have a valid protection here */
2164         ASSERT(ProtectionCode != 0x100);
2165
2166         /* Write the prototype PTE */
2167         TempPte = PrototypePte;
2168         TempPte.u.Soft.Protection = ProtectionCode;
2169         NT_ASSERT(TempPte.u.Long != 0);
2170         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
2171     }
2172     else
2173     {
2174         /* Get the protection code and check if this is a proto PTE */
2175         ProtectionCode = (ULONG)TempPte.u.Soft.Protection;
2176         if (TempPte.u.Soft.Prototype)
2177         {
2178             /* Do we need to go find the real PTE? */
2179             if (TempPte.u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
2180             {
2181                 /* Get the prototype pte and VAD for it */
2182                 ProtoPte = MiCheckVirtualAddress(Address,
2183                                                  &ProtectionCode,
2184                                                  &Vad);
2185                 if (!ProtoPte)
2186                 {
2187                     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
2188                     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2189                     return STATUS_ACCESS_VIOLATION;
2190                 }
2191             }
2192             else
2193             {
2194                 /* Get the prototype PTE! */
2195                 ProtoPte = MiProtoPteToPte(&TempPte);
2196
2197                 /* Is it read-only */
2198                 if (TempPte.u.Proto.ReadOnly)
2199                 {
2200                     /* Set read-only code */
2201                     ProtectionCode = MM_READONLY;
2202                 }
2203                 else
2204                 {
2205                     /* Set unknown protection */
2206                     ProtectionCode = 0x100;
2207                     ASSERT(CurrentProcess->CloneRoot != NULL);
2208                 }
2209             }
2210         }
2211     }
2212
2213     /* Do we have a valid protection code? */
2214     if (ProtectionCode != 0x100)
2215     {
2216         /* Run a software access check first, including to detect guard pages */
2217         Status = MiAccessCheck(PointerPte,
2218                                StoreInstruction,
2219                                Mode,
2220                                ProtectionCode,
2221                                TrapInformation,
2222                                FALSE);
2223         if (Status != STATUS_SUCCESS)
2224         {
2225             /* Not supported */
2226             ASSERT(CurrentThread->ApcNeeded == 0);
2227
2228             /* Drop the working set lock */
2229             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2230             ASSERT(KeGetCurrentIrql() == OldIrql);
2231
2232             /* Did we hit a guard page? */
2233             if (Status == STATUS_GUARD_PAGE_VIOLATION)
2234             {
2235                 /* Handle stack expansion */
2236                 return MiCheckForUserStackOverflow(Address, TrapInformation);
2237             }
2238
2239             /* Otherwise, fail back to the caller directly */
2240             return Status;
2241         }
2242     }
2243
2244     /* Dispatch the fault */
2245     Status = MiDispatchFault(StoreInstruction,
2246                              Address,
2247                              PointerPte,
2248                              ProtoPte,
2249                              FALSE,
2250                              CurrentProcess,
2251                              TrapInformation,
2252                              Vad);
2253
2254     /* Return the status */
2255     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
2256     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2257     return Status;
2258 }
2259
2260 NTSTATUS
2261 NTAPI
2262 MmGetExecuteOptions(IN PULONG ExecuteOptions)
2263 {
2264     PKPROCESS CurrentProcess = &PsGetCurrentProcess()->Pcb;
2265     ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
2266
2267     *ExecuteOptions = 0;
2268
2269     if (CurrentProcess->Flags.ExecuteDisable)
2270     {
2271         *ExecuteOptions |= MEM_EXECUTE_OPTION_DISABLE;
2272     }
2273
2274     if (CurrentProcess->Flags.ExecuteEnable)
2275     {
2276         *ExecuteOptions |= MEM_EXECUTE_OPTION_ENABLE;
2277     }
2278
2279     if (CurrentProcess->Flags.DisableThunkEmulation)
2280     {
2281         *ExecuteOptions |= MEM_EXECUTE_OPTION_DISABLE_THUNK_EMULATION;
2282     }
2283
2284     if (CurrentProcess->Flags.Permanent)
2285     {
2286         *ExecuteOptions |= MEM_EXECUTE_OPTION_PERMANENT;
2287     }
2288
2289     if (CurrentProcess->Flags.ExecuteDispatchEnable)
2290     {
2291         *ExecuteOptions |= MEM_EXECUTE_OPTION_EXECUTE_DISPATCH_ENABLE;
2292     }
2293
2294     if (CurrentProcess->Flags.ImageDispatchEnable)
2295     {
2296         *ExecuteOptions |= MEM_EXECUTE_OPTION_IMAGE_DISPATCH_ENABLE;
2297     }
2298
2299     return STATUS_SUCCESS;
2300 }
2301
2302 NTSTATUS
2303 NTAPI
2304 MmSetExecuteOptions(IN ULONG ExecuteOptions)
2305 {
2306     PKPROCESS CurrentProcess = &PsGetCurrentProcess()->Pcb;
2307     KLOCK_QUEUE_HANDLE ProcessLock;
2308     NTSTATUS Status = STATUS_ACCESS_DENIED;
2309     ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
2310
2311     /* Only accept valid flags */
2312     if (ExecuteOptions & ~MEM_EXECUTE_OPTION_VALID_FLAGS)
2313     {
2314         /* Fail */
2315         DPRINT1("Invalid no-execute options\n");
2316         return STATUS_INVALID_PARAMETER;
2317     }
2318
2319     /* Change the NX state in the process lock */
2320     KiAcquireProcessLock(CurrentProcess, &ProcessLock);
2321
2322     /* Don't change anything if the permanent flag was set */
2323     if (!CurrentProcess->Flags.Permanent)
2324     {
2325         /* Start by assuming it's not disabled */
2326         CurrentProcess->Flags.ExecuteDisable = FALSE;
2327
2328         /* Now process each flag and turn the equivalent bit on */
2329         if (ExecuteOptions & MEM_EXECUTE_OPTION_DISABLE)
2330         {
2331             CurrentProcess->Flags.ExecuteDisable = TRUE;
2332         }
2333         if (ExecuteOptions & MEM_EXECUTE_OPTION_ENABLE)
2334         {
2335             CurrentProcess->Flags.ExecuteEnable = TRUE;
2336         }
2337         if (ExecuteOptions & MEM_EXECUTE_OPTION_DISABLE_THUNK_EMULATION)
2338         {
2339             CurrentProcess->Flags.DisableThunkEmulation = TRUE;
2340         }
2341         if (ExecuteOptions & MEM_EXECUTE_OPTION_PERMANENT)
2342         {
2343             CurrentProcess->Flags.Permanent = TRUE;
2344         }
2345         if (ExecuteOptions & MEM_EXECUTE_OPTION_EXECUTE_DISPATCH_ENABLE)
2346         {
2347             CurrentProcess->Flags.ExecuteDispatchEnable = TRUE;
2348         }
2349         if (ExecuteOptions & MEM_EXECUTE_OPTION_IMAGE_DISPATCH_ENABLE)
2350         {
2351             CurrentProcess->Flags.ImageDispatchEnable = TRUE;
2352         }
2353
2354         /* These are turned on by default if no-execution is also eanbled */
2355         if (CurrentProcess->Flags.ExecuteEnable)
2356         {
2357             CurrentProcess->Flags.ExecuteDispatchEnable = TRUE;
2358             CurrentProcess->Flags.ImageDispatchEnable = TRUE;
2359         }
2360
2361         /* All good */
2362         Status = STATUS_SUCCESS;
2363     }
2364
2365     /* Release the lock and return status */
2366     KiReleaseProcessLock(&ProcessLock);
2367     return Status;
2368 }
2369
2370 /* EOF */