ntoskrnl/mm/ARM3/pagfault.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pagfault.c
   5  * PURPOSE:         ARM Memory Manager Page Fault Handling
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include <mm/ARM3/miarm.h>
  17
  18 VOID
  19 NTAPI
  20 MmRebalanceMemoryConsumersAndWait(VOID);
  21
  22 /* GLOBALS ********************************************************************/
  23
  24 #define HYDRA_PROCESS (PEPROCESS)1
  25 #if MI_TRACE_PFNS
  26 BOOLEAN UserPdeFault = FALSE;
  27 #endif
  28
  29 /* PRIVATE FUNCTIONS **********************************************************/
  30
  31 static
  32 NTSTATUS
  33 NTAPI
  34 MiCheckForUserStackOverflow(IN PVOID Address,
  35                             IN PVOID TrapInformation)
  36 {
  37     PETHREAD CurrentThread = PsGetCurrentThread();
  38     PTEB Teb = CurrentThread->Tcb.Teb;
  39     PVOID StackBase, DeallocationStack, NextStackAddress;
  40     SIZE_T GuaranteedSize;
  41     NTSTATUS Status;
  42
  43     /* Do we own the address space lock? */
  44     if (CurrentThread->AddressSpaceOwner == 1)
  45     {
  46         /* This isn't valid */
  47         DPRINT1("Process owns address space lock\n");
  48         ASSERT(KeAreAllApcsDisabled() == TRUE);
  49         return STATUS_GUARD_PAGE_VIOLATION;
  50     }
  51
  52     /* Are we attached? */
  53     if (KeIsAttachedProcess())
  54     {
  55         /* This isn't valid */
  56         DPRINT1("Process is attached\n");
  57         return STATUS_GUARD_PAGE_VIOLATION;
  58     }
  59
  60     /* Read the current settings */
  61     StackBase = Teb->NtTib.StackBase;
  62     DeallocationStack = Teb->DeallocationStack;
  63     GuaranteedSize = Teb->GuaranteedStackBytes;
  64     DPRINT("Handling guard page fault with Stacks Addresses 0x%p and 0x%p, guarantee: %lx\n",
  65             StackBase, DeallocationStack, GuaranteedSize);
  66
  67     /* Guarantees make this code harder, for now, assume there aren't any */
  68     ASSERT(GuaranteedSize == 0);
  69
  70     /* So allocate only the minimum guard page size */
  71     GuaranteedSize = PAGE_SIZE;
  72
  73     /* Does this faulting stack address actually exist in the stack? */
  74     if ((Address >= StackBase) || (Address < DeallocationStack))
  75     {
  76         /* That's odd... */
  77         DPRINT1("Faulting address outside of stack bounds. Address=%p, StackBase=%p, DeallocationStack=%p\n",
  78                 Address, StackBase, DeallocationStack);
  79         return STATUS_GUARD_PAGE_VIOLATION;
  80     }
  81
  82     /* This is where the stack will start now */
  83     NextStackAddress = (PVOID)((ULONG_PTR)PAGE_ALIGN(Address) - GuaranteedSize);
  84
  85     /* Do we have at least one page between here and the end of the stack? */
  86     if (((ULONG_PTR)NextStackAddress - PAGE_SIZE) <= (ULONG_PTR)DeallocationStack)
  87     {
  88         /* We don't -- Trying to make this guard page valid now */
  89         DPRINT1("Close to our death...\n");
  90
  91         /* Calculate the next memory address */
  92         NextStackAddress = (PVOID)((ULONG_PTR)PAGE_ALIGN(DeallocationStack) + GuaranteedSize);
  93
  94         /* Allocate the memory */
  95         Status = ZwAllocateVirtualMemory(NtCurrentProcess(),
  96                                          &NextStackAddress,
  97                                          0,
  98                                          &GuaranteedSize,
  99                                          MEM_COMMIT,
 100                                          PAGE_READWRITE);
 101         if (NT_SUCCESS(Status))
 102         {
 103             /* Success! */
 104             Teb->NtTib.StackLimit = NextStackAddress;
 105         }
 106         else
 107         {
 108             DPRINT1("Failed to allocate memory\n");
 109         }
 110
 111         return STATUS_STACK_OVERFLOW;
 112     }
 113
 114     /* Don't handle this flag yet */
 115     ASSERT((PsGetCurrentProcess()->Peb->NtGlobalFlag & FLG_DISABLE_STACK_EXTENSION) == 0);
 116
 117     /* Update the stack limit */
 118     Teb->NtTib.StackLimit = (PVOID)((ULONG_PTR)NextStackAddress + GuaranteedSize);
 119
 120     /* Now move the guard page to the next page */
 121     Status = ZwAllocateVirtualMemory(NtCurrentProcess(),
 122                                      &NextStackAddress,
 123                                      0,
 124                                      &GuaranteedSize,
 125                                      MEM_COMMIT,
 126                                      PAGE_READWRITE | PAGE_GUARD);
 127     if ((NT_SUCCESS(Status) || (Status == STATUS_ALREADY_COMMITTED)))
 128     {
 129         /* We did it! */
 130         DPRINT("Guard page handled successfully for %p\n", Address);
 131         return STATUS_PAGE_FAULT_GUARD_PAGE;
 132     }
 133
 134     /* Fail, we couldn't move the guard page */
 135     DPRINT1("Guard page failure: %lx\n", Status);
 136     ASSERT(FALSE);
 137     return STATUS_STACK_OVERFLOW;
 138 }
 139
 140 FORCEINLINE
 141 BOOLEAN
 142 MiIsAccessAllowed(
 143     _In_ ULONG ProtectionMask,
 144     _In_ BOOLEAN Write,
 145     _In_ BOOLEAN Execute)
 146 {
 147     #define _BYTE_MASK(Bit0, Bit1, Bit2, Bit3, Bit4, Bit5, Bit6, Bit7) \
 148         (Bit0) | ((Bit1) << 1) | ((Bit2) << 2) | ((Bit3) << 3) | \
 149         ((Bit4) << 4) | ((Bit5) << 5) | ((Bit6) << 6) | ((Bit7) << 7)
 150     static const UCHAR AccessAllowedMask[2][2] =
 151     {
 152         {   // Protect 0  1  2  3  4  5  6  7
 153             _BYTE_MASK(0, 1, 1, 1, 1, 1, 1, 1), // READ
 154             _BYTE_MASK(0, 0, 1, 1, 0, 0, 1, 1), // EXECUTE READ
 155         },
 156         {
 157             _BYTE_MASK(0, 0, 0, 0, 1, 1, 1, 1), // WRITE
 158             _BYTE_MASK(0, 0, 0, 0, 0, 0, 1, 1), // EXECUTE WRITE
 159         }
 160     };
 161
 162     /* We want only the lower access bits */
 163     ProtectionMask &= MM_PROTECT_ACCESS;
 164
 165     /* Look it up in the table */
 166     return (AccessAllowedMask[Write != 0][Execute != 0] >> ProtectionMask) & 1;
 167 }
 168
 169 static
 170 NTSTATUS
 171 NTAPI
 172 MiAccessCheck(IN PMMPTE PointerPte,
 173               IN BOOLEAN StoreInstruction,
 174               IN KPROCESSOR_MODE PreviousMode,
 175               IN ULONG_PTR ProtectionMask,
 176               IN PVOID TrapFrame,
 177               IN BOOLEAN LockHeld)
 178 {
 179     MMPTE TempPte;
 180
 181     /* Check for invalid user-mode access */
 182     if ((PreviousMode == UserMode) && (PointerPte > MiHighestUserPte))
 183     {
 184         return STATUS_ACCESS_VIOLATION;
 185     }
 186
 187     /* Capture the PTE -- is it valid? */
 188     TempPte = *PointerPte;
 189     if (TempPte.u.Hard.Valid)
 190     {
 191         /* Was someone trying to write to it? */
 192         if (StoreInstruction)
 193         {
 194             /* Is it writable?*/
 195             if (MI_IS_PAGE_WRITEABLE(&TempPte) ||
 196                 MI_IS_PAGE_COPY_ON_WRITE(&TempPte))
 197             {
 198                 /* Then there's nothing to worry about */
 199                 return STATUS_SUCCESS;
 200             }
 201
 202             /* Oops! This isn't allowed */
 203             return STATUS_ACCESS_VIOLATION;
 204         }
 205
 206         /* Someone was trying to read from a valid PTE, that's fine too */
 207         return STATUS_SUCCESS;
 208     }
 209
 210     /* Check if the protection on the page allows what is being attempted */
 211     if (!MiIsAccessAllowed(ProtectionMask, StoreInstruction, FALSE))
 212     {
 213         return STATUS_ACCESS_VIOLATION;
 214     }
 215
 216     /* Check if this is a guard page */
 217     if ((ProtectionMask & MM_PROTECT_SPECIAL) == MM_GUARDPAGE)
 218     {
 219         ASSERT(ProtectionMask != MM_DECOMMIT);
 220
 221         /* Attached processes can't expand their stack */
 222         if (KeIsAttachedProcess()) return STATUS_ACCESS_VIOLATION;
 223
 224         /* No support for prototype PTEs yet */
 225         ASSERT(TempPte.u.Soft.Prototype == 0);
 226
 227         /* Remove the guard page bit, and return a guard page violation */
 228         TempPte.u.Soft.Protection = ProtectionMask & ~MM_GUARDPAGE;
 229         ASSERT(TempPte.u.Long != 0);
 230         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 231         return STATUS_GUARD_PAGE_VIOLATION;
 232     }
 233
 234     /* Nothing to do */
 235     return STATUS_SUCCESS;
 236 }
 237
 238 static
 239 PMMPTE
 240 NTAPI
 241 MiCheckVirtualAddress(IN PVOID VirtualAddress,
 242                       OUT PULONG ProtectCode,
 243                       OUT PMMVAD *ProtoVad)
 244 {
 245     PMMVAD Vad;
 246     PMMPTE PointerPte;
 247
 248     /* No prototype/section support for now */
 249     *ProtoVad = NULL;
 250
 251     /* User or kernel fault? */
 252     if (VirtualAddress <= MM_HIGHEST_USER_ADDRESS)
 253     {
 254         /* Special case for shared data */
 255         if (PAGE_ALIGN(VirtualAddress) == (PVOID)MM_SHARED_USER_DATA_VA)
 256         {
 257             /* It's a read-only page */
 258             *ProtectCode = MM_READONLY;
 259             return MmSharedUserDataPte;
 260         }
 261
 262         /* Find the VAD, it might not exist if the address is bogus */
 263         Vad = MiLocateAddress(VirtualAddress);
 264         if (!Vad)
 265         {
 266             /* Bogus virtual address */
 267             *ProtectCode = MM_NOACCESS;
 268             return NULL;
 269         }
 270
 271         /* ReactOS does not handle physical memory VADs yet */
 272         ASSERT(Vad->u.VadFlags.VadType != VadDevicePhysicalMemory);
 273
 274         /* Check if it's a section, or just an allocation */
 275         if (Vad->u.VadFlags.PrivateMemory)
 276         {
 277             /* ReactOS does not handle AWE VADs yet */
 278             ASSERT(Vad->u.VadFlags.VadType != VadAwe);
 279
 280             /* This must be a TEB/PEB VAD */
 281             if (Vad->u.VadFlags.MemCommit)
 282             {
 283                 /* It's committed, so return the VAD protection */
 284                 *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
 285             }
 286             else
 287             {
 288                 /* It has not yet been committed, so return no access */
 289                 *ProtectCode = MM_NOACCESS;
 290             }
 291
 292             /* In both cases, return no PTE */
 293             return NULL;
 294         }
 295         else
 296         {
 297             /* ReactOS does not supoprt these VADs yet */
 298             ASSERT(Vad->u.VadFlags.VadType != VadImageMap);
 299             ASSERT(Vad->u2.VadFlags2.ExtendableFile == 0);
 300
 301             /* Return the proto VAD */
 302             *ProtoVad = Vad;
 303
 304             /* Get the prototype PTE for this page */
 305             PointerPte = (((ULONG_PTR)VirtualAddress >> PAGE_SHIFT) - Vad->StartingVpn) + Vad->FirstPrototypePte;
 306             ASSERT(PointerPte != NULL);
 307             ASSERT(PointerPte <= Vad->LastContiguousPte);
 308
 309             /* Return the Prototype PTE and the protection for the page mapping */
 310             *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
 311             return PointerPte;
 312         }
 313     }
 314     else if (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress))
 315     {
 316         /* This should never happen, as these addresses are handled by the double-maping */
 317         if (((PMMPTE)VirtualAddress >= MiAddressToPte(MmPagedPoolStart)) &&
 318             ((PMMPTE)VirtualAddress <= MmPagedPoolInfo.LastPteForPagedPool))
 319         {
 320             /* Fail such access */
 321             *ProtectCode = MM_NOACCESS;
 322             return NULL;
 323         }
 324
 325         /* Return full access rights */
 326         *ProtectCode = MM_EXECUTE_READWRITE;
 327         return NULL;
 328     }
 329     else if (MI_IS_SESSION_ADDRESS(VirtualAddress))
 330     {
 331         /* ReactOS does not have an image list yet, so bail out to failure case */
 332         ASSERT(IsListEmpty(&MmSessionSpace->ImageList));
 333     }
 334
 335     /* Default case -- failure */
 336     *ProtectCode = MM_NOACCESS;
 337     return NULL;
 338 }
 339
 340 #if (_MI_PAGING_LEVELS == 2)
 341 static
 342 NTSTATUS
 343 FASTCALL
 344 MiCheckPdeForSessionSpace(IN PVOID Address)
 345 {
 346     MMPTE TempPde;
 347     PMMPDE PointerPde;
 348     PVOID SessionAddress;
 349     ULONG Index;
 350
 351     /* Is this a session PTE? */
 352     if (MI_IS_SESSION_PTE(Address))
 353     {
 354         /* Make sure the PDE for session space is valid */
 355         PointerPde = MiAddressToPde(MmSessionSpace);
 356         if (!PointerPde->u.Hard.Valid)
 357         {
 358             /* This means there's no valid session, bail out */
 359             DbgPrint("MiCheckPdeForSessionSpace: No current session for PTE %p\n",
 360                      Address);
 361             DbgBreakPoint();
 362             return STATUS_ACCESS_VIOLATION;
 363         }
 364
 365         /* Now get the session-specific page table for this address */
 366         SessionAddress = MiPteToAddress(Address);
 367         PointerPde = MiAddressToPte(Address);
 368         if (PointerPde->u.Hard.Valid) return STATUS_WAIT_1;
 369
 370         /* It's not valid, so find it in the page table array */
 371         Index = ((ULONG_PTR)SessionAddress - (ULONG_PTR)MmSessionBase) >> 22;
 372         TempPde.u.Long = MmSessionSpace->PageTables[Index].u.Long;
 373         if (TempPde.u.Hard.Valid)
 374         {
 375             /* The copy is valid, so swap it in */
 376             InterlockedExchange((PLONG)PointerPde, TempPde.u.Long);
 377             return STATUS_WAIT_1;
 378         }
 379
 380         /* We don't seem to have allocated a page table for this address yet? */
 381         DbgPrint("MiCheckPdeForSessionSpace: No Session PDE for PTE %p, %p\n",
 382                  PointerPde->u.Long, SessionAddress);
 383         DbgBreakPoint();
 384         return STATUS_ACCESS_VIOLATION;
 385     }
 386
 387     /* Is the address also a session address? If not, we're done */
 388     if (!MI_IS_SESSION_ADDRESS(Address)) return STATUS_SUCCESS;
 389
 390     /* It is, so again get the PDE for session space */
 391     PointerPde = MiAddressToPde(MmSessionSpace);
 392     if (!PointerPde->u.Hard.Valid)
 393     {
 394         /* This means there's no valid session, bail out */
 395         DbgPrint("MiCheckPdeForSessionSpace: No current session for VA %p\n",
 396                     Address);
 397         DbgBreakPoint();
 398         return STATUS_ACCESS_VIOLATION;
 399     }
 400
 401     /* Now get the PDE for the address itself */
 402     PointerPde = MiAddressToPde(Address);
 403     if (!PointerPde->u.Hard.Valid)
 404     {
 405         /* Do the swap, we should be good to go */
 406         Index = ((ULONG_PTR)Address - (ULONG_PTR)MmSessionBase) >> 22;
 407         PointerPde->u.Long = MmSessionSpace->PageTables[Index].u.Long;
 408         if (PointerPde->u.Hard.Valid) return STATUS_WAIT_1;
 409
 410         /* We had not allocated a page table for this session address yet, fail! */
 411         DbgPrint("MiCheckPdeForSessionSpace: No Session PDE for VA %p, %p\n",
 412                  PointerPde->u.Long, Address);
 413         DbgBreakPoint();
 414         return STATUS_ACCESS_VIOLATION;
 415     }
 416
 417     /* It's valid, so there's nothing to do */
 418     return STATUS_SUCCESS;
 419 }
 420
 421 NTSTATUS
 422 FASTCALL
 423 MiCheckPdeForPagedPool(IN PVOID Address)
 424 {
 425     PMMPDE PointerPde;
 426     NTSTATUS Status = STATUS_SUCCESS;
 427
 428     /* Check session PDE */
 429     if (MI_IS_SESSION_ADDRESS(Address)) return MiCheckPdeForSessionSpace(Address);
 430     if (MI_IS_SESSION_PTE(Address)) return MiCheckPdeForSessionSpace(Address);
 431
 432     //
 433     // Check if this is a fault while trying to access the page table itself
 434     //
 435     if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
 436     {
 437         //
 438         // Send a hint to the page fault handler that this is only a valid fault
 439         // if we already detected this was access within the page table range
 440         //
 441         PointerPde = (PMMPDE)MiAddressToPte(Address);
 442         Status = STATUS_WAIT_1;
 443     }
 444     else if (Address < MmSystemRangeStart)
 445     {
 446         //
 447         // This is totally illegal
 448         //
 449         return STATUS_ACCESS_VIOLATION;
 450     }
 451     else
 452     {
 453         //
 454         // Get the PDE for the address
 455         //
 456         PointerPde = MiAddressToPde(Address);
 457     }
 458
 459     //
 460     // Check if it's not valid
 461     //
 462     if (PointerPde->u.Hard.Valid == 0)
 463     {
 464         //
 465         // Copy it from our double-mapped system page directory
 466         //
 467         InterlockedExchangePte(PointerPde,
 468                                MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)].u.Long);
 469     }
 470
 471     //
 472     // Return status
 473     //
 474     return Status;
 475 }
 476 #else
 477 NTSTATUS
 478 FASTCALL
 479 MiCheckPdeForPagedPool(IN PVOID Address)
 480 {
 481     return STATUS_ACCESS_VIOLATION;
 482 }
 483 #endif
 484
 485 VOID
 486 NTAPI
 487 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
 488 {
 489     PMMPTE ZeroPte;
 490     MMPTE TempPte;
 491     PMMPFN Pfn1;
 492     PVOID ZeroAddress;
 493
 494     /* Get the PFN for this page */
 495     Pfn1 = MiGetPfnEntry(PageFrameNumber);
 496     ASSERT(Pfn1);
 497
 498     /* Grab a system PTE we can use to zero the page */
 499     ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
 500     ASSERT(ZeroPte);
 501
 502     /* Initialize the PTE for it */
 503     TempPte = ValidKernelPte;
 504     TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
 505
 506     /* Setup caching */
 507     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 508     {
 509         /* Write combining, no caching */
 510         MI_PAGE_DISABLE_CACHE(&TempPte);
 511         MI_PAGE_WRITE_COMBINED(&TempPte);
 512     }
 513     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 514     {
 515         /* Write through, no caching */
 516         MI_PAGE_DISABLE_CACHE(&TempPte);
 517         MI_PAGE_WRITE_THROUGH(&TempPte);
 518     }
 519
 520     /* Make the system PTE valid with our PFN */
 521     MI_WRITE_VALID_PTE(ZeroPte, TempPte);
 522
 523     /* Get the address it maps to, and zero it out */
 524     ZeroAddress = MiPteToAddress(ZeroPte);
 525     KeZeroPages(ZeroAddress, PAGE_SIZE);
 526
 527     /* Now get rid of it */
 528     MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
 529 }
 530
 531 VOID
 532 NTAPI
 533 MiCopyPfn(
 534     _In_ PFN_NUMBER DestPage,
 535     _In_ PFN_NUMBER SrcPage)
 536 {
 537     PMMPTE SysPtes;
 538     MMPTE TempPte;
 539     PMMPFN DestPfn, SrcPfn;
 540     PVOID DestAddress;
 541     const VOID* SrcAddress;
 542
 543     /* Get the PFNs */
 544     DestPfn = MiGetPfnEntry(DestPage);
 545     ASSERT(DestPfn);
 546     SrcPfn = MiGetPfnEntry(SrcPage);
 547     ASSERT(SrcPfn);
 548
 549     /* Grab 2 system PTEs */
 550     SysPtes = MiReserveSystemPtes(2, SystemPteSpace);
 551     ASSERT(SysPtes);
 552
 553     /* Initialize the destination PTE */
 554     TempPte = ValidKernelPte;
 555     TempPte.u.Hard.PageFrameNumber = DestPage;
 556
 557     /* Setup caching */
 558     if (DestPfn->u3.e1.CacheAttribute == MiWriteCombined)
 559     {
 560         /* Write combining, no caching */
 561         MI_PAGE_DISABLE_CACHE(&TempPte);
 562         MI_PAGE_WRITE_COMBINED(&TempPte);
 563     }
 564     else if (DestPfn->u3.e1.CacheAttribute == MiNonCached)
 565     {
 566         /* Write through, no caching */
 567         MI_PAGE_DISABLE_CACHE(&TempPte);
 568         MI_PAGE_WRITE_THROUGH(&TempPte);
 569     }
 570
 571     /* Make the system PTE valid with our PFN */
 572     MI_WRITE_VALID_PTE(&SysPtes[0], TempPte);
 573
 574     /* Initialize the source PTE */
 575     TempPte = ValidKernelPte;
 576     TempPte.u.Hard.PageFrameNumber = SrcPage;
 577
 578     /* Setup caching */
 579     if (SrcPfn->u3.e1.CacheAttribute == MiNonCached)
 580     {
 581         MI_PAGE_DISABLE_CACHE(&TempPte);
 582     }
 583
 584     /* Make the system PTE valid with our PFN */
 585     MI_WRITE_VALID_PTE(&SysPtes[1], TempPte);
 586
 587     /* Get the addresses and perform the copy */
 588     DestAddress = MiPteToAddress(&SysPtes[0]);
 589     SrcAddress = MiPteToAddress(&SysPtes[1]);
 590     RtlCopyMemory(DestAddress, SrcAddress, PAGE_SIZE);
 591
 592     /* Now get rid of it */
 593     MiReleaseSystemPtes(SysPtes, 2, SystemPteSpace);
 594 }
 595
 596 static
 597 NTSTATUS
 598 NTAPI
 599 MiResolveDemandZeroFault(IN PVOID Address,
 600                          IN PMMPTE PointerPte,
 601                          IN ULONG Protection,
 602                          IN PEPROCESS Process,
 603                          IN KIRQL OldIrql)
 604 {
 605     PFN_NUMBER PageFrameNumber = 0;
 606     MMPTE TempPte;
 607     BOOLEAN NeedZero = FALSE, HaveLock = FALSE;
 608     ULONG Color;
 609     PMMPFN Pfn1;
 610     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
 611             Address,
 612             Process);
 613
 614     /* Must currently only be called by paging path */
 615     if ((Process > HYDRA_PROCESS) && (OldIrql == MM_NOIRQL))
 616     {
 617         /* Sanity check */
 618         ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
 619
 620         /* No forking yet */
 621         ASSERT(Process->ForkInProgress == NULL);
 622
 623         /* Get process color */
 624         Color = MI_GET_NEXT_PROCESS_COLOR(Process);
 625         ASSERT(Color != 0xFFFFFFFF);
 626
 627         /* We'll need a zero page */
 628         NeedZero = TRUE;
 629     }
 630     else
 631     {
 632         /* Check if we need a zero page */
 633         NeedZero = (OldIrql != MM_NOIRQL);
 634
 635         /* Session-backed image views must be zeroed */
 636         if ((Process == HYDRA_PROCESS) &&
 637             ((MI_IS_SESSION_IMAGE_ADDRESS(Address)) ||
 638              ((Address >= MiSessionViewStart) && (Address < MiSessionSpaceWs))))
 639         {
 640             NeedZero = TRUE;
 641         }
 642
 643         /* Hardcode unknown color */
 644         Color = 0xFFFFFFFF;
 645     }
 646
 647     /* Check if the PFN database should be acquired */
 648     if (OldIrql == MM_NOIRQL)
 649     {
 650         /* Acquire it and remember we should release it after */
 651         OldIrql = MiAcquirePfnLock();
 652         HaveLock = TRUE;
 653     }
 654
 655     /* We either manually locked the PFN DB, or already came with it locked */
 656     MI_ASSERT_PFN_LOCK_HELD();
 657     ASSERT(PointerPte->u.Hard.Valid == 0);
 658
 659     /* Assert we have enough pages */
 660     //ASSERT(MmAvailablePages >= 32);
 661
 662 #if MI_TRACE_PFNS
 663     if (UserPdeFault) MI_SET_USAGE(MI_USAGE_PAGE_TABLE);
 664     if (!UserPdeFault) MI_SET_USAGE(MI_USAGE_DEMAND_ZERO);
 665 #endif
 666     if (Process == HYDRA_PROCESS) MI_SET_PROCESS2("Hydra");
 667     else if (Process) MI_SET_PROCESS2(Process->ImageFileName);
 668     else MI_SET_PROCESS2("Kernel Demand 0");
 669
 670     /* Do we need a zero page? */
 671     if (Color != 0xFFFFFFFF)
 672     {
 673         /* Try to get one, if we couldn't grab a free page and zero it */
 674         PageFrameNumber = MiRemoveZeroPageSafe(Color);
 675         if (!PageFrameNumber)
 676         {
 677             /* We'll need a free page and zero it manually */
 678             PageFrameNumber = MiRemoveAnyPage(Color);
 679             NeedZero = TRUE;
 680         }
 681         else
 682         {
 683             /* Page guaranteed to be zero-filled */
 684             NeedZero = FALSE;
 685         }
 686     }
 687     else
 688     {
 689         /* Get a color, and see if we should grab a zero or non-zero page */
 690         Color = MI_GET_NEXT_COLOR();
 691         if (!NeedZero)
 692         {
 693             /* Process or system doesn't want a zero page, grab anything */
 694             PageFrameNumber = MiRemoveAnyPage(Color);
 695         }
 696         else
 697         {
 698             /* System wants a zero page, obtain one */
 699             PageFrameNumber = MiRemoveZeroPage(Color);
 700             /* No need to zero-fill it */
 701             NeedZero = FALSE;
 702         }
 703     }
 704
 705     if (PageFrameNumber == 0)
 706     {
 707         MiReleasePfnLock(OldIrql);
 708         return STATUS_NO_MEMORY;
 709     }
 710
 711     /* Initialize it */
 712     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
 713
 714     /* Increment demand zero faults */
 715     KeGetCurrentPrcb()->MmDemandZeroCount++;
 716
 717     /* Do we have the lock? */
 718     if (HaveLock)
 719     {
 720         /* Release it */
 721         MiReleasePfnLock(OldIrql);
 722
 723         /* Update performance counters */
 724         if (Process > HYDRA_PROCESS) Process->NumberOfPrivatePages++;
 725     }
 726
 727     /* Zero the page if need be */
 728     if (NeedZero) MiZeroPfn(PageFrameNumber);
 729
 730     /* Fault on user PDE, or fault on user PTE? */
 731     if (PointerPte <= MiHighestUserPte)
 732     {
 733         /* User fault, build a user PTE */
 734         MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 735                                   PointerPte,
 736                                   Protection,
 737                                   PageFrameNumber);
 738     }
 739     else
 740     {
 741         /* This is a user-mode PDE, create a kernel PTE for it */
 742         MI_MAKE_HARDWARE_PTE(&TempPte,
 743                              PointerPte,
 744                              Protection,
 745                              PageFrameNumber);
 746     }
 747
 748     /* Set it dirty if it's a writable page */
 749     if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
 750
 751     /* Write it */
 752     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 753
 754     /* Did we manually acquire the lock */
 755     if (HaveLock)
 756     {
 757         /* Get the PFN entry */
 758         Pfn1 = MI_PFN_ELEMENT(PageFrameNumber);
 759
 760         /* Windows does these sanity checks */
 761         ASSERT(Pfn1->u1.Event == 0);
 762         ASSERT(Pfn1->u3.e1.PrototypePte == 0);
 763     }
 764
 765     //
 766     // It's all good now
 767     //
 768     DPRINT("Demand zero page has now been paged in\n");
 769     return STATUS_PAGE_FAULT_DEMAND_ZERO;
 770 }
 771
 772 static
 773 NTSTATUS
 774 NTAPI
 775 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
 776                         IN PVOID Address,
 777                         IN PMMPTE PointerPte,
 778                         IN PMMPTE PointerProtoPte,
 779                         IN KIRQL OldIrql,
 780                         IN PMMPFN* LockedProtoPfn)
 781 {
 782     MMPTE TempPte;
 783     PMMPTE OriginalPte, PageTablePte;
 784     ULONG_PTR Protection;
 785     PFN_NUMBER PageFrameIndex;
 786     PMMPFN Pfn1, Pfn2;
 787     BOOLEAN OriginalProtection, DirtyPage;
 788
 789     /* Must be called with an valid prototype PTE, with the PFN lock held */
 790     MI_ASSERT_PFN_LOCK_HELD();
 791     ASSERT(PointerProtoPte->u.Hard.Valid == 1);
 792
 793     /* Get the page */
 794     PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
 795
 796     /* Get the PFN entry and set it as a prototype PTE */
 797     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 798     Pfn1->u3.e1.PrototypePte = 1;
 799
 800     /* Increment the share count for the page table */
 801     PageTablePte = MiAddressToPte(PointerPte);
 802     Pfn2 = MiGetPfnEntry(PageTablePte->u.Hard.PageFrameNumber);
 803     Pfn2->u2.ShareCount++;
 804
 805     /* Check where we should be getting the protection information from */
 806     if (PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
 807     {
 808         /* Get the protection from the PTE, there's no real Proto PTE data */
 809         Protection = PointerPte->u.Soft.Protection;
 810
 811         /* Remember that we did not use the proto protection */
 812         OriginalProtection = FALSE;
 813     }
 814     else
 815     {
 816         /* Get the protection from the original PTE link */
 817         OriginalPte = &Pfn1->OriginalPte;
 818         Protection = OriginalPte->u.Soft.Protection;
 819
 820         /* Remember that we used the original protection */
 821         OriginalProtection = TRUE;
 822
 823         /* Check if this was a write on a read only proto */
 824         if ((StoreInstruction) && !(Protection & MM_READWRITE))
 825         {
 826             /* Clear the flag */
 827             StoreInstruction = 0;
 828         }
 829     }
 830
 831     /* Check if this was a write on a non-COW page */
 832     DirtyPage = FALSE;
 833     if ((StoreInstruction) && ((Protection & MM_WRITECOPY) != MM_WRITECOPY))
 834     {
 835         /* Then the page should be marked dirty */
 836         DirtyPage = TRUE;
 837
 838         /* ReactOS check */
 839         ASSERT(Pfn1->OriginalPte.u.Soft.Prototype != 0);
 840     }
 841
 842     /* Did we get a locked incoming PFN? */
 843     if (*LockedProtoPfn)
 844     {
 845         /* Drop a reference */
 846         ASSERT((*LockedProtoPfn)->u3.e2.ReferenceCount >= 1);
 847         MiDereferencePfnAndDropLockCount(*LockedProtoPfn);
 848         *LockedProtoPfn = NULL;
 849     }
 850
 851     /* Release the PFN lock */
 852     MiReleasePfnLock(OldIrql);
 853
 854     /* Remove special/caching bits */
 855     Protection &= ~MM_PROTECT_SPECIAL;
 856
 857     /* Setup caching */
 858     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 859     {
 860         /* Write combining, no caching */
 861         MI_PAGE_DISABLE_CACHE(&TempPte);
 862         MI_PAGE_WRITE_COMBINED(&TempPte);
 863     }
 864     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 865     {
 866         /* Write through, no caching */
 867         MI_PAGE_DISABLE_CACHE(&TempPte);
 868         MI_PAGE_WRITE_THROUGH(&TempPte);
 869     }
 870
 871     /* Check if this is a kernel or user address */
 872     if (Address < MmSystemRangeStart)
 873     {
 874         /* Build the user PTE */
 875         MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, Protection, PageFrameIndex);
 876     }
 877     else
 878     {
 879         /* Build the kernel PTE */
 880         MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, Protection, PageFrameIndex);
 881     }
 882
 883     /* Set the dirty flag if needed */
 884     if (DirtyPage) MI_MAKE_DIRTY_PAGE(&TempPte);
 885
 886     /* Write the PTE */
 887     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 888
 889     /* Reset the protection if needed */
 890     if (OriginalProtection) Protection = MM_ZERO_ACCESS;
 891
 892     /* Return success */
 893     ASSERT(PointerPte == MiAddressToPte(Address));
 894     return STATUS_SUCCESS;
 895 }
 896
 897 static
 898 NTSTATUS
 899 NTAPI
 900 MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction,
 901                        _In_ PVOID FaultingAddress,
 902                        _In_ PMMPTE PointerPte,
 903                        _In_ PEPROCESS CurrentProcess,
 904                        _Inout_ KIRQL *OldIrql)
 905 {
 906     ULONG Color;
 907     PFN_NUMBER Page;
 908     NTSTATUS Status;
 909     MMPTE TempPte = *PointerPte;
 910     PMMPFN Pfn1;
 911     ULONG PageFileIndex = TempPte.u.Soft.PageFileLow;
 912     ULONG_PTR PageFileOffset = TempPte.u.Soft.PageFileHigh;
 913     ULONG Protection = TempPte.u.Soft.Protection;
 914
 915     /* Things we don't support yet */
 916     ASSERT(CurrentProcess > HYDRA_PROCESS);
 917     ASSERT(*OldIrql != MM_NOIRQL);
 918
 919     MI_SET_USAGE(MI_USAGE_PAGE_FILE);
 920     MI_SET_PROCESS(CurrentProcess);
 921
 922     /* We must hold the PFN lock */
 923     MI_ASSERT_PFN_LOCK_HELD();
 924
 925     /* Some sanity checks */
 926     ASSERT(TempPte.u.Hard.Valid == 0);
 927     ASSERT(TempPte.u.Soft.PageFileHigh != 0);
 928     ASSERT(TempPte.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED);
 929
 930     /* Get any page, it will be overwritten */
 931     Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
 932     Page = MiRemoveAnyPage(Color);
 933     if (Page == 0)
 934     {
 935         return STATUS_NO_MEMORY;
 936     }
 937
 938     /* Initialize this PFN */
 939     MiInitializePfn(Page, PointerPte, StoreInstruction);
 940
 941     /* Sets the PFN as being in IO operation */
 942     Pfn1 = MI_PFN_ELEMENT(Page);
 943     ASSERT(Pfn1->u1.Event == NULL);
 944     ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
 945     ASSERT(Pfn1->u3.e1.WriteInProgress == 0);
 946     Pfn1->u3.e1.ReadInProgress = 1;
 947
 948     /* We must write the PTE now as the PFN lock will be released while performing the IO operation */
 949     MI_MAKE_TRANSITION_PTE(&TempPte, Page, Protection);
 950
 951     MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 952
 953     /* Release the PFN lock while we proceed */
 954     MiReleasePfnLock(*OldIrql);
 955
 956     /* Do the paging IO */
 957     Status = MiReadPageFile(Page, PageFileIndex, PageFileOffset);
 958
 959     /* Lock the PFN database again */
 960     *OldIrql = MiAcquirePfnLock();
 961
 962     /* Nobody should have changed that while we were not looking */
 963     ASSERT(Pfn1->u3.e1.ReadInProgress == 1);
 964     ASSERT(Pfn1->u3.e1.WriteInProgress == 0);
 965
 966     if (!NT_SUCCESS(Status))
 967     {
 968         /* Malheur! */
 969         ASSERT(FALSE);
 970         Pfn1->u4.InPageError = 1;
 971         Pfn1->u1.ReadStatus = Status;
 972     }
 973
 974     /* And the PTE can finally be valid */
 975     MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, Protection, Page);
 976     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 977
 978     Pfn1->u3.e1.ReadInProgress = 0;
 979     /* Did someone start to wait on us while we proceeded ? */
 980     if (Pfn1->u1.Event)
 981     {
 982         /* Tell them we're done */
 983         KeSetEvent(Pfn1->u1.Event, IO_NO_INCREMENT, FALSE);
 984     }
 985
 986     return Status;
 987 }
 988
 989 static
 990 NTSTATUS
 991 NTAPI
 992 MiResolveTransitionFault(IN BOOLEAN StoreInstruction,
 993                          IN PVOID FaultingAddress,
 994                          IN PMMPTE PointerPte,
 995                          IN PEPROCESS CurrentProcess,
 996                          IN KIRQL OldIrql,
 997                          OUT PKEVENT **InPageBlock)
 998 {
 999     PFN_NUMBER PageFrameIndex;
1000     PMMPFN Pfn1;
1001     MMPTE TempPte;
1002     PMMPTE PointerToPteForProtoPage;
1003     DPRINT("Transition fault on 0x%p with PTE 0x%p in process %s\n",
1004             FaultingAddress, PointerPte, CurrentProcess->ImageFileName);
1005
1006     /* Windowss does this check */
1007     ASSERT(*InPageBlock == NULL);
1008
1009     /* ARM3 doesn't support this path */
1010     ASSERT(OldIrql != MM_NOIRQL);
1011
1012     /* Capture the PTE and make sure it's in transition format */
1013     TempPte = *PointerPte;
1014     ASSERT((TempPte.u.Soft.Valid == 0) &&
1015            (TempPte.u.Soft.Prototype == 0) &&
1016            (TempPte.u.Soft.Transition == 1));
1017
1018     /* Get the PFN and the PFN entry */
1019     PageFrameIndex = TempPte.u.Trans.PageFrameNumber;
1020     DPRINT("Transition PFN: %lx\n", PageFrameIndex);
1021     Pfn1 = MiGetPfnEntry(PageFrameIndex);
1022
1023     /* One more transition fault! */
1024     InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
1025
1026     /* This is from ARM3 -- Windows normally handles this here */
1027     ASSERT(Pfn1->u4.InPageError == 0);
1028
1029     /* See if we should wait before terminating the fault */
1030     if ((Pfn1->u3.e1.ReadInProgress == 1)
1031             || ((Pfn1->u3.e1.WriteInProgress == 1) && StoreInstruction))
1032     {
1033         DPRINT1("The page is currently in a page transition !\n");
1034         *InPageBlock = &Pfn1->u1.Event;
1035         if (PointerPte == Pfn1->PteAddress)
1036         {
1037             DPRINT1("And this if for this particular PTE.\n");
1038             /* The PTE will be made valid by the thread serving the fault */
1039             return STATUS_SUCCESS; // FIXME: Maybe something more descriptive
1040         }
1041     }
1042
1043     /* Windows checks there's some free pages and this isn't an in-page error */
1044     ASSERT(MmAvailablePages > 0);
1045     ASSERT(Pfn1->u4.InPageError == 0);
1046
1047     /* ReactOS checks for this */
1048     ASSERT(MmAvailablePages > 32);
1049
1050     /* Was this a transition page in the valid list, or free/zero list? */
1051     if (Pfn1->u3.e1.PageLocation == ActiveAndValid)
1052     {
1053         /* All Windows does here is a bunch of sanity checks */
1054         DPRINT("Transition in active list\n");
1055         ASSERT((Pfn1->PteAddress >= MiAddressToPte(MmPagedPoolStart)) &&
1056                (Pfn1->PteAddress <= MiAddressToPte(MmPagedPoolEnd)));
1057         ASSERT(Pfn1->u2.ShareCount != 0);
1058         ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1059     }
1060     else
1061     {
1062         /* Otherwise, the page is removed from its list */
1063         DPRINT("Transition page in free/zero list\n");
1064         MiUnlinkPageFromList(Pfn1);
1065         MiReferenceUnusedPageAndBumpLockCount(Pfn1);
1066     }
1067
1068     /* At this point, there should no longer be any in-page errors */
1069     ASSERT(Pfn1->u4.InPageError == 0);
1070
1071     /* Check if this was a PFN with no more share references */
1072     if (Pfn1->u2.ShareCount == 0) MiDropLockCount(Pfn1);
1073
1074     /* Bump the share count and make the page valid */
1075     Pfn1->u2.ShareCount++;
1076     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1077
1078     /* Prototype PTEs are in paged pool, which itself might be in transition */
1079     if (FaultingAddress >= MmSystemRangeStart)
1080     {
1081         /* Check if this is a paged pool PTE in transition state */
1082         PointerToPteForProtoPage = MiAddressToPte(PointerPte);
1083         TempPte = *PointerToPteForProtoPage;
1084         if ((TempPte.u.Hard.Valid == 0) && (TempPte.u.Soft.Transition == 1))
1085         {
1086             /* This isn't yet supported */
1087             DPRINT1("Double transition fault not yet supported\n");
1088             ASSERT(FALSE);
1089         }
1090     }
1091
1092     /* Build the final PTE */
1093     ASSERT(PointerPte->u.Hard.Valid == 0);
1094     ASSERT(PointerPte->u.Trans.Prototype == 0);
1095     ASSERT(PointerPte->u.Trans.Transition == 1);
1096     TempPte.u.Long = (PointerPte->u.Long & ~0xFFF) |
1097                      (MmProtectToPteMask[PointerPte->u.Trans.Protection]) |
1098                      MiDetermineUserGlobalPteMask(PointerPte);
1099
1100     /* Is the PTE writeable? */
1101     if ((Pfn1->u3.e1.Modified) &&
1102         MI_IS_PAGE_WRITEABLE(&TempPte) &&
1103         !MI_IS_PAGE_COPY_ON_WRITE(&TempPte))
1104     {
1105         /* Make it dirty */
1106         MI_MAKE_DIRTY_PAGE(&TempPte);
1107     }
1108     else
1109     {
1110         /* Make it clean */
1111         MI_MAKE_CLEAN_PAGE(&TempPte);
1112     }
1113
1114     /* Write the valid PTE */
1115     MI_WRITE_VALID_PTE(PointerPte, TempPte);
1116
1117     /* Return success */
1118     return STATUS_PAGE_FAULT_TRANSITION;
1119 }
1120
1121 static
1122 NTSTATUS
1123 NTAPI
1124 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
1125                        IN PVOID Address,
1126                        IN PMMPTE PointerPte,
1127                        IN PMMPTE PointerProtoPte,
1128                        IN OUT PMMPFN *OutPfn,
1129                        OUT PVOID *PageFileData,
1130                        OUT PMMPTE PteValue,
1131                        IN PEPROCESS Process,
1132                        IN KIRQL OldIrql,
1133                        IN PVOID TrapInformation)
1134 {
1135     MMPTE TempPte, PteContents;
1136     PMMPFN Pfn1;
1137     PFN_NUMBER PageFrameIndex;
1138     NTSTATUS Status;
1139     PKEVENT* InPageBlock = NULL;
1140     ULONG Protection;
1141
1142     /* Must be called with an invalid, prototype PTE, with the PFN lock held */
1143     MI_ASSERT_PFN_LOCK_HELD();
1144     ASSERT(PointerPte->u.Hard.Valid == 0);
1145     ASSERT(PointerPte->u.Soft.Prototype == 1);
1146
1147     /* Read the prototype PTE and check if it's valid */
1148     TempPte = *PointerProtoPte;
1149     if (TempPte.u.Hard.Valid == 1)
1150     {
1151         /* One more user of this mapped page */
1152         PageFrameIndex = PFN_FROM_PTE(&TempPte);
1153         Pfn1 = MiGetPfnEntry(PageFrameIndex);
1154         Pfn1->u2.ShareCount++;
1155
1156         /* Call it a transition */
1157         InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
1158
1159         /* Complete the prototype PTE fault -- this will release the PFN lock */
1160         return MiCompleteProtoPteFault(StoreInstruction,
1161                                        Address,
1162                                        PointerPte,
1163                                        PointerProtoPte,
1164                                        OldIrql,
1165                                        OutPfn);
1166     }
1167
1168     /* Make sure there's some protection mask */
1169     if (TempPte.u.Long == 0)
1170     {
1171         /* Release the lock */
1172         DPRINT1("Access on reserved section?\n");
1173         MiReleasePfnLock(OldIrql);
1174         return STATUS_ACCESS_VIOLATION;
1175     }
1176
1177     /* There is no such thing as a decommitted prototype PTE */
1178     ASSERT(TempPte.u.Long != MmDecommittedPte.u.Long);
1179
1180     /* Check for access rights on the PTE proper */
1181     PteContents = *PointerPte;
1182     if (PteContents.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED)
1183     {
1184         if (!PteContents.u.Proto.ReadOnly)
1185         {
1186             Protection = TempPte.u.Soft.Protection;
1187         }
1188         else
1189         {
1190             Protection = MM_READONLY;
1191         }
1192         /* Check for page acess in software */
1193         Status = MiAccessCheck(PointerProtoPte,
1194                                StoreInstruction,
1195                                KernelMode,
1196                                TempPte.u.Soft.Protection,
1197                                TrapInformation,
1198                                TRUE);
1199         ASSERT(Status == STATUS_SUCCESS);
1200     }
1201     else
1202     {
1203         Protection = PteContents.u.Soft.Protection;
1204     }
1205
1206     /* Check for writing copy on write page */
1207     if (((Protection & MM_WRITECOPY) == MM_WRITECOPY) && StoreInstruction)
1208     {
1209         PFN_NUMBER PageFrameIndex, ProtoPageFrameIndex;
1210         ULONG Color;
1211
1212         /* Resolve the proto fault as if it was a read operation */
1213         Status = MiResolveProtoPteFault(FALSE,
1214                                         Address,
1215                                         PointerPte,
1216                                         PointerProtoPte,
1217                                         OutPfn,
1218                                         PageFileData,
1219                                         PteValue,
1220                                         Process,
1221                                         OldIrql,
1222                                         TrapInformation);
1223
1224         if (!NT_SUCCESS(Status))
1225         {
1226             return Status;
1227         }
1228
1229         /* Lock again the PFN lock, MiResolveProtoPteFault unlocked it */
1230         OldIrql = MiAcquirePfnLock();
1231
1232         /* And re-read the proto PTE */
1233         TempPte = *PointerProtoPte;
1234         ASSERT(TempPte.u.Hard.Valid == 1);
1235         ProtoPageFrameIndex = PFN_FROM_PTE(&TempPte);
1236
1237         MI_SET_USAGE(MI_USAGE_COW);
1238         MI_SET_PROCESS(Process);
1239
1240         /* Get a new page for the private copy */
1241         if (Process > HYDRA_PROCESS)
1242             Color = MI_GET_NEXT_PROCESS_COLOR(Process);
1243         else
1244             Color = MI_GET_NEXT_COLOR();
1245
1246         PageFrameIndex = MiRemoveAnyPage(Color);
1247         if (PageFrameIndex == 0)
1248         {
1249             MiReleasePfnLock(OldIrql);
1250             return STATUS_NO_MEMORY;
1251         }
1252
1253         /* Perform the copy */
1254         MiCopyPfn(PageFrameIndex, ProtoPageFrameIndex);
1255
1256         /* This will drop everything MiResolveProtoPteFault referenced */
1257         MiDeletePte(PointerPte, Address, Process, PointerProtoPte);
1258
1259         /* Because now we use this */
1260         Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1261         MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
1262
1263         /* Fix the protection */
1264         Protection &= ~MM_WRITECOPY;
1265         Protection |= MM_READWRITE;
1266         if (Address < MmSystemRangeStart)
1267         {
1268             /* Build the user PTE */
1269             MI_MAKE_HARDWARE_PTE_USER(&PteContents, PointerPte, Protection, PageFrameIndex);
1270         }
1271         else
1272         {
1273             /* Build the kernel PTE */
1274             MI_MAKE_HARDWARE_PTE(&PteContents, PointerPte, Protection, PageFrameIndex);
1275         }
1276
1277         /* And finally, write the valid PTE */
1278         MI_WRITE_VALID_PTE(PointerPte, PteContents);
1279
1280         /* The caller expects us to release the PFN lock */
1281         MiReleasePfnLock(OldIrql);
1282         return Status;
1283     }
1284
1285     /* Check for clone PTEs */
1286     if (PointerPte <= MiHighestUserPte) ASSERT(Process->CloneRoot == NULL);
1287
1288     /* We don't support mapped files yet */
1289     ASSERT(TempPte.u.Soft.Prototype == 0);
1290
1291     /* We might however have transition PTEs */
1292     if (TempPte.u.Soft.Transition == 1)
1293     {
1294         /* Resolve the transition fault */
1295         ASSERT(OldIrql != MM_NOIRQL);
1296         Status = MiResolveTransitionFault(StoreInstruction,
1297                                           Address,
1298                                           PointerProtoPte,
1299                                           Process,
1300                                           OldIrql,
1301                                           &InPageBlock);
1302         ASSERT(NT_SUCCESS(Status));
1303     }
1304     else
1305     {
1306         /* We also don't support paged out pages */
1307         ASSERT(TempPte.u.Soft.PageFileHigh == 0);
1308
1309         /* Resolve the demand zero fault */
1310         Status = MiResolveDemandZeroFault(Address,
1311                                           PointerProtoPte,
1312                                           (ULONG)TempPte.u.Soft.Protection,
1313                                           Process,
1314                                           OldIrql);
1315 #if MI_TRACE_PFNS
1316         /* Update debug info */
1317         if (TrapInformation)
1318             MiGetPfnEntry(PointerProtoPte->u.Hard.PageFrameNumber)->CallSite = (PVOID)((PKTRAP_FRAME)TrapInformation)->Eip;
1319         else
1320             MiGetPfnEntry(PointerProtoPte->u.Hard.PageFrameNumber)->CallSite = _ReturnAddress();
1321 #endif
1322
1323         ASSERT(NT_SUCCESS(Status));
1324     }
1325
1326     /* Complete the prototype PTE fault -- this will release the PFN lock */
1327     ASSERT(PointerPte->u.Hard.Valid == 0);
1328     return MiCompleteProtoPteFault(StoreInstruction,
1329                                    Address,
1330                                    PointerPte,
1331                                    PointerProtoPte,
1332                                    OldIrql,
1333                                    OutPfn);
1334 }
1335
1336 NTSTATUS
1337 NTAPI
1338 MiDispatchFault(IN ULONG FaultCode,
1339                 IN PVOID Address,
1340                 IN PMMPTE PointerPte,
1341                 IN PMMPTE PointerProtoPte,
1342                 IN BOOLEAN Recursive,
1343                 IN PEPROCESS Process,
1344                 IN PVOID TrapInformation,
1345                 IN PMMVAD Vad)
1346 {
1347     MMPTE TempPte;
1348     KIRQL OldIrql, LockIrql;
1349     NTSTATUS Status;
1350     PMMPTE SuperProtoPte;
1351     PMMPFN Pfn1, OutPfn = NULL;
1352     PFN_NUMBER PageFrameIndex;
1353     PFN_COUNT PteCount, ProcessedPtes;
1354     DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
1355              Address,
1356              Process);
1357
1358     /* Make sure the addresses are ok */
1359     ASSERT(PointerPte == MiAddressToPte(Address));
1360
1361     //
1362     // Make sure APCs are off and we're not at dispatch
1363     //
1364     OldIrql = KeGetCurrentIrql();
1365     ASSERT(OldIrql <= APC_LEVEL);
1366     ASSERT(KeAreAllApcsDisabled() == TRUE);
1367
1368     //
1369     // Grab a copy of the PTE
1370     //
1371     TempPte = *PointerPte;
1372
1373     /* Do we have a prototype PTE? */
1374     if (PointerProtoPte)
1375     {
1376         /* This should never happen */
1377         ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
1378
1379         /* Check if this is a kernel-mode address */
1380         SuperProtoPte = MiAddressToPte(PointerProtoPte);
1381         if (Address >= MmSystemRangeStart)
1382         {
1383             /* Lock the PFN database */
1384             LockIrql = MiAcquirePfnLock();
1385
1386             /* Has the PTE been made valid yet? */
1387             if (!SuperProtoPte->u.Hard.Valid)
1388             {
1389                 ASSERT(FALSE);
1390             }
1391             else if (PointerPte->u.Hard.Valid == 1)
1392             {
1393                 ASSERT(FALSE);
1394             }
1395
1396             /* Resolve the fault -- this will release the PFN lock */
1397             Status = MiResolveProtoPteFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode),
1398                                             Address,
1399                                             PointerPte,
1400                                             PointerProtoPte,
1401                                             &OutPfn,
1402                                             NULL,
1403                                             NULL,
1404                                             Process,
1405                                             LockIrql,
1406                                             TrapInformation);
1407             ASSERT(Status == STATUS_SUCCESS);
1408
1409             /* Complete this as a transition fault */
1410             ASSERT(OldIrql == KeGetCurrentIrql());
1411             ASSERT(OldIrql <= APC_LEVEL);
1412             ASSERT(KeAreAllApcsDisabled() == TRUE);
1413             return Status;
1414         }
1415         else
1416         {
1417             /* We only handle the lookup path */
1418             ASSERT(PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED);
1419
1420             /* Is there a non-image VAD? */
1421             if ((Vad) &&
1422                 (Vad->u.VadFlags.VadType != VadImageMap) &&
1423                 !(Vad->u2.VadFlags2.ExtendableFile))
1424             {
1425                 /* One day, ReactOS will cluster faults */
1426                 ASSERT(Address <= MM_HIGHEST_USER_ADDRESS);
1427                 DPRINT("Should cluster fault, but won't\n");
1428             }
1429
1430             /* Only one PTE to handle for now */
1431             PteCount = 1;
1432             ProcessedPtes = 0;
1433
1434             /* Lock the PFN database */
1435             LockIrql = MiAcquirePfnLock();
1436
1437             /* We only handle the valid path */
1438             ASSERT(SuperProtoPte->u.Hard.Valid == 1);
1439
1440             /* Capture the PTE */
1441             TempPte = *PointerProtoPte;
1442
1443             /* Loop to handle future case of clustered faults */
1444             while (TRUE)
1445             {
1446                 /* For our current usage, this should be true */
1447                 if (TempPte.u.Hard.Valid == 1)
1448                 {
1449                     /* Bump the share count on the PTE */
1450                     PageFrameIndex = PFN_FROM_PTE(&TempPte);
1451                     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1452                     Pfn1->u2.ShareCount++;
1453                 }
1454                 else if ((TempPte.u.Soft.Prototype == 0) &&
1455                          (TempPte.u.Soft.Transition == 1))
1456                 {
1457                     /* This is a standby page, bring it back from the cache */
1458                     PageFrameIndex = TempPte.u.Trans.PageFrameNumber;
1459                     DPRINT("oooh, shiny, a soft fault! 0x%lx\n", PageFrameIndex);
1460                     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1461                     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1462
1463                     /* Should not yet happen in ReactOS */
1464                     ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
1465                     ASSERT(Pfn1->u4.InPageError == 0);
1466
1467                     /* Get the page */
1468                     MiUnlinkPageFromList(Pfn1);
1469
1470                     /* Bump its reference count */
1471                     ASSERT(Pfn1->u2.ShareCount == 0);
1472                     InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1473                     Pfn1->u2.ShareCount++;
1474
1475                     /* Make it valid again */
1476                     /* This looks like another macro.... */
1477                     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1478                     ASSERT(PointerProtoPte->u.Hard.Valid == 0);
1479                     ASSERT(PointerProtoPte->u.Trans.Prototype == 0);
1480                     ASSERT(PointerProtoPte->u.Trans.Transition == 1);
1481                     TempPte.u.Long = (PointerProtoPte->u.Long & ~0xFFF) |
1482                                      MmProtectToPteMask[PointerProtoPte->u.Trans.Protection];
1483                     TempPte.u.Hard.Valid = 1;
1484                     MI_MAKE_ACCESSED_PAGE(&TempPte);
1485
1486                     /* Is the PTE writeable? */
1487                     if ((Pfn1->u3.e1.Modified) &&
1488                         MI_IS_PAGE_WRITEABLE(&TempPte) &&
1489                         !MI_IS_PAGE_COPY_ON_WRITE(&TempPte))
1490                     {
1491                         /* Make it dirty */
1492                         MI_MAKE_DIRTY_PAGE(&TempPte);
1493                     }
1494                     else
1495                     {
1496                         /* Make it clean */
1497                         MI_MAKE_CLEAN_PAGE(&TempPte);
1498                     }
1499
1500                     /* Write the valid PTE */
1501                     MI_WRITE_VALID_PTE(PointerProtoPte, TempPte);
1502                     ASSERT(PointerPte->u.Hard.Valid == 0);
1503                 }
1504                 else
1505                 {
1506                     /* Page is invalid, get out of the loop */
1507                     break;
1508                 }
1509
1510                 /* One more done, was it the last? */
1511                 if (++ProcessedPtes == PteCount)
1512                 {
1513                     /* Complete the fault */
1514                     MiCompleteProtoPteFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode),
1515                                             Address,
1516                                             PointerPte,
1517                                             PointerProtoPte,
1518                                             LockIrql,
1519                                             &OutPfn);
1520
1521                     /* THIS RELEASES THE PFN LOCK! */
1522                     break;
1523                 }
1524
1525                 /* No clustered faults yet */
1526                 ASSERT(FALSE);
1527             }
1528
1529             /* Did we resolve the fault? */
1530             if (ProcessedPtes)
1531             {
1532                 /* Bump the transition count */
1533                 InterlockedExchangeAddSizeT(&KeGetCurrentPrcb()->MmTransitionCount, ProcessedPtes);
1534                 ProcessedPtes--;
1535
1536                 /* Loop all the processing we did */
1537                 ASSERT(ProcessedPtes == 0);
1538
1539                 /* Complete this as a transition fault */
1540                 ASSERT(OldIrql == KeGetCurrentIrql());
1541                 ASSERT(OldIrql <= APC_LEVEL);
1542                 ASSERT(KeAreAllApcsDisabled() == TRUE);
1543                 return STATUS_PAGE_FAULT_TRANSITION;
1544             }
1545
1546             /* We did not -- PFN lock is still held, prepare to resolve prototype PTE fault */
1547             OutPfn = MI_PFN_ELEMENT(SuperProtoPte->u.Hard.PageFrameNumber);
1548             MiReferenceUsedPageAndBumpLockCount(OutPfn);
1549             ASSERT(OutPfn->u3.e2.ReferenceCount > 1);
1550             ASSERT(PointerPte->u.Hard.Valid == 0);
1551
1552             /* Resolve the fault -- this will release the PFN lock */
1553             Status = MiResolveProtoPteFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode),
1554                                             Address,
1555                                             PointerPte,
1556                                             PointerProtoPte,
1557                                             &OutPfn,
1558                                             NULL,
1559                                             NULL,
1560                                             Process,
1561                                             LockIrql,
1562                                             TrapInformation);
1563             //ASSERT(Status != STATUS_ISSUE_PAGING_IO);
1564             //ASSERT(Status != STATUS_REFAULT);
1565             //ASSERT(Status != STATUS_PTE_CHANGED);
1566
1567             /* Did the routine clean out the PFN or should we? */
1568             if (OutPfn)
1569             {
1570                 /* We had a locked PFN, so acquire the PFN lock to dereference it */
1571                 ASSERT(PointerProtoPte != NULL);
1572                 OldIrql = MiAcquirePfnLock();
1573
1574                 /* Dereference the locked PFN */
1575                 MiDereferencePfnAndDropLockCount(OutPfn);
1576                 ASSERT(OutPfn->u3.e2.ReferenceCount >= 1);
1577
1578                 /* And now release the lock */
1579                 MiReleasePfnLock(OldIrql);
1580             }
1581
1582             /* Complete this as a transition fault */
1583             ASSERT(OldIrql == KeGetCurrentIrql());
1584             ASSERT(OldIrql <= APC_LEVEL);
1585             ASSERT(KeAreAllApcsDisabled() == TRUE);
1586             return Status;
1587         }
1588     }
1589
1590     /* Is this a transition PTE */
1591     if (TempPte.u.Soft.Transition)
1592     {
1593         PKEVENT* InPageBlock = NULL;
1594         PKEVENT PreviousPageEvent;
1595         KEVENT CurrentPageEvent;
1596
1597         /* Lock the PFN database */
1598         LockIrql = MiAcquirePfnLock();
1599
1600         /* Resolve */
1601         Status = MiResolveTransitionFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode), Address, PointerPte, Process, LockIrql, &InPageBlock);
1602
1603         ASSERT(NT_SUCCESS(Status));
1604
1605         if (InPageBlock != NULL)
1606         {
1607             /* Another thread is reading or writing this page. Put us into the waiting queue. */
1608             KeInitializeEvent(&CurrentPageEvent, NotificationEvent, FALSE);
1609             PreviousPageEvent = *InPageBlock;
1610             *InPageBlock = &CurrentPageEvent;
1611         }
1612
1613         /* And now release the lock and leave*/
1614         MiReleasePfnLock(LockIrql);
1615
1616         if (InPageBlock != NULL)
1617         {
1618             KeWaitForSingleObject(&CurrentPageEvent, WrPageIn, KernelMode, FALSE, NULL);
1619
1620             /* Let's the chain go on */
1621             if (PreviousPageEvent)
1622             {
1623                 KeSetEvent(PreviousPageEvent, IO_NO_INCREMENT, FALSE);
1624             }
1625         }
1626
1627         ASSERT(OldIrql == KeGetCurrentIrql());
1628         ASSERT(OldIrql <= APC_LEVEL);
1629         ASSERT(KeAreAllApcsDisabled() == TRUE);
1630         return Status;
1631     }
1632
1633     /* Should we page the data back in ? */
1634     if (TempPte.u.Soft.PageFileHigh != 0)
1635     {
1636         /* Lock the PFN database */
1637         LockIrql = MiAcquirePfnLock();
1638
1639         /* Resolve */
1640         Status = MiResolvePageFileFault(!MI_IS_NOT_PRESENT_FAULT(FaultCode), Address, PointerPte, Process, &LockIrql);
1641
1642         /* And now release the lock and leave*/
1643         MiReleasePfnLock(LockIrql);
1644
1645         ASSERT(OldIrql == KeGetCurrentIrql());
1646         ASSERT(OldIrql <= APC_LEVEL);
1647         ASSERT(KeAreAllApcsDisabled() == TRUE);
1648         return Status;
1649     }
1650
1651     //
1652     // The PTE must be invalid but not completely empty. It must also not be a
1653     // prototype a transition or a paged-out PTE as those scenarii should've been handled above.
1654     // These are all Windows checks
1655     //
1656     ASSERT(TempPte.u.Hard.Valid == 0);
1657     ASSERT(TempPte.u.Soft.Prototype == 0);
1658     ASSERT(TempPte.u.Soft.Transition == 0);
1659     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
1660     ASSERT(TempPte.u.Long != 0);
1661
1662     //
1663     // If we got this far, the PTE can only be a demand zero PTE, which is what
1664     // we want. Go handle it!
1665     //
1666     Status = MiResolveDemandZeroFault(Address,
1667                                       PointerPte,
1668                                       (ULONG)TempPte.u.Soft.Protection,
1669                                       Process,
1670                                       MM_NOIRQL);
1671     ASSERT(KeAreAllApcsDisabled() == TRUE);
1672     if (NT_SUCCESS(Status))
1673     {
1674 #if MI_TRACE_PFNS
1675         /* Update debug info */
1676         if (TrapInformation)
1677             MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber)->CallSite = (PVOID)((PKTRAP_FRAME)TrapInformation)->Eip;
1678         else
1679             MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber)->CallSite = _ReturnAddress();
1680 #endif
1681
1682         //
1683         // Make sure we're returning in a sane state and pass the status down
1684         //
1685         ASSERT(OldIrql == KeGetCurrentIrql());
1686         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
1687         return Status;
1688     }
1689
1690     //
1691     // Return status
1692     //
1693     return Status;
1694 }
1695
1696 NTSTATUS
1697 NTAPI
1698 MmArmAccessFault(IN ULONG FaultCode,
1699                  IN PVOID Address,
1700                  IN KPROCESSOR_MODE Mode,
1701                  IN PVOID TrapInformation)
1702 {
1703     KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
1704     PMMPTE ProtoPte = NULL;
1705     PMMPTE PointerPte = MiAddressToPte(Address);
1706     PMMPDE PointerPde = MiAddressToPde(Address);
1707 #if (_MI_PAGING_LEVELS >= 3)
1708     PMMPDE PointerPpe = MiAddressToPpe(Address);
1709 #if (_MI_PAGING_LEVELS == 4)
1710     PMMPDE PointerPxe = MiAddressToPxe(Address);
1711 #endif
1712 #endif
1713     MMPTE TempPte;
1714     PETHREAD CurrentThread;
1715     PEPROCESS CurrentProcess;
1716     NTSTATUS Status;
1717     PMMSUPPORT WorkingSet;
1718     ULONG ProtectionCode;
1719     PMMVAD Vad = NULL;
1720     PFN_NUMBER PageFrameIndex;
1721     ULONG Color;
1722     BOOLEAN IsSessionAddress;
1723     PMMPFN Pfn1;
1724     DPRINT("ARM3 FAULT AT: %p\n", Address);
1725
1726     /* Check for page fault on high IRQL */
1727     if (OldIrql > APC_LEVEL)
1728     {
1729 #if (_MI_PAGING_LEVELS < 3)
1730         /* Could be a page table for paged pool, which we'll allow */
1731         if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address)) MiSynchronizeSystemPde((PMMPDE)PointerPte);
1732         MiCheckPdeForPagedPool(Address);
1733 #endif
1734         /* Check if any of the top-level pages are invalid */
1735         if (
1736 #if (_MI_PAGING_LEVELS == 4)
1737             (PointerPxe->u.Hard.Valid == 0) ||
1738 #endif
1739 #if (_MI_PAGING_LEVELS >= 3)
1740             (PointerPpe->u.Hard.Valid == 0) ||
1741 #endif
1742             (PointerPde->u.Hard.Valid == 0) ||
1743             (PointerPte->u.Hard.Valid == 0))
1744         {
1745             /* This fault is not valid, print out some debugging help */
1746             DbgPrint("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
1747                      Address,
1748                      OldIrql);
1749             if (TrapInformation)
1750             {
1751                 PKTRAP_FRAME TrapFrame = TrapInformation;
1752 #ifdef _M_IX86
1753                 DbgPrint("MM:***EIP %p, EFL %p\n", TrapFrame->Eip, TrapFrame->EFlags);
1754                 DbgPrint("MM:***EAX %p, ECX %p EDX %p\n", TrapFrame->Eax, TrapFrame->Ecx, TrapFrame->Edx);
1755                 DbgPrint("MM:***EBX %p, ESI %p EDI %p\n", TrapFrame->Ebx, TrapFrame->Esi, TrapFrame->Edi);
1756 #elif defined(_M_AMD64)
1757                 DbgPrint("MM:***RIP %p, EFL %p\n", TrapFrame->Rip, TrapFrame->EFlags);
1758                 DbgPrint("MM:***RAX %p, RCX %p RDX %p\n", TrapFrame->Rax, TrapFrame->Rcx, TrapFrame->Rdx);
1759                 DbgPrint("MM:***RBX %p, RSI %p RDI %p\n", TrapFrame->Rbx, TrapFrame->Rsi, TrapFrame->Rdi);
1760 #elif defined(_M_ARM)
1761                 DbgPrint("MM:***PC %p\n", TrapFrame->Pc);
1762                 DbgPrint("MM:***R0 %p, R1 %p R2 %p, R3 %p\n", TrapFrame->R0, TrapFrame->R1, TrapFrame->R2, TrapFrame->R3);
1763                 DbgPrint("MM:***R11 %p, R12 %p SP %p, LR %p\n", TrapFrame->R11, TrapFrame->R12, TrapFrame->Sp, TrapFrame->Lr);
1764 #endif
1765             }
1766
1767             /* Tell the trap handler to fail */
1768             return STATUS_IN_PAGE_ERROR | 0x10000000;
1769         }
1770
1771         /* Not yet implemented in ReactOS */
1772         ASSERT(MI_IS_PAGE_LARGE(PointerPde) == FALSE);
1773         ASSERT((!MI_IS_NOT_PRESENT_FAULT(FaultCode) && MI_IS_PAGE_COPY_ON_WRITE(PointerPte)) == FALSE);
1774
1775         /* Check if this was a write */
1776         if (MI_IS_WRITE_ACCESS(FaultCode))
1777         {
1778             /* Was it to a read-only page? */
1779             Pfn1 = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
1780             if (!(PointerPte->u.Long & PTE_READWRITE) &&
1781                 !(Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE))
1782             {
1783                 /* Crash with distinguished bugcheck code */
1784                 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
1785                              (ULONG_PTR)Address,
1786                              PointerPte->u.Long,
1787                              (ULONG_PTR)TrapInformation,
1788                              10);
1789             }
1790         }
1791
1792         /* Nothing is actually wrong */
1793         DPRINT1("Fault at IRQL %u is ok (%p)\n", OldIrql, Address);
1794         return STATUS_SUCCESS;
1795     }
1796
1797     /* Check for kernel fault address */
1798     if (Address >= MmSystemRangeStart)
1799     {
1800         /* Bail out, if the fault came from user mode */
1801         if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
1802
1803 #if (_MI_PAGING_LEVELS == 2)
1804         if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address)) MiSynchronizeSystemPde((PMMPDE)PointerPte);
1805         MiCheckPdeForPagedPool(Address);
1806 #endif
1807
1808         /* Check if the higher page table entries are invalid */
1809         if (
1810 #if (_MI_PAGING_LEVELS == 4)
1811             /* AMD64 system, check if PXE is invalid */
1812             (PointerPxe->u.Hard.Valid == 0) ||
1813 #endif
1814 #if (_MI_PAGING_LEVELS >= 3)
1815             /* PAE/AMD64 system, check if PPE is invalid */
1816             (PointerPpe->u.Hard.Valid == 0) ||
1817 #endif
1818             /* Always check if the PDE is valid */
1819             (PointerPde->u.Hard.Valid == 0))
1820         {
1821             /* PXE/PPE/PDE (still) not valid, kill the system */
1822             KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
1823                          (ULONG_PTR)Address,
1824                          FaultCode,
1825                          (ULONG_PTR)TrapInformation,
1826                          2);
1827         }
1828
1829         /* Not handling session faults yet */
1830         IsSessionAddress = MI_IS_SESSION_ADDRESS(Address);
1831
1832         /* The PDE is valid, so read the PTE */
1833         TempPte = *PointerPte;
1834         if (TempPte.u.Hard.Valid == 1)
1835         {
1836             /* Check if this was system space or session space */
1837             if (!IsSessionAddress)
1838             {
1839                 /* Check if the PTE is still valid under PFN lock */
1840                 OldIrql = MiAcquirePfnLock();
1841                 TempPte = *PointerPte;
1842                 if (TempPte.u.Hard.Valid)
1843                 {
1844                     /* Check if this was a write */
1845                     if (MI_IS_WRITE_ACCESS(FaultCode))
1846                     {
1847                         /* Was it to a read-only page? */
1848                         Pfn1 = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
1849                         if (!(PointerPte->u.Long & PTE_READWRITE) &&
1850                             !(Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE))
1851                         {
1852                             /* Crash with distinguished bugcheck code */
1853                             KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
1854                                          (ULONG_PTR)Address,
1855                                          PointerPte->u.Long,
1856                                          (ULONG_PTR)TrapInformation,
1857                                          11);
1858                         }
1859                     }
1860
1861                     /* Check for execution of non-executable memory */
1862                     if (MI_IS_INSTRUCTION_FETCH(FaultCode) &&
1863                         !MI_IS_PAGE_EXECUTABLE(&TempPte))
1864                     {
1865                         KeBugCheckEx(ATTEMPTED_EXECUTE_OF_NOEXECUTE_MEMORY,
1866                                      (ULONG_PTR)Address,
1867                                      (ULONG_PTR)TempPte.u.Long,
1868                                      (ULONG_PTR)TrapInformation,
1869                                      1);
1870                     }
1871                 }
1872
1873                 /* Release PFN lock and return all good */
1874                 MiReleasePfnLock(OldIrql);
1875                 return STATUS_SUCCESS;
1876             }
1877         }
1878 #if (_MI_PAGING_LEVELS == 2)
1879         /* Check if this was a session PTE that needs to remap the session PDE */
1880         if (MI_IS_SESSION_PTE(Address))
1881         {
1882             /* Do the remapping */
1883             Status = MiCheckPdeForSessionSpace(Address);
1884             if (!NT_SUCCESS(Status))
1885             {
1886                 /* It failed, this address is invalid */
1887                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
1888                              (ULONG_PTR)Address,
1889                              FaultCode,
1890                              (ULONG_PTR)TrapInformation,
1891                              6);
1892             }
1893         }
1894 #else
1895
1896 _WARN("Session space stuff is not implemented yet!")
1897
1898 #endif
1899
1900         /* Check for a fault on the page table or hyperspace */
1901         if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
1902         {
1903 #if (_MI_PAGING_LEVELS < 3)
1904             /* Windows does this check but I don't understand why -- it's done above! */
1905             ASSERT(MiCheckPdeForPagedPool(Address) != STATUS_WAIT_1);
1906 #endif
1907             /* Handle this as a user mode fault */
1908             goto UserFault;
1909         }
1910
1911         /* Get the current thread */
1912         CurrentThread = PsGetCurrentThread();
1913
1914         /* What kind of address is this */
1915         if (!IsSessionAddress)
1916         {
1917             /* Use the system working set */
1918             WorkingSet = &MmSystemCacheWs;
1919             CurrentProcess = NULL;
1920
1921             /* Make sure we don't have a recursive working set lock */
1922             if ((CurrentThread->OwnsProcessWorkingSetExclusive) ||
1923                 (CurrentThread->OwnsProcessWorkingSetShared) ||
1924                 (CurrentThread->OwnsSystemWorkingSetExclusive) ||
1925                 (CurrentThread->OwnsSystemWorkingSetShared) ||
1926                 (CurrentThread->OwnsSessionWorkingSetExclusive) ||
1927                 (CurrentThread->OwnsSessionWorkingSetShared))
1928             {
1929                 /* Fail */
1930                 return STATUS_IN_PAGE_ERROR | 0x10000000;
1931             }
1932         }
1933         else
1934         {
1935             /* Use the session process and working set */
1936             CurrentProcess = HYDRA_PROCESS;
1937             WorkingSet = &MmSessionSpace->GlobalVirtualAddress->Vm;
1938
1939             /* Make sure we don't have a recursive working set lock */
1940             if ((CurrentThread->OwnsSessionWorkingSetExclusive) ||
1941                 (CurrentThread->OwnsSessionWorkingSetShared))
1942             {
1943                 /* Fail */
1944                 return STATUS_IN_PAGE_ERROR | 0x10000000;
1945             }
1946         }
1947 RetryKernel:
1948         /* Acquire the working set lock */
1949         KeRaiseIrql(APC_LEVEL, &LockIrql);
1950         MiLockWorkingSet(CurrentThread, WorkingSet);
1951
1952         /* Re-read PTE now that we own the lock */
1953         TempPte = *PointerPte;
1954         if (TempPte.u.Hard.Valid == 1)
1955         {
1956             /* Check if this was a write */
1957             if (MI_IS_WRITE_ACCESS(FaultCode))
1958             {
1959                 /* Was it to a read-only page that is not copy on write? */
1960                 Pfn1 = MI_PFN_ELEMENT(PointerPte->u.Hard.PageFrameNumber);
1961                 if (!(TempPte.u.Long & PTE_READWRITE) &&
1962                     !(Pfn1->OriginalPte.u.Soft.Protection & MM_READWRITE) &&
1963                     !MI_IS_PAGE_COPY_ON_WRITE(&TempPte))
1964                 {
1965                     /* Case not yet handled */
1966                     ASSERT(!IsSessionAddress);
1967
1968                     /* Crash with distinguished bugcheck code */
1969                     KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
1970                                  (ULONG_PTR)Address,
1971                                  TempPte.u.Long,
1972                                  (ULONG_PTR)TrapInformation,
1973                                  12);
1974                 }
1975             }
1976
1977             /* Check for execution of non-executable memory */
1978             if (MI_IS_INSTRUCTION_FETCH(FaultCode) &&
1979                 !MI_IS_PAGE_EXECUTABLE(&TempPte))
1980             {
1981                 KeBugCheckEx(ATTEMPTED_EXECUTE_OF_NOEXECUTE_MEMORY,
1982                              (ULONG_PTR)Address,
1983                              (ULONG_PTR)TempPte.u.Long,
1984                              (ULONG_PTR)TrapInformation,
1985                              2);
1986             }
1987
1988             /* Check for read-only write in session space */
1989             if ((IsSessionAddress) &&
1990                 MI_IS_WRITE_ACCESS(FaultCode) &&
1991                 !MI_IS_PAGE_WRITEABLE(&TempPte))
1992             {
1993                 /* Sanity check */
1994                 ASSERT(MI_IS_SESSION_IMAGE_ADDRESS(Address));
1995
1996                 /* Was this COW? */
1997                 if (!MI_IS_PAGE_COPY_ON_WRITE(&TempPte))
1998                 {
1999                     /* Then this is not allowed */
2000                     KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
2001                                  (ULONG_PTR)Address,
2002                                  (ULONG_PTR)TempPte.u.Long,
2003                                  (ULONG_PTR)TrapInformation,
2004                                  13);
2005                 }
2006
2007                 /* Otherwise, handle COW */
2008                 ASSERT(FALSE);
2009             }
2010
2011             /* Release the working set */
2012             MiUnlockWorkingSet(CurrentThread, WorkingSet);
2013             KeLowerIrql(LockIrql);
2014
2015             /* Otherwise, the PDE was probably invalid, and all is good now */
2016             return STATUS_SUCCESS;
2017         }
2018
2019         /* Check one kind of prototype PTE */
2020         if (TempPte.u.Soft.Prototype)
2021         {
2022             /* Make sure protected pool is on, and that this is a pool address */
2023             if ((MmProtectFreedNonPagedPool) &&
2024                 (((Address >= MmNonPagedPoolStart) &&
2025                   (Address < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
2026                                      MmSizeOfNonPagedPoolInBytes))) ||
2027                  ((Address >= MmNonPagedPoolExpansionStart) &&
2028                   (Address < MmNonPagedPoolEnd))))
2029             {
2030                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
2031                 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL,
2032                              (ULONG_PTR)Address,
2033                              FaultCode,
2034                              Mode,
2035                              4);
2036             }
2037
2038             /* Get the prototype PTE! */
2039             ProtoPte = MiProtoPteToPte(&TempPte);
2040
2041             /* Do we need to locate the prototype PTE in session space? */
2042             if ((IsSessionAddress) &&
2043                 (TempPte.u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED))
2044             {
2045                 /* Yep, go find it as well as the VAD for it */
2046                 ProtoPte = MiCheckVirtualAddress(Address,
2047                                                  &ProtectionCode,
2048                                                  &Vad);
2049                 ASSERT(ProtoPte != NULL);
2050             }
2051         }
2052         else
2053         {
2054             /* We don't implement transition PTEs */
2055             ASSERT(TempPte.u.Soft.Transition == 0);
2056
2057             /* Check for no-access PTE */
2058             if (TempPte.u.Soft.Protection == MM_NOACCESS)
2059             {
2060                 /* Bugcheck the system! */
2061                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
2062                              (ULONG_PTR)Address,
2063                              FaultCode,
2064                              (ULONG_PTR)TrapInformation,
2065                              1);
2066             }
2067
2068             /* Check for no protecton at all */
2069             if (TempPte.u.Soft.Protection == MM_ZERO_ACCESS)
2070             {
2071                 /* Bugcheck the system! */
2072                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
2073                              (ULONG_PTR)Address,
2074                              FaultCode,
2075                              (ULONG_PTR)TrapInformation,
2076                              0);
2077             }
2078         }
2079
2080         /* Check for demand page */
2081         if (MI_IS_WRITE_ACCESS(FaultCode) &&
2082             !(ProtoPte) &&
2083             !(IsSessionAddress) &&
2084             !(TempPte.u.Hard.Valid))
2085         {
2086             /* Get the protection code */
2087             ASSERT(TempPte.u.Soft.Transition == 0);
2088             if (!(TempPte.u.Soft.Protection & MM_READWRITE))
2089             {
2090                 /* Bugcheck the system! */
2091                 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
2092                              (ULONG_PTR)Address,
2093                              TempPte.u.Long,
2094                              (ULONG_PTR)TrapInformation,
2095                              14);
2096             }
2097         }
2098
2099         /* Now do the real fault handling */
2100         Status = MiDispatchFault(FaultCode,
2101                                  Address,
2102                                  PointerPte,
2103                                  ProtoPte,
2104                                  FALSE,
2105                                  CurrentProcess,
2106                                  TrapInformation,
2107                                  NULL);
2108
2109         /* Release the working set */
2110         ASSERT(KeAreAllApcsDisabled() == TRUE);
2111         MiUnlockWorkingSet(CurrentThread, WorkingSet);
2112         KeLowerIrql(LockIrql);
2113
2114         if (Status == STATUS_NO_MEMORY)
2115         {
2116             MmRebalanceMemoryConsumersAndWait();
2117             goto RetryKernel;
2118         }
2119
2120         /* We are done! */
2121         DPRINT("Fault resolved with status: %lx\n", Status);
2122         return Status;
2123     }
2124
2125     /* This is a user fault */
2126 UserFault:
2127     CurrentThread = PsGetCurrentThread();
2128     CurrentProcess = (PEPROCESS)CurrentThread->Tcb.ApcState.Process;
2129
2130     /* Lock the working set */
2131     MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
2132
2133     ProtectionCode = MM_INVALID_PROTECTION;
2134
2135 #if (_MI_PAGING_LEVELS == 4)
2136     /* Check if the PXE is valid */
2137     if (PointerPxe->u.Hard.Valid == 0)
2138     {
2139         /* Right now, we only handle scenarios where the PXE is totally empty */
2140         ASSERT(PointerPxe->u.Long == 0);
2141
2142         /* This is only possible for user mode addresses! */
2143         ASSERT(PointerPte <= MiHighestUserPte);
2144
2145         /* Check if we have a VAD */
2146         MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
2147         if (ProtectionCode == MM_NOACCESS)
2148         {
2149             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2150             return STATUS_ACCESS_VIOLATION;
2151         }
2152
2153         /* Resolve a demand zero fault */
2154         Status = MiResolveDemandZeroFault(PointerPpe,
2155                                  PointerPxe,
2156                                  MM_EXECUTE_READWRITE,
2157                                  CurrentProcess,
2158                                  MM_NOIRQL);
2159         if (!NT_SUCCESS(Status))
2160         {
2161             goto ExitUser;
2162         }
2163
2164         /* We should come back with a valid PXE */
2165         ASSERT(PointerPxe->u.Hard.Valid == 1);
2166     }
2167 #endif
2168
2169 #if (_MI_PAGING_LEVELS >= 3)
2170     /* Check if the PPE is valid */
2171     if (PointerPpe->u.Hard.Valid == 0)
2172     {
2173         /* Right now, we only handle scenarios where the PPE is totally empty */
2174         ASSERT(PointerPpe->u.Long == 0);
2175
2176         /* This is only possible for user mode addresses! */
2177         ASSERT(PointerPte <= MiHighestUserPte);
2178
2179         /* Check if we have a VAD, unless we did this already */
2180         if (ProtectionCode == MM_INVALID_PROTECTION)
2181         {
2182             MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
2183         }
2184
2185         if (ProtectionCode == MM_NOACCESS)
2186         {
2187             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2188             return STATUS_ACCESS_VIOLATION;
2189         }
2190
2191         /* Resolve a demand zero fault */
2192         Status = MiResolveDemandZeroFault(PointerPde,
2193                                  PointerPpe,
2194                                  MM_EXECUTE_READWRITE,
2195                                  CurrentProcess,
2196                                  MM_NOIRQL);
2197         if (!NT_SUCCESS(Status))
2198         {
2199             goto ExitUser;
2200         }
2201
2202         /* We should come back with a valid PPE */
2203         ASSERT(PointerPpe->u.Hard.Valid == 1);
2204         MiIncrementPageTableReferences(PointerPde);
2205     }
2206 #endif
2207
2208     /* Check if the PDE is invalid */
2209     if (PointerPde->u.Hard.Valid == 0)
2210     {
2211         /* Right now, we only handle scenarios where the PDE is totally empty */
2212         ASSERT(PointerPde->u.Long == 0);
2213
2214         /* And go dispatch the fault on the PDE. This should handle the demand-zero */
2215 #if MI_TRACE_PFNS
2216         UserPdeFault = TRUE;
2217 #endif
2218         /* Check if we have a VAD, unless we did this already */
2219         if (ProtectionCode == MM_INVALID_PROTECTION)
2220         {
2221             MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
2222         }
2223
2224         if (ProtectionCode == MM_NOACCESS)
2225         {
2226 #if (_MI_PAGING_LEVELS == 2)
2227             /* Could be a page table for paged pool */
2228             MiCheckPdeForPagedPool(Address);
2229 #endif
2230             /* Has the code above changed anything -- is this now a valid PTE? */
2231             Status = (PointerPde->u.Hard.Valid == 1) ? STATUS_SUCCESS : STATUS_ACCESS_VIOLATION;
2232
2233             /* Either this was a bogus VA or we've fixed up a paged pool PDE */
2234             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2235             return Status;
2236         }
2237
2238         /* Resolve a demand zero fault */
2239         Status = MiResolveDemandZeroFault(PointerPte,
2240                                  PointerPde,
2241                                  MM_EXECUTE_READWRITE,
2242                                  CurrentProcess,
2243                                  MM_NOIRQL);
2244         if (!NT_SUCCESS(Status))
2245         {
2246             goto ExitUser;
2247         }
2248
2249 #if _MI_PAGING_LEVELS >= 3
2250         MiIncrementPageTableReferences(PointerPte);
2251 #endif
2252
2253 #if MI_TRACE_PFNS
2254         UserPdeFault = FALSE;
2255         /* Update debug info */
2256         if (TrapInformation)
2257             MiGetPfnEntry(PointerPde->u.Hard.PageFrameNumber)->CallSite = (PVOID)((PKTRAP_FRAME)TrapInformation)->Eip;
2258         else
2259             MiGetPfnEntry(PointerPde->u.Hard.PageFrameNumber)->CallSite = _ReturnAddress();
2260 #endif
2261         /* We should come back with APCs enabled, and with a valid PDE */
2262         ASSERT(KeAreAllApcsDisabled() == TRUE);
2263         ASSERT(PointerPde->u.Hard.Valid == 1);
2264     }
2265     else
2266     {
2267         /* Not yet implemented in ReactOS */
2268         ASSERT(MI_IS_PAGE_LARGE(PointerPde) == FALSE);
2269     }
2270
2271     /* Now capture the PTE. */
2272     TempPte = *PointerPte;
2273
2274     /* Check if the PTE is valid */
2275     if (TempPte.u.Hard.Valid)
2276     {
2277         /* Check if this is a write on a readonly PTE */
2278         if (MI_IS_WRITE_ACCESS(FaultCode))
2279         {
2280             /* Is this a copy on write PTE? */
2281             if (MI_IS_PAGE_COPY_ON_WRITE(&TempPte))
2282             {
2283                 PFN_NUMBER PageFrameIndex, OldPageFrameIndex;
2284                 PMMPFN Pfn1;
2285
2286                 LockIrql = MiAcquirePfnLock();
2287
2288                 ASSERT(MmAvailablePages > 0);
2289
2290                 MI_SET_USAGE(MI_USAGE_COW);
2291                 MI_SET_PROCESS(CurrentProcess);
2292
2293                 /* Allocate a new page and copy it */
2294                 PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_PROCESS_COLOR(CurrentProcess));
2295                 if (PageFrameIndex == 0)
2296                 {
2297                     MiReleasePfnLock(LockIrql);
2298                     Status = STATUS_NO_MEMORY;
2299                     goto ExitUser;
2300                 }
2301                 OldPageFrameIndex = PFN_FROM_PTE(&TempPte);
2302
2303                 MiCopyPfn(PageFrameIndex, OldPageFrameIndex);
2304
2305                 /* Dereference whatever this PTE is referencing */
2306                 Pfn1 = MI_PFN_ELEMENT(OldPageFrameIndex);
2307                 ASSERT(Pfn1->u3.e1.PrototypePte == 1);
2308                 ASSERT(!MI_IS_PFN_DELETED(Pfn1));
2309                 ProtoPte = Pfn1->PteAddress;
2310                 MiDeletePte(PointerPte, Address, CurrentProcess, ProtoPte);
2311
2312                 /* And make a new shiny one with our page */
2313                 MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
2314                 TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
2315                 TempPte.u.Hard.Write = 1;
2316                 TempPte.u.Hard.CopyOnWrite = 0;
2317
2318                 MI_WRITE_VALID_PTE(PointerPte, TempPte);
2319
2320                 MiReleasePfnLock(LockIrql);
2321
2322                 /* Return the status */
2323                 MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2324                 return STATUS_PAGE_FAULT_COPY_ON_WRITE;
2325             }
2326
2327             /* Is this a read-only PTE? */
2328             if (!MI_IS_PAGE_WRITEABLE(&TempPte))
2329             {
2330                 /* Return the status */
2331                 MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2332                 return STATUS_ACCESS_VIOLATION;
2333             }
2334         }
2335
2336 #if _MI_HAS_NO_EXECUTE
2337         /* Check for execution of non-executable memory */
2338         if (MI_IS_INSTRUCTION_FETCH(FaultCode) &&
2339             !MI_IS_PAGE_EXECUTABLE(&TempPte))
2340         {
2341             /* Check if execute enable was set */
2342             if (CurrentProcess->Pcb.Flags.ExecuteEnable)
2343             {
2344                 /* Fix up the PTE to be executable */
2345                 TempPte.u.Hard.NoExecute = 0;
2346                 MI_UPDATE_VALID_PTE(PointerPte, TempPte);
2347                 MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2348                 return STATUS_SUCCESS;
2349             }
2350
2351             /* Return the status */
2352             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2353             return STATUS_ACCESS_VIOLATION;
2354         }
2355 #endif
2356
2357         /* The fault has already been resolved by a different thread */
2358         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2359         return STATUS_SUCCESS;
2360     }
2361
2362     /* Quick check for demand-zero */
2363     if ((TempPte.u.Long == (MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS)) ||
2364         (TempPte.u.Long == (MM_EXECUTE_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS)))
2365     {
2366         /* Resolve the fault */
2367         Status = MiResolveDemandZeroFault(Address,
2368                                  PointerPte,
2369                                  TempPte.u.Soft.Protection,
2370                                  CurrentProcess,
2371                                  MM_NOIRQL);
2372         if (!NT_SUCCESS(Status))
2373         {
2374             goto ExitUser;
2375         }
2376
2377 #if MI_TRACE_PFNS
2378         /* Update debug info */
2379         if (TrapInformation)
2380             MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber)->CallSite = (PVOID)((PKTRAP_FRAME)TrapInformation)->Eip;
2381         else
2382             MiGetPfnEntry(PointerPte->u.Hard.PageFrameNumber)->CallSite = _ReturnAddress();
2383 #endif
2384
2385         /* Return the status */
2386         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2387         return STATUS_PAGE_FAULT_DEMAND_ZERO;
2388     }
2389
2390     /* Check for zero PTE */
2391     if (TempPte.u.Long == 0)
2392     {
2393         /* Check if this address range belongs to a valid allocation (VAD) */
2394         ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
2395         if (ProtectionCode == MM_NOACCESS)
2396         {
2397 #if (_MI_PAGING_LEVELS == 2)
2398             /* Could be a page table for paged pool */
2399             MiCheckPdeForPagedPool(Address);
2400 #endif
2401             /* Has the code above changed anything -- is this now a valid PTE? */
2402             Status = (PointerPte->u.Hard.Valid == 1) ? STATUS_SUCCESS : STATUS_ACCESS_VIOLATION;
2403
2404             /* Either this was a bogus VA or we've fixed up a paged pool PDE */
2405             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2406             return Status;
2407         }
2408
2409         /*
2410          * Check if this is a real user-mode address or actually a kernel-mode
2411          * page table for a user mode address
2412          */
2413         if (Address <= MM_HIGHEST_USER_ADDRESS
2414 #if _MI_PAGING_LEVELS >= 3
2415             || MiIsUserPte(Address)
2416 #if _MI_PAGING_LEVELS == 4
2417             || MiIsUserPde(Address)
2418 #endif
2419 #endif
2420         )
2421         {
2422             /* Add an additional page table reference */
2423             MiIncrementPageTableReferences(Address);
2424         }
2425
2426         /* Is this a guard page? */
2427         if ((ProtectionCode & MM_PROTECT_SPECIAL) == MM_GUARDPAGE)
2428         {
2429             /* The VAD protection cannot be MM_DECOMMIT! */
2430             ASSERT(ProtectionCode != MM_DECOMMIT);
2431
2432             /* Remove the bit */
2433             TempPte.u.Soft.Protection = ProtectionCode & ~MM_GUARDPAGE;
2434             MI_WRITE_INVALID_PTE(PointerPte, TempPte);
2435
2436             /* Not supported */
2437             ASSERT(ProtoPte == NULL);
2438             ASSERT(CurrentThread->ApcNeeded == 0);
2439
2440             /* Drop the working set lock */
2441             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2442             ASSERT(KeGetCurrentIrql() == OldIrql);
2443
2444             /* Handle stack expansion */
2445             return MiCheckForUserStackOverflow(Address, TrapInformation);
2446         }
2447
2448         /* Did we get a prototype PTE back? */
2449         if (!ProtoPte)
2450         {
2451             /* Is this PTE actually part of the PDE-PTE self-mapping directory? */
2452             if (PointerPde == MiAddressToPde(PTE_BASE))
2453             {
2454                 /* Then it's really a demand-zero PDE (on behalf of user-mode) */
2455 #ifdef _M_ARM
2456                 _WARN("This is probably completely broken!");
2457                 MI_WRITE_INVALID_PDE((PMMPDE)PointerPte, DemandZeroPde);
2458 #else
2459                 MI_WRITE_INVALID_PDE(PointerPte, DemandZeroPde);
2460 #endif
2461             }
2462             else
2463             {
2464                 /* No, create a new PTE. First, write the protection */
2465                 TempPte.u.Soft.Protection = ProtectionCode;
2466                 MI_WRITE_INVALID_PTE(PointerPte, TempPte);
2467             }
2468
2469             /* Lock the PFN database since we're going to grab a page */
2470             OldIrql = MiAcquirePfnLock();
2471
2472             /* Make sure we have enough pages */
2473             //ASSERT(MmAvailablePages >= 32);
2474
2475             /* Try to get a zero page */
2476             MI_SET_USAGE(MI_USAGE_PEB_TEB);
2477             MI_SET_PROCESS2(CurrentProcess->ImageFileName);
2478             Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
2479             PageFrameIndex = MiRemoveZeroPageSafe(Color);
2480             if (!PageFrameIndex)
2481             {
2482                 /* Grab a page out of there. Later we should grab a colored zero page */
2483                 PageFrameIndex = MiRemoveAnyPage(Color);
2484
2485                 /* Release the lock since we need to do some zeroing */
2486                 MiReleasePfnLock(OldIrql);
2487
2488                 if (PageFrameIndex == 0)
2489                 {
2490                     Status = STATUS_NO_MEMORY;
2491                     goto ExitUser;
2492                 }
2493
2494                 /* Zero out the page, since it's for user-mode */
2495                 MiZeroPfn(PageFrameIndex);
2496
2497                 /* Grab the lock again so we can initialize the PFN entry */
2498                 OldIrql = MiAcquirePfnLock();
2499             }
2500
2501             /* Initialize the PFN entry now */
2502             MiInitializePfn(PageFrameIndex, PointerPte, 1);
2503
2504             /* Increment the count of pages in the process */
2505             CurrentProcess->NumberOfPrivatePages++;
2506
2507             /* One more demand-zero fault */
2508             KeGetCurrentPrcb()->MmDemandZeroCount++;
2509
2510             /* And we're done with the lock */
2511             MiReleasePfnLock(OldIrql);
2512
2513             /* Fault on user PDE, or fault on user PTE? */
2514             if (PointerPte <= MiHighestUserPte)
2515             {
2516                 /* User fault, build a user PTE */
2517                 MI_MAKE_HARDWARE_PTE_USER(&TempPte,
2518                                           PointerPte,
2519                                           PointerPte->u.Soft.Protection,
2520                                           PageFrameIndex);
2521             }
2522             else
2523             {
2524                 /* This is a user-mode PDE, create a kernel PTE for it */
2525                 MI_MAKE_HARDWARE_PTE(&TempPte,
2526                                      PointerPte,
2527                                      PointerPte->u.Soft.Protection,
2528                                      PageFrameIndex);
2529             }
2530
2531             /* Write the dirty bit for writeable pages */
2532             if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
2533
2534             /* And now write down the PTE, making the address valid */
2535             MI_WRITE_VALID_PTE(PointerPte, TempPte);
2536             Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
2537             ASSERT(Pfn1->u1.Event == NULL);
2538
2539             /* Demand zero */
2540             ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
2541             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2542             return STATUS_PAGE_FAULT_DEMAND_ZERO;
2543         }
2544
2545         /* We should have a valid protection here */
2546         ASSERT(ProtectionCode != 0x100);
2547
2548         /* Write the prototype PTE */
2549         TempPte = PrototypePte;
2550         TempPte.u.Soft.Protection = ProtectionCode;
2551         ASSERT(TempPte.u.Long != 0);
2552         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
2553     }
2554     else
2555     {
2556         /* Get the protection code and check if this is a proto PTE */
2557         ProtectionCode = (ULONG)TempPte.u.Soft.Protection;
2558         if (TempPte.u.Soft.Prototype)
2559         {
2560             /* Do we need to go find the real PTE? */
2561             if (TempPte.u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
2562             {
2563                 /* Get the prototype pte and VAD for it */
2564                 ProtoPte = MiCheckVirtualAddress(Address,
2565                                                  &ProtectionCode,
2566                                                  &Vad);
2567                 if (!ProtoPte)
2568                 {
2569                     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
2570                     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2571                     return STATUS_ACCESS_VIOLATION;
2572                 }
2573             }
2574             else
2575             {
2576                 /* Get the prototype PTE! */
2577                 ProtoPte = MiProtoPteToPte(&TempPte);
2578
2579                 /* Is it read-only */
2580                 if (TempPte.u.Proto.ReadOnly)
2581                 {
2582                     /* Set read-only code */
2583                     ProtectionCode = MM_READONLY;
2584                 }
2585                 else
2586                 {
2587                     /* Set unknown protection */
2588                     ProtectionCode = 0x100;
2589                     ASSERT(CurrentProcess->CloneRoot != NULL);
2590                 }
2591             }
2592         }
2593     }
2594
2595     /* Do we have a valid protection code? */
2596     if (ProtectionCode != 0x100)
2597     {
2598         /* Run a software access check first, including to detect guard pages */
2599         Status = MiAccessCheck(PointerPte,
2600                                !MI_IS_NOT_PRESENT_FAULT(FaultCode),
2601                                Mode,
2602                                ProtectionCode,
2603                                TrapInformation,
2604                                FALSE);
2605         if (Status != STATUS_SUCCESS)
2606         {
2607             /* Not supported */
2608             ASSERT(CurrentThread->ApcNeeded == 0);
2609
2610             /* Drop the working set lock */
2611             MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2612             ASSERT(KeGetCurrentIrql() == OldIrql);
2613
2614             /* Did we hit a guard page? */
2615             if (Status == STATUS_GUARD_PAGE_VIOLATION)
2616             {
2617                 /* Handle stack expansion */
2618                 return MiCheckForUserStackOverflow(Address, TrapInformation);
2619             }
2620
2621             /* Otherwise, fail back to the caller directly */
2622             return Status;
2623         }
2624     }
2625
2626     /* Dispatch the fault */
2627     Status = MiDispatchFault(FaultCode,
2628                              Address,
2629                              PointerPte,
2630                              ProtoPte,
2631                              FALSE,
2632                              CurrentProcess,
2633                              TrapInformation,
2634                              Vad);
2635
2636 ExitUser:
2637
2638     /* Return the status */
2639     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
2640     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
2641
2642     if (Status == STATUS_NO_MEMORY)
2643     {
2644         MmRebalanceMemoryConsumersAndWait();
2645         goto UserFault;
2646     }
2647
2648     return Status;
2649 }
2650
2651 NTSTATUS
2652 NTAPI
2653 MmGetExecuteOptions(IN PULONG ExecuteOptions)
2654 {
2655     PKPROCESS CurrentProcess = &PsGetCurrentProcess()->Pcb;
2656     ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
2657
2658     *ExecuteOptions = 0;
2659
2660     if (CurrentProcess->Flags.ExecuteDisable)
2661     {
2662         *ExecuteOptions |= MEM_EXECUTE_OPTION_DISABLE;
2663     }
2664
2665     if (CurrentProcess->Flags.ExecuteEnable)
2666     {
2667         *ExecuteOptions |= MEM_EXECUTE_OPTION_ENABLE;
2668     }
2669
2670     if (CurrentProcess->Flags.DisableThunkEmulation)
2671     {
2672         *ExecuteOptions |= MEM_EXECUTE_OPTION_DISABLE_THUNK_EMULATION;
2673     }
2674
2675     if (CurrentProcess->Flags.Permanent)
2676     {
2677         *ExecuteOptions |= MEM_EXECUTE_OPTION_PERMANENT;
2678     }
2679
2680     if (CurrentProcess->Flags.ExecuteDispatchEnable)
2681     {
2682         *ExecuteOptions |= MEM_EXECUTE_OPTION_EXECUTE_DISPATCH_ENABLE;
2683     }
2684
2685     if (CurrentProcess->Flags.ImageDispatchEnable)
2686     {
2687         *ExecuteOptions |= MEM_EXECUTE_OPTION_IMAGE_DISPATCH_ENABLE;
2688     }
2689
2690     return STATUS_SUCCESS;
2691 }
2692
2693 NTSTATUS
2694 NTAPI
2695 MmSetExecuteOptions(IN ULONG ExecuteOptions)
2696 {
2697     PKPROCESS CurrentProcess = &PsGetCurrentProcess()->Pcb;
2698     KLOCK_QUEUE_HANDLE ProcessLock;
2699     NTSTATUS Status = STATUS_ACCESS_DENIED;
2700     ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL);
2701
2702     /* Only accept valid flags */
2703     if (ExecuteOptions & ~MEM_EXECUTE_OPTION_VALID_FLAGS)
2704     {
2705         /* Fail */
2706         DPRINT1("Invalid no-execute options\n");
2707         return STATUS_INVALID_PARAMETER;
2708     }
2709
2710     /* Change the NX state in the process lock */
2711     KiAcquireProcessLockRaiseToSynch(CurrentProcess, &ProcessLock);
2712
2713     /* Don't change anything if the permanent flag was set */
2714     if (!CurrentProcess->Flags.Permanent)
2715     {
2716         /* Start by assuming it's not disabled */
2717         CurrentProcess->Flags.ExecuteDisable = FALSE;
2718
2719         /* Now process each flag and turn the equivalent bit on */
2720         if (ExecuteOptions & MEM_EXECUTE_OPTION_DISABLE)
2721         {
2722             CurrentProcess->Flags.ExecuteDisable = TRUE;
2723         }
2724         if (ExecuteOptions & MEM_EXECUTE_OPTION_ENABLE)
2725         {
2726             CurrentProcess->Flags.ExecuteEnable = TRUE;
2727         }
2728         if (ExecuteOptions & MEM_EXECUTE_OPTION_DISABLE_THUNK_EMULATION)
2729         {
2730             CurrentProcess->Flags.DisableThunkEmulation = TRUE;
2731         }
2732         if (ExecuteOptions & MEM_EXECUTE_OPTION_PERMANENT)
2733         {
2734             CurrentProcess->Flags.Permanent = TRUE;
2735         }
2736         if (ExecuteOptions & MEM_EXECUTE_OPTION_EXECUTE_DISPATCH_ENABLE)
2737         {
2738             CurrentProcess->Flags.ExecuteDispatchEnable = TRUE;
2739         }
2740         if (ExecuteOptions & MEM_EXECUTE_OPTION_IMAGE_DISPATCH_ENABLE)
2741         {
2742             CurrentProcess->Flags.ImageDispatchEnable = TRUE;
2743         }
2744
2745         /* These are turned on by default if no-execution is also enabled */
2746         if (CurrentProcess->Flags.ExecuteEnable)
2747         {
2748             CurrentProcess->Flags.ExecuteDispatchEnable = TRUE;
2749             CurrentProcess->Flags.ImageDispatchEnable = TRUE;
2750         }
2751
2752         /* All good */
2753         Status = STATUS_SUCCESS;
2754     }
2755
2756     /* Release the lock and return status */
2757     KiReleaseProcessLock(&ProcessLock);
2758     return Status;
2759 }
2760
2761 /* EOF */