reactos/ntoskrnl/mm/ARM3/pagfault.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pagfault.c
   5  * PURPOSE:         ARM Memory Manager Page Fault Handling
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include "../ARM3/miarm.h"
  17
  18 /* GLOBALS ********************************************************************/
  19
  20 #if MI_TRACE_PFNS
  21 BOOLEAN UserPdeFault = FALSE;
  22 #endif
  23
  24 /* PRIVATE FUNCTIONS **********************************************************/
  25
  26 PMMPTE
  27 NTAPI
  28 MiCheckVirtualAddress(IN PVOID VirtualAddress,
  29                       OUT PULONG ProtectCode,
  30                       OUT PMMVAD *ProtoVad)
  31 {
  32     PMMVAD Vad;
  33     PMMPTE PointerPte;
  34
  35     /* No prototype/section support for now */
  36     *ProtoVad = NULL;
  37
  38     /* Check if this is a page table address */
  39     if (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress))
  40     {
  41         /* This should never happen, as these addresses are handled by the double-maping */
  42         if (((PMMPTE)VirtualAddress >= MiAddressToPte(MmPagedPoolStart)) &&
  43             ((PMMPTE)VirtualAddress <= MmPagedPoolInfo.LastPteForPagedPool))
  44         {
  45             /* Fail such access */
  46             *ProtectCode = MM_NOACCESS;
  47             return NULL;
  48         }
  49
  50         /* Return full access rights */
  51         *ProtectCode = MM_READWRITE;
  52         return NULL;
  53     }
  54
  55     /* Should not be a session address */
  56     ASSERT(MI_IS_SESSION_ADDRESS(VirtualAddress) == FALSE);
  57
  58     /* Special case for shared data */
  59     if (PAGE_ALIGN(VirtualAddress) == (PVOID)MM_SHARED_USER_DATA_VA)
  60     {
  61         /* It's a read-only page */
  62         *ProtectCode = MM_READONLY;
  63         return MmSharedUserDataPte;
  64     }
  65
  66     /* Find the VAD, it might not exist if the address is bogus */
  67     Vad = MiLocateAddress(VirtualAddress);
  68     if (!Vad)
  69     {
  70         /* Bogus virtual address */
  71         *ProtectCode = MM_NOACCESS;
  72         return NULL;
  73     }
  74
  75     /* This must be a VM VAD */
  76     ASSERT(Vad->u.VadFlags.VadType == VadNone);
  77
  78     /* Check if it's a section, or just an allocation */
  79     if (Vad->u.VadFlags.PrivateMemory == TRUE)
  80     {
  81         /* This must be a TEB/PEB VAD */
  82         ASSERT(Vad->u.VadFlags.MemCommit == TRUE);
  83         *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
  84         return NULL;
  85     }
  86     else
  87     {
  88         /* Return the proto VAD */
  89         ASSERT(Vad->u2.VadFlags2.ExtendableFile == 0);
  90         *ProtoVad = Vad;
  91
  92         /* Get the prototype PTE for this page */
  93         PointerPte = (((ULONG_PTR)VirtualAddress >> PAGE_SHIFT) - Vad->StartingVpn) + Vad->FirstPrototypePte;
  94         ASSERT(PointerPte <= Vad->LastContiguousPte);
  95         ASSERT(PointerPte != NULL);
  96
  97         /* Return the Prototype PTE and the protection for the page mapping */
  98         *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
  99         return PointerPte;
 100     }
 101 }
 102
 103 NTSTATUS
 104 FASTCALL
 105 MiCheckPdeForPagedPool(IN PVOID Address)
 106 {
 107     PMMPDE PointerPde;
 108     NTSTATUS Status = STATUS_SUCCESS;
 109
 110     /* No session support in ReactOS yet */
 111     ASSERT(MI_IS_SESSION_ADDRESS(Address) == FALSE);
 112     ASSERT(MI_IS_SESSION_PTE(Address) == FALSE);
 113
 114     //
 115     // Check if this is a fault while trying to access the page table itself
 116     //
 117     if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
 118     {
 119         //
 120         // Send a hint to the page fault handler that this is only a valid fault
 121         // if we already detected this was access within the page table range
 122         //
 123         PointerPde = (PMMPDE)MiAddressToPte(Address);
 124         Status = STATUS_WAIT_1;
 125     }
 126     else if (Address < MmSystemRangeStart)
 127     {
 128         //
 129         // This is totally illegal
 130         //
 131         return STATUS_ACCESS_VIOLATION;
 132     }
 133     else
 134     {
 135         //
 136         // Get the PDE for the address
 137         //
 138         PointerPde = MiAddressToPde(Address);
 139     }
 140
 141     //
 142     // Check if it's not valid
 143     //
 144     if (PointerPde->u.Hard.Valid == 0)
 145     {
 146 #ifdef _M_AMD64
 147         ASSERT(FALSE);
 148 #else
 149         //
 150         // Copy it from our double-mapped system page directory
 151         //
 152         InterlockedExchangePte(PointerPde,
 153                                MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)].u.Long);
 154 #endif
 155     }
 156
 157     //
 158     // Return status
 159     //
 160     return Status;
 161 }
 162
 163 VOID
 164 NTAPI
 165 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
 166 {
 167     PMMPTE ZeroPte;
 168     MMPTE TempPte;
 169     PMMPFN Pfn1;
 170     PVOID ZeroAddress;
 171
 172     /* Get the PFN for this page */
 173     Pfn1 = MiGetPfnEntry(PageFrameNumber);
 174     ASSERT(Pfn1);
 175
 176     /* Grab a system PTE we can use to zero the page */
 177     ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
 178     ASSERT(ZeroPte);
 179
 180     /* Initialize the PTE for it */
 181     TempPte = ValidKernelPte;
 182     TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
 183
 184     /* Setup caching */
 185     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 186     {
 187         /* Write combining, no caching */
 188         MI_PAGE_DISABLE_CACHE(&TempPte);
 189         MI_PAGE_WRITE_COMBINED(&TempPte);
 190     }
 191     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 192     {
 193         /* Write through, no caching */
 194         MI_PAGE_DISABLE_CACHE(&TempPte);
 195         MI_PAGE_WRITE_THROUGH(&TempPte);
 196     }
 197
 198     /* Make the system PTE valid with our PFN */
 199     MI_WRITE_VALID_PTE(ZeroPte, TempPte);
 200
 201     /* Get the address it maps to, and zero it out */
 202     ZeroAddress = MiPteToAddress(ZeroPte);
 203     KeZeroPages(ZeroAddress, PAGE_SIZE);
 204
 205     /* Now get rid of it */
 206     MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
 207 }
 208
 209 NTSTATUS
 210 NTAPI
 211 MiResolveDemandZeroFault(IN PVOID Address,
 212                          IN PMMPTE PointerPte,
 213                          IN PEPROCESS Process,
 214                          IN KIRQL OldIrql)
 215 {
 216     PFN_NUMBER PageFrameNumber = 0;
 217     MMPTE TempPte;
 218     BOOLEAN NeedZero = FALSE, HaveLock = FALSE;
 219     ULONG Color;
 220     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
 221             Address,
 222             Process);
 223
 224     /* Must currently only be called by paging path */
 225     if ((Process) && (OldIrql == MM_NOIRQL))
 226     {
 227         /* Sanity check */
 228         ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
 229
 230         /* No forking yet */
 231         ASSERT(Process->ForkInProgress == NULL);
 232
 233         /* Get process color */
 234         Color = MI_GET_NEXT_PROCESS_COLOR(Process);
 235         ASSERT(Color != 0xFFFFFFFF);
 236
 237         /* We'll need a zero page */
 238         NeedZero = TRUE;
 239     }
 240     else
 241     {
 242         /* Check if we need a zero page */
 243         NeedZero = (OldIrql != MM_NOIRQL);
 244
 245         /* Get the next system page color */
 246         Color = MI_GET_NEXT_COLOR();
 247     }
 248
 249     /* Check if the PFN database should be acquired */
 250     if (OldIrql == MM_NOIRQL)
 251     {
 252         /* Acquire it and remember we should release it after */
 253         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 254         HaveLock = TRUE;
 255     }
 256
 257     /* We either manually locked the PFN DB, or already came with it locked */
 258     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 259
 260     /* Do we need a zero page? */
 261     ASSERT(PointerPte->u.Hard.Valid == 0);
 262 #if MI_TRACE_PFNS
 263     if (UserPdeFault) MI_SET_USAGE(MI_USAGE_PAGE_TABLE);
 264     if (!UserPdeFault) MI_SET_USAGE(MI_USAGE_DEMAND_ZERO);
 265 #endif
 266     if (Process) MI_SET_PROCESS2(Process->ImageFileName);
 267     if (!Process) MI_SET_PROCESS2("Kernel Demand 0");
 268     if ((NeedZero) && (Process))
 269     {
 270         /* Try to get one, if we couldn't grab a free page and zero it */
 271         PageFrameNumber = MiRemoveZeroPageSafe(Color);
 272         if (PageFrameNumber)
 273         {
 274             /* We got a genuine zero page, stop worrying about it */
 275             NeedZero = FALSE;
 276         }
 277         else
 278         {
 279             /* We'll need a free page and zero it manually */
 280             PageFrameNumber = MiRemoveAnyPage(Color);
 281         }
 282     }
 283     else if (!NeedZero)
 284     {
 285         /* Process or system doesn't want a zero page, grab anything */
 286         PageFrameNumber = MiRemoveAnyPage(Color);
 287     }
 288     else
 289     {
 290         /* System wants a zero page, obtain one */
 291         PageFrameNumber = MiRemoveZeroPage(Color);
 292     }
 293
 294     /* Initialize it */
 295     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
 296
 297     /* Release PFN lock if needed */
 298     if (HaveLock) KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 299
 300     //
 301     // Increment demand zero faults
 302     //
 303     InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
 304
 305     /* Zero the page if need be */
 306     if (NeedZero) MiZeroPfn(PageFrameNumber);
 307
 308     /* Build the PTE */
 309     if (PointerPte <= MiHighestUserPte)
 310     {
 311         /* For user mode */
 312         MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 313                                   PointerPte,
 314                                   PointerPte->u.Soft.Protection,
 315                                   PageFrameNumber);
 316     }
 317     else
 318     {
 319         /* For kernel mode */
 320         MI_MAKE_HARDWARE_PTE(&TempPte,
 321                              PointerPte,
 322                              PointerPte->u.Soft.Protection,
 323                              PageFrameNumber);
 324     }
 325
 326     /* Set it dirty if it's a writable page */
 327     if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
 328
 329     /* Write it */
 330     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 331
 332     //
 333     // It's all good now
 334     //
 335     DPRINT("Paged pool page has now been paged in\n");
 336     return STATUS_PAGE_FAULT_DEMAND_ZERO;
 337 }
 338
 339 NTSTATUS
 340 NTAPI
 341 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
 342                         IN PVOID Address,
 343                         IN PMMPTE PointerPte,
 344                         IN PMMPTE PointerProtoPte,
 345                         IN KIRQL OldIrql,
 346                         IN PMMPFN Pfn1)
 347 {
 348     MMPTE TempPte;
 349     PMMPTE OriginalPte;
 350     ULONG_PTR Protection;
 351     PFN_NUMBER PageFrameIndex;
 352
 353     /* Must be called with an valid prototype PTE, with the PFN lock held */
 354     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 355     ASSERT(PointerProtoPte->u.Hard.Valid == 1);
 356
 357     /* Get the page */
 358     PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
 359
 360     /* Get the PFN entry and set it as a prototype PTE */
 361     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 362     Pfn1->u3.e1.PrototypePte = 1;
 363
 364     /* FIXME: Increment the share count for the page table */
 365
 366     /* Check where we should be getting the protection information from */
 367     if (PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
 368     {
 369         /* Get the protection from the PTE, there's no real Proto PTE data */
 370         Protection = PointerPte->u.Soft.Protection;
 371     }
 372     else
 373     {
 374         /* Get the protection from the original PTE link */
 375         OriginalPte = &Pfn1->OriginalPte;
 376         Protection = OriginalPte->u.Soft.Protection;
 377     }
 378
 379     /* Release the PFN lock */
 380     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 381
 382     /* Remove caching bits */
 383     Protection &= ~(MM_NOCACHE | MM_NOACCESS);
 384
 385     /* Check if this is a kernel or user address */
 386     if (Address < MmSystemRangeStart)
 387     {
 388         /* Build the user PTE */
 389         MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, Protection, PageFrameIndex);
 390     }
 391     else
 392     {
 393         /* Build the kernel PTE */
 394         MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, Protection, PageFrameIndex);
 395     }
 396
 397     /* Write the PTE */
 398     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 399
 400     /* Return success */
 401     return STATUS_SUCCESS;
 402 }
 403
 404 NTSTATUS
 405 NTAPI
 406 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
 407                        IN PVOID Address,
 408                        IN PMMPTE PointerPte,
 409                        IN PMMPTE PointerProtoPte,
 410                        IN OUT PMMPFN *OutPfn,
 411                        OUT PVOID *PageFileData,
 412                        OUT PMMPTE PteValue,
 413                        IN PEPROCESS Process,
 414                        IN KIRQL OldIrql,
 415                        IN PVOID TrapInformation)
 416 {
 417     MMPTE TempPte;
 418     PMMPFN Pfn1;
 419     PFN_NUMBER PageFrameIndex;
 420     NTSTATUS Status;
 421
 422     /* Must be called with an invalid, prototype PTE, with the PFN lock held */
 423     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 424     ASSERT(PointerPte->u.Hard.Valid == 0);
 425     ASSERT(PointerPte->u.Soft.Prototype == 1);
 426
 427     /* Read the prototype PTE and check if it's valid */
 428     TempPte = *PointerProtoPte;
 429     if (TempPte.u.Hard.Valid == 1)
 430     {
 431         /* One more user of this mapped page */
 432         PageFrameIndex = PFN_FROM_PTE(&TempPte);
 433         Pfn1 = MiGetPfnEntry(PageFrameIndex);
 434         Pfn1->u2.ShareCount++;
 435
 436         /* Call it a transition */
 437         InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
 438
 439         /* Complete the prototype PTE fault -- this will release the PFN lock */
 440         return MiCompleteProtoPteFault(StoreInstruction,
 441                                        Address,
 442                                        PointerPte,
 443                                        PointerProtoPte,
 444                                        OldIrql,
 445                                        NULL);
 446     }
 447
 448     /* Make sure there's some protection mask */
 449     if (TempPte.u.Long == 0)
 450     {
 451         /* Release the lock */
 452         DPRINT1("Access on reserved section?\n");
 453         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 454         return STATUS_ACCESS_VIOLATION;
 455     }
 456
 457     /* This is the only thing we support right now */
 458     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 459     ASSERT(TempPte.u.Proto.ReadOnly == 0);
 460     ASSERT(PointerPte > MiHighestUserPte);
 461     ASSERT(TempPte.u.Soft.Prototype == 0);
 462     ASSERT(TempPte.u.Soft.Transition == 0);
 463
 464     /* Resolve the demand zero fault */
 465     Status = MiResolveDemandZeroFault(Address, PointerProtoPte, Process, OldIrql);
 466     ASSERT(NT_SUCCESS(Status));
 467
 468     /* Complete the prototype PTE fault -- this will release the PFN lock */
 469     ASSERT(PointerPte->u.Hard.Valid == 0);
 470     return MiCompleteProtoPteFault(StoreInstruction,
 471                                    Address,
 472                                    PointerPte,
 473                                    PointerProtoPte,
 474                                    OldIrql,
 475                                    NULL);
 476 }
 477
 478 NTSTATUS
 479 NTAPI
 480 MiDispatchFault(IN BOOLEAN StoreInstruction,
 481                 IN PVOID Address,
 482                 IN PMMPTE PointerPte,
 483                 IN PMMPTE PointerProtoPte,
 484                 IN BOOLEAN Recursive,
 485                 IN PEPROCESS Process,
 486                 IN PVOID TrapInformation,
 487                 IN PVOID Vad)
 488 {
 489     MMPTE TempPte;
 490     KIRQL OldIrql, LockIrql;
 491     NTSTATUS Status;
 492     PMMPTE SuperProtoPte;
 493     DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
 494              Address,
 495              Process);
 496
 497     //
 498     // Make sure APCs are off and we're not at dispatch
 499     //
 500     OldIrql = KeGetCurrentIrql();
 501     ASSERT(OldIrql <= APC_LEVEL);
 502     ASSERT(KeAreAllApcsDisabled() == TRUE);
 503
 504     //
 505     // Grab a copy of the PTE
 506     //
 507     TempPte = *PointerPte;
 508
 509     /* Do we have a prototype PTE? */
 510     if (PointerProtoPte)
 511     {
 512         /* This should never happen */
 513         ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
 514
 515         /* Check if this is a kernel-mode address */
 516         SuperProtoPte = MiAddressToPte(PointerProtoPte);
 517         if (Address >= MmSystemRangeStart)
 518         {
 519             /* Lock the PFN database */
 520             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 521
 522             /* Has the PTE been made valid yet? */
 523             if (!SuperProtoPte->u.Hard.Valid)
 524             {
 525                 UNIMPLEMENTED;
 526                 while (TRUE);
 527             }
 528             else
 529             {
 530                 /* Resolve the fault -- this will release the PFN lock */
 531                 ASSERT(PointerPte->u.Hard.Valid == 0);
 532                 Status = MiResolveProtoPteFault(StoreInstruction,
 533                                                 Address,
 534                                                 PointerPte,
 535                                                 PointerProtoPte,
 536                                                 NULL,
 537                                                 NULL,
 538                                                 NULL,
 539                                                 Process,
 540                                                 LockIrql,
 541                                                 TrapInformation);
 542                 ASSERT(Status == STATUS_SUCCESS);
 543
 544                 /* Complete this as a transition fault */
 545                 ASSERT(OldIrql == KeGetCurrentIrql());
 546                 ASSERT(OldIrql <= APC_LEVEL);
 547                 ASSERT(KeAreAllApcsDisabled() == TRUE);
 548                 return Status;
 549             }
 550         }
 551         else
 552         {
 553             /* We currently only handle very limited paths */
 554             ASSERT(PointerPte->u.Soft.Prototype == 1);
 555             ASSERT(PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED);
 556
 557             /* Lock the PFN database */
 558             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 559
 560             /* For our current usage, this should be true */
 561             ASSERT(SuperProtoPte->u.Hard.Valid == 1);
 562             ASSERT(TempPte.u.Hard.Valid == 0);
 563
 564             /* Resolve the fault -- this will release the PFN lock */
 565             Status = MiResolveProtoPteFault(StoreInstruction,
 566                                             Address,
 567                                             PointerPte,
 568                                             PointerProtoPte,
 569                                             NULL,
 570                                             NULL,
 571                                             NULL,
 572                                             Process,
 573                                             LockIrql,
 574                                             TrapInformation);
 575             ASSERT(Status == STATUS_SUCCESS);
 576
 577             /* Complete this as a transition fault */
 578             ASSERT(OldIrql == KeGetCurrentIrql());
 579             ASSERT(OldIrql <= APC_LEVEL);
 580             ASSERT(KeAreAllApcsDisabled() == TRUE);
 581             return STATUS_PAGE_FAULT_TRANSITION;
 582         }
 583     }
 584
 585     //
 586     // The PTE must be invalid
 587     //
 588     ASSERT(TempPte.u.Hard.Valid == 0);
 589
 590     /* Check if the PTE is completely empty */
 591     if (TempPte.u.Long == 0)
 592     {
 593         KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 594                      (ULONG_PTR)Address,
 595                      StoreInstruction,
 596                      (ULONG_PTR)TrapInformation,
 597                      2);
 598     }
 599
 600     //
 601     // No prototype, transition or page file software PTEs in ARM3 yet
 602     //
 603     ASSERT(TempPte.u.Soft.Prototype == 0);
 604     ASSERT(TempPte.u.Soft.Transition == 0);
 605     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 606
 607     //
 608     // If we got this far, the PTE can only be a demand zero PTE, which is what
 609     // we want. Go handle it!
 610     //
 611     Status = MiResolveDemandZeroFault(Address,
 612                                       PointerPte,
 613                                       Process,
 614                                       MM_NOIRQL);
 615     ASSERT(KeAreAllApcsDisabled() == TRUE);
 616     if (NT_SUCCESS(Status))
 617     {
 618         //
 619         // Make sure we're returning in a sane state and pass the status down
 620         //
 621         ASSERT(OldIrql == KeGetCurrentIrql());
 622         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 623         return Status;
 624     }
 625
 626     //
 627     // Generate an access fault
 628     //
 629     return STATUS_ACCESS_VIOLATION;
 630 }
 631
 632 NTSTATUS
 633 NTAPI
 634 MmArmAccessFault(IN BOOLEAN StoreInstruction,
 635                  IN PVOID Address,
 636                  IN KPROCESSOR_MODE Mode,
 637                  IN PVOID TrapInformation)
 638 {
 639     KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
 640     PMMPTE ProtoPte = NULL;
 641     PMMPTE PointerPte = MiAddressToPte(Address);
 642     PMMPDE PointerPde = MiAddressToPde(Address);
 643 #if (_MI_PAGING_LEVELS >= 3)
 644     PMMPDE PointerPpe = MiAddressToPpe(Address);
 645 #if (_MI_PAGING_LEVELS == 4)
 646     PMMPDE PointerPxe = MiAddressToPxe(Address);
 647 #endif
 648 #endif
 649     MMPTE TempPte;
 650     PETHREAD CurrentThread;
 651     PEPROCESS CurrentProcess;
 652     NTSTATUS Status;
 653     PMMSUPPORT WorkingSet;
 654     ULONG ProtectionCode;
 655     PMMVAD Vad;
 656     PFN_NUMBER PageFrameIndex;
 657     ULONG Color;
 658
 659     DPRINT("ARM3 FAULT AT: %p\n", Address);
 660
 661     //
 662     // Check for dispatch-level snafu
 663     //
 664     if (OldIrql > APC_LEVEL)
 665     {
 666         //
 667         // There are some special cases where this is okay, but not in ARM3 yet
 668         //
 669         DbgPrint("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
 670                  Address,
 671                  OldIrql);
 672         ASSERT(OldIrql <= APC_LEVEL);
 673     }
 674
 675     //
 676     // Check for kernel fault address
 677     //
 678     while (Address >= MmSystemRangeStart)
 679     {
 680         //
 681         // What are you even DOING here?
 682         //
 683         if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
 684
 685 #if (_MI_PAGING_LEVELS >= 3)
 686         if (
 687 #if (_MI_PAGING_LEVELS == 4)
 688             (PointerPxe->u.Hard.Valid == 0) ||
 689 #endif
 690             (PointerPpe->u.Hard.Valid == 0))
 691         {
 692             KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 693                          (ULONG_PTR)Address,
 694                          StoreInstruction,
 695                          (ULONG_PTR)TrapInformation,
 696                          2);
 697         }
 698 #endif
 699
 700         //
 701         // Is the PDE valid?
 702         //
 703         if (PointerPde->u.Hard.Valid == 0)
 704         {
 705             //
 706             // Debug spew (eww!)
 707             //
 708             DPRINT("Invalid PDE\n");
 709 #if (_MI_PAGING_LEVELS == 2)
 710             //
 711             // Handle mapping in "Special" PDE directoreis
 712             //
 713             MiCheckPdeForPagedPool(Address);
 714 #endif
 715             //
 716             // Now we SHOULD be good
 717             //
 718             if (PointerPde->u.Hard.Valid == 0)
 719             {
 720                 //
 721                 // FIXFIX: Do the S-LIST hack
 722                 //
 723
 724                 //
 725                 // Kill the system
 726                 //
 727                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 728                              (ULONG_PTR)Address,
 729                              StoreInstruction,
 730                              (ULONG_PTR)TrapInformation,
 731                              2);
 732             }
 733         }
 734
 735         //
 736         // The PDE is valid, so read the PTE
 737         //
 738         TempPte = *PointerPte;
 739         if (TempPte.u.Hard.Valid == 1)
 740         {
 741             //
 742             // Only two things can go wrong here:
 743             // Executing NX page (we couldn't care less)
 744             // Writing to a read-only page (the stuff ARM3 works with is write,
 745             // so again, moot point).
 746             //
 747
 748             //
 749             // Otherwise, the PDE was probably invalid, and all is good now
 750             //
 751             return STATUS_SUCCESS;
 752         }
 753
 754         //
 755         // Check for a fault on the page table or hyperspace itself
 756         //
 757         if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
 758         {
 759 #if (_MI_PAGING_LEVELS == 2)
 760             /* Could be paged pool access from a new process -- synchronize the page directories */
 761             if (MiCheckPdeForPagedPool(Address) == STATUS_WAIT_1)
 762             {
 763                 DPRINT1("PAGE TABLES FAULTED IN!\n");
 764                 return STATUS_SUCCESS;
 765             }
 766 #endif
 767             /* Otherwise this could be a commit of a virtual address */
 768             break;
 769         }
 770
 771         /* In this path, we are using the system working set */
 772         CurrentThread = PsGetCurrentThread();
 773         WorkingSet = &MmSystemCacheWs;
 774
 775         /* Acquire it */
 776         KeRaiseIrql(APC_LEVEL, &LockIrql);
 777         MiLockWorkingSet(CurrentThread, WorkingSet);
 778
 779         //
 780         // Re-read PTE now that the IRQL has been raised
 781         //
 782         TempPte = *PointerPte;
 783         if (TempPte.u.Hard.Valid == 1)
 784         {
 785             //
 786             // Only two things can go wrong here:
 787             // Executing NX page (we couldn't care less)
 788             // Writing to a read-only page (the stuff ARM3 works with is write,
 789             // so again, moot point.
 790             //
 791
 792             /* Release the working set */
 793             MiUnlockWorkingSet(CurrentThread, WorkingSet);
 794             KeLowerIrql(LockIrql);
 795
 796             //
 797             // Otherwise, the PDE was probably invalid, and all is good now
 798             //
 799             return STATUS_SUCCESS;
 800         }
 801
 802         /* Check one kind of prototype PTE */
 803         if (TempPte.u.Soft.Prototype)
 804         {
 805             /* Make sure protected pool is on, and that this is a pool address */
 806             if ((MmProtectFreedNonPagedPool) &&
 807                 (((Address >= MmNonPagedPoolStart) &&
 808                   (Address < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
 809                                      MmSizeOfNonPagedPoolInBytes))) ||
 810                  ((Address >= MmNonPagedPoolExpansionStart) &&
 811                   (Address < MmNonPagedPoolEnd))))
 812             {
 813                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
 814                 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL,
 815                              (ULONG_PTR)Address,
 816                              StoreInstruction,
 817                              Mode,
 818                              4);
 819             }
 820
 821             /* Get the prototype PTE! */
 822             ProtoPte = MiProtoPteToPte(&TempPte);
 823         }
 824         else
 825         {
 826             //
 827             // We don't implement transition PTEs
 828             //
 829             ASSERT(TempPte.u.Soft.Transition == 0);
 830
 831             /* Check for no-access PTE */
 832             if (TempPte.u.Soft.Protection == MM_NOACCESS)
 833             {
 834                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
 835                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 836                              (ULONG_PTR)Address,
 837                              StoreInstruction,
 838                              (ULONG_PTR)TrapInformation,
 839                              1);
 840             }
 841         }
 842
 843         /* Check for demand page */
 844         if ((StoreInstruction) && !(ProtoPte) && !(TempPte.u.Hard.Valid))
 845         {
 846             /* Get the protection code */
 847             if (!(TempPte.u.Soft.Protection & MM_READWRITE))
 848             {
 849                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
 850                 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
 851                              (ULONG_PTR)Address,
 852                              TempPte.u.Long,
 853                              (ULONG_PTR)TrapInformation,
 854                              14);
 855             }
 856         }
 857
 858         //
 859         // Now do the real fault handling
 860         //
 861         Status = MiDispatchFault(StoreInstruction,
 862                                  Address,
 863                                  PointerPte,
 864                                  ProtoPte,
 865                                  FALSE,
 866                                  NULL,
 867                                  TrapInformation,
 868                                  NULL);
 869
 870         /* Release the working set */
 871         ASSERT(KeAreAllApcsDisabled() == TRUE);
 872         MiUnlockWorkingSet(CurrentThread, WorkingSet);
 873         KeLowerIrql(LockIrql);
 874
 875         //
 876         // We are done!
 877         //
 878         DPRINT("Fault resolved with status: %lx\n", Status);
 879         return Status;
 880     }
 881
 882 #if (_MI_PAGING_LEVELS == 4)
 883     /* On these systems we have PXEs and PPEs ready for everything we need */
 884     if (PointerPxe->u.Hard.Valid == 0) return STATUS_ACCESS_VIOLATION;
 885 #endif
 886 #if (_MI_PAGING_LEVELS >= 3)
 887     if (PointerPpe->u.Hard.Valid == 0) return STATUS_ACCESS_VIOLATION;
 888 #endif
 889
 890     /* This is a user fault (<- And this is a lie!) */
 891     CurrentThread = PsGetCurrentThread();
 892     CurrentProcess = PsGetCurrentProcess();
 893
 894     /* Lock the working set */
 895     MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
 896
 897     /* First things first, is the PDE valid? */
 898     ASSERT(PointerPde->u.Hard.LargePage == 0);
 899     if (PointerPde->u.Hard.Valid == 0)
 900     {
 901         /* Right now, we only handle scenarios where the PDE is totally empty */
 902         ASSERT(PointerPde->u.Long == 0);
 903
 904         /* Check if this address range belongs to a valid allocation (VAD) */
 905         MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 906
 907         /* Right now, we expect a valid protection mask on the VAD */
 908         ASSERT(ProtectionCode != MM_NOACCESS);
 909
 910         /* Make the PDE demand-zero */
 911         MI_WRITE_INVALID_PDE(PointerPde, DemandZeroPde);
 912
 913         /* And go dispatch the fault on the PDE. This should handle the demand-zero */
 914 #if MI_TRACE_PFNS
 915         UserPdeFault = TRUE;
 916 #endif
 917         Status = MiDispatchFault(TRUE,
 918                                  PointerPte,
 919                                  (PMMPTE)PointerPde,
 920                                  NULL,
 921                                  FALSE,
 922                                  PsGetCurrentProcess(),
 923                                  TrapInformation,
 924                                  NULL);
 925 #if MI_TRACE_PFNS
 926         UserPdeFault = FALSE;
 927 #endif
 928         /* We should come back with APCs enabled, and with a valid PDE */
 929         ASSERT(KeAreAllApcsDisabled() == TRUE);
 930         ASSERT(PointerPde->u.Hard.Valid == 1);
 931     }
 932
 933     /* Now capture the PTE. Ignore virtual faults for now */
 934     TempPte = *PointerPte;
 935     ASSERT(TempPte.u.Hard.Valid == 0);
 936
 937     /* Quick check for demand-zero */
 938     if (TempPte.u.Long == (MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS))
 939     {
 940         /* Resolve the fault */
 941         MiResolveDemandZeroFault(Address,
 942                                  PointerPte,
 943                                  CurrentProcess,
 944                                  MM_NOIRQL);
 945
 946         /* Return the status */
 947         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 948         return STATUS_PAGE_FAULT_DEMAND_ZERO;
 949     }
 950
 951     /* Get protection and check if it's a prototype PTE */
 952     ProtectionCode = (ULONG)TempPte.u.Soft.Protection;
 953     ASSERT(TempPte.u.Soft.Prototype == 0);
 954
 955     /* Check for non-demand zero PTE */
 956     if (TempPte.u.Long != 0)
 957     {
 958         /* This is a page fault, check for valid protection */
 959         ASSERT(ProtectionCode != 0x100);
 960
 961         /* FIXME: Run MiAccessCheck */
 962
 963         /* Dispatch the fault */
 964         Status = MiDispatchFault(StoreInstruction,
 965                                  Address,
 966                                  PointerPte,
 967                                  NULL,
 968                                  FALSE,
 969                                  PsGetCurrentProcess(),
 970                                  TrapInformation,
 971                                  NULL);
 972
 973         /* Return the status */
 974         ASSERT(NT_SUCCESS(Status));
 975         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 976         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 977         return Status;
 978     }
 979
 980     /* Check if this address range belongs to a valid allocation (VAD) */
 981     ASSERT(TempPte.u.Long == 0);
 982     ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 983     if (ProtectionCode == MM_NOACCESS)
 984     {
 985         /* This is a bogus VA */
 986         Status = STATUS_ACCESS_VIOLATION;
 987
 988         /* Could be a not-yet-mapped paged pool page table */
 989 #if (_MI_PAGING_LEVELS == 2)
 990         MiCheckPdeForPagedPool(Address);
 991 #endif
 992         /* See if that fixed it */
 993         if (PointerPte->u.Hard.Valid == 1) Status = STATUS_SUCCESS;
 994
 995         /* Return the status */
 996         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 997         return Status;
 998     }
 999
1000     /* Is this a user address? */
1001     if (Address <= MM_HIGHEST_USER_ADDRESS)
1002     {
1003         /* Add an additional page table reference */
1004         MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]++;
1005         ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT);
1006     }
1007
1008     /* Did we get a prototype PTE back? */
1009     if (!ProtoPte)
1010     {
1011         /* No, create a new PTE. First, write the protection */
1012         PointerPte->u.Soft.Protection = ProtectionCode;
1013
1014         /* Lock the PFN database since we're going to grab a page */
1015         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1016
1017         /* Try to get a zero page */
1018         MI_SET_USAGE(MI_USAGE_PEB_TEB);
1019         MI_SET_PROCESS2(CurrentProcess->ImageFileName);
1020         Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
1021         PageFrameIndex = MiRemoveZeroPageSafe(Color);
1022         if (!PageFrameIndex)
1023         {
1024             /* Grab a page out of there. Later we should grab a colored zero page */
1025             PageFrameIndex = MiRemoveAnyPage(Color);
1026             ASSERT(PageFrameIndex);
1027
1028             /* Release the lock since we need to do some zeroing */
1029             KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1030
1031             /* Zero out the page, since it's for user-mode */
1032             MiZeroPfn(PageFrameIndex);
1033
1034             /* Grab the lock again so we can initialize the PFN entry */
1035             OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1036         }
1037
1038         /* Initialize the PFN entry now */
1039         MiInitializePfn(PageFrameIndex, PointerPte, 1);
1040
1041         /* And we're done with the lock */
1042         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1043
1044         /* One more demand-zero fault */
1045         InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
1046
1047         /* Was the fault on an actual user page, or a kernel page for the user? */
1048         if (PointerPte <= MiHighestUserPte)
1049         {
1050             /* User fault, build a user PTE */
1051             MI_MAKE_HARDWARE_PTE_USER(&TempPte,
1052                                       PointerPte,
1053                                       PointerPte->u.Soft.Protection,
1054                                       PageFrameIndex);
1055         }
1056         else
1057         {
1058             /* Session, kernel, or user PTE, figure it out and build it */
1059             MI_MAKE_HARDWARE_PTE(&TempPte,
1060                                  PointerPte,
1061                                  PointerPte->u.Soft.Protection,
1062                                  PageFrameIndex);
1063         }
1064
1065         /* Write the dirty bit for writeable pages */
1066         if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
1067
1068         /* And now write down the PTE, making the address valid */
1069         MI_WRITE_VALID_PTE(PointerPte, TempPte);
1070
1071         /* Demand zero */
1072         Status = STATUS_PAGE_FAULT_DEMAND_ZERO;
1073     }
1074     else
1075     {
1076         /* No guard page support yet */
1077         ASSERT((ProtectionCode & MM_DECOMMIT) == 0);
1078         ASSERT(ProtectionCode != 0x100);
1079
1080         /* Write the prototype PTE */
1081         TempPte = PrototypePte;
1082         TempPte.u.Soft.Protection = ProtectionCode;
1083         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
1084
1085         /* Handle the fault */
1086         Status = MiDispatchFault(StoreInstruction,
1087                                  Address,
1088                                  PointerPte,
1089                                  ProtoPte,
1090                                  FALSE,
1091                                  CurrentProcess,
1092                                  TrapInformation,
1093                                  Vad);
1094         ASSERT(Status == STATUS_PAGE_FAULT_TRANSITION);
1095         ASSERT(PointerPte->u.Hard.Valid == 1);
1096         ASSERT(PointerPte->u.Hard.PageFrameNumber != 0);
1097     }
1098
1099     /* Release the working set */
1100     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
1101     return Status;
1102 }
1103
1104 /* EOF */