ntoskrnl/mm/ARM3/pagfault.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pagfault.c
   5  * PURPOSE:         ARM Memory Manager Page Fault Handling
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include "../ARM3/miarm.h"
  17
  18 /* GLOBALS ********************************************************************/
  19
  20 #if MI_TRACE_PFNS
  21 BOOLEAN UserPdeFault = FALSE;
  22 #endif
  23
  24 /* PRIVATE FUNCTIONS **********************************************************/
  25
  26 PMMPTE
  27 NTAPI
  28 MiCheckVirtualAddress(IN PVOID VirtualAddress,
  29                       OUT PULONG ProtectCode,
  30                       OUT PMMVAD *ProtoVad)
  31 {
  32     PMMVAD Vad;
  33     PMMPTE PointerPte;
  34
  35     /* No prototype/section support for now */
  36     *ProtoVad = NULL;
  37
  38     /* Check if this is a page table address */
  39     if (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress))
  40     {
  41         /* This should never happen, as these addresses are handled by the double-maping */
  42         if (((PMMPTE)VirtualAddress >= MiAddressToPte(MmPagedPoolStart)) &&
  43             ((PMMPTE)VirtualAddress <= MmPagedPoolInfo.LastPteForPagedPool))
  44         {
  45             /* Fail such access */
  46             *ProtectCode = MM_NOACCESS;
  47             return NULL;
  48         }
  49
  50         /* Return full access rights */
  51         *ProtectCode = MM_READWRITE;
  52         return NULL;
  53     }
  54
  55     /* Should not be a session address */
  56     ASSERT(MI_IS_SESSION_ADDRESS(VirtualAddress) == FALSE);
  57
  58     /* Special case for shared data */
  59     if (PAGE_ALIGN(VirtualAddress) == (PVOID)USER_SHARED_DATA)
  60     {
  61         /* It's a read-only page */
  62         *ProtectCode = MM_READONLY;
  63         return MmSharedUserDataPte;
  64     }
  65
  66     /* Find the VAD, it might not exist if the address is bogus */
  67     Vad = MiLocateAddress(VirtualAddress);
  68     if (!Vad)
  69     {
  70         /* Bogus virtual address */
  71         *ProtectCode = MM_NOACCESS;
  72         return NULL;
  73     }
  74
  75     /* This must be a VM VAD */
  76     ASSERT(Vad->u.VadFlags.VadType == VadNone);
  77
  78     /* Check if it's a section, or just an allocation */
  79     if (Vad->u.VadFlags.PrivateMemory == TRUE)
  80     {
  81         /* This must be a TEB/PEB VAD */
  82         ASSERT(Vad->u.VadFlags.MemCommit == TRUE);
  83         *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
  84         return NULL;
  85     }
  86     else
  87     {
  88         /* Return the proto VAD */
  89         ASSERT(Vad->u2.VadFlags2.ExtendableFile == 0);
  90         *ProtoVad = Vad;
  91
  92         /* Get the prototype PTE for this page */
  93         PointerPte = (((ULONG_PTR)VirtualAddress >> PAGE_SHIFT) - Vad->StartingVpn) + Vad->FirstPrototypePte;
  94         ASSERT(PointerPte <= Vad->LastContiguousPte);
  95         ASSERT(PointerPte != NULL);
  96
  97         /* Return the Prototype PTE and the protection for the page mapping */
  98         *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
  99         return PointerPte;
 100     }
 101 }
 102
 103 NTSTATUS
 104 FASTCALL
 105 MiCheckPdeForPagedPool(IN PVOID Address)
 106 {
 107     PMMPDE PointerPde;
 108     NTSTATUS Status = STATUS_SUCCESS;
 109
 110     /* No session support in ReactOS yet */
 111     ASSERT(MI_IS_SESSION_ADDRESS(Address) == FALSE);
 112     ASSERT(MI_IS_SESSION_PTE(Address) == FALSE);
 113
 114     //
 115     // Check if this is a fault while trying to access the page table itself
 116     //
 117     if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
 118     {
 119         //
 120         // Send a hint to the page fault handler that this is only a valid fault
 121         // if we already detected this was access within the page table range
 122         //
 123         PointerPde = (PMMPDE)MiAddressToPte(Address);
 124         Status = STATUS_WAIT_1;
 125     }
 126     else if (Address < MmSystemRangeStart)
 127     {
 128         //
 129         // This is totally illegal
 130         //
 131         return STATUS_ACCESS_VIOLATION;
 132     }
 133     else
 134     {
 135         //
 136         // Get the PDE for the address
 137         //
 138         PointerPde = MiAddressToPde(Address);
 139     }
 140
 141     //
 142     // Check if it's not valid
 143     //
 144     if (PointerPde->u.Hard.Valid == 0)
 145     {
 146 #ifdef _M_AMD64
 147         ASSERT(FALSE);
 148 #else
 149         //
 150         // Copy it from our double-mapped system page directory
 151         //
 152         InterlockedExchangePte(PointerPde,
 153                                MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)].u.Long);
 154 #endif
 155     }
 156
 157     //
 158     // Return status
 159     //
 160     return Status;
 161 }
 162
 163 VOID
 164 NTAPI
 165 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
 166 {
 167     PMMPTE ZeroPte;
 168     MMPTE TempPte;
 169     PMMPFN Pfn1;
 170     PVOID ZeroAddress;
 171
 172     /* Get the PFN for this page */
 173     Pfn1 = MiGetPfnEntry(PageFrameNumber);
 174     ASSERT(Pfn1);
 175
 176     /* Grab a system PTE we can use to zero the page */
 177     ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
 178     ASSERT(ZeroPte);
 179
 180     /* Initialize the PTE for it */
 181     TempPte = ValidKernelPte;
 182     TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
 183
 184     /* Setup caching */
 185     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 186     {
 187         /* Write combining, no caching */
 188         MI_PAGE_DISABLE_CACHE(&TempPte);
 189         MI_PAGE_WRITE_COMBINED(&TempPte);
 190     }
 191     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 192     {
 193         /* Write through, no caching */
 194         MI_PAGE_DISABLE_CACHE(&TempPte);
 195         MI_PAGE_WRITE_THROUGH(&TempPte);
 196     }
 197
 198     /* Make the system PTE valid with our PFN */
 199     MI_WRITE_VALID_PTE(ZeroPte, TempPte);
 200
 201     /* Get the address it maps to, and zero it out */
 202     ZeroAddress = MiPteToAddress(ZeroPte);
 203     KeZeroPages(ZeroAddress, PAGE_SIZE);
 204
 205     /* Now get rid of it */
 206     MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
 207 }
 208
 209 NTSTATUS
 210 NTAPI
 211 MiResolveDemandZeroFault(IN PVOID Address,
 212                          IN PMMPTE PointerPte,
 213                          IN PEPROCESS Process,
 214                          IN KIRQL OldIrql)
 215 {
 216     PFN_NUMBER PageFrameNumber = 0;
 217     MMPTE TempPte;
 218     BOOLEAN NeedZero = FALSE, HaveLock = FALSE;
 219     ULONG Color;
 220     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
 221             Address,
 222             Process);
 223
 224     /* Must currently only be called by paging path */
 225     if ((Process) && (OldIrql == MM_NOIRQL))
 226     {
 227         /* Sanity check */
 228         ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
 229
 230         /* No forking yet */
 231         ASSERT(Process->ForkInProgress == NULL);
 232
 233         /* Get process color */
 234         Color = MI_GET_NEXT_PROCESS_COLOR(Process);
 235         ASSERT(Color != 0xFFFFFFFF);
 236
 237         /* We'll need a zero page */
 238         NeedZero = TRUE;
 239     }
 240     else
 241     {
 242         /* Check if we need a zero page */
 243         NeedZero = (OldIrql != MM_NOIRQL);
 244
 245         /* Get the next system page color */
 246         Color = MI_GET_NEXT_COLOR();
 247     }
 248
 249     /* Check if the PFN database should be acquired */
 250     if (OldIrql == MM_NOIRQL)
 251     {
 252         /* Acquire it and remember we should release it after */
 253         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 254         HaveLock = TRUE;
 255     }
 256
 257     /* We either manually locked the PFN DB, or already came with it locked */
 258     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 259
 260     /* Do we need a zero page? */
 261     ASSERT(PointerPte->u.Hard.Valid == 0);
 262 #if MI_TRACE_PFNS
 263     if (UserPdeFault) MI_SET_USAGE(MI_USAGE_PAGE_TABLE);
 264     if (!UserPdeFault) MI_SET_USAGE(MI_USAGE_DEMAND_ZERO);
 265 #endif
 266     if (Process) MI_SET_PROCESS2(Process->ImageFileName);
 267     if (!Process) MI_SET_PROCESS2("Kernel Demand 0");
 268     if ((NeedZero) && (Process))
 269     {
 270         /* Try to get one, if we couldn't grab a free page and zero it */
 271         PageFrameNumber = MiRemoveZeroPageSafe(Color);
 272         if (PageFrameNumber)
 273         {
 274             /* We got a genuine zero page, stop worrying about it */
 275             NeedZero = FALSE;
 276         }
 277         else
 278         {
 279             /* We'll need a free page and zero it manually */
 280             PageFrameNumber = MiRemoveAnyPage(Color);
 281         }
 282     }
 283     else if (!NeedZero)
 284     {
 285         /* Process or system doesn't want a zero page, grab anything */
 286         PageFrameNumber = MiRemoveAnyPage(Color);
 287     }
 288     else
 289     {
 290         /* System wants a zero page, obtain one */
 291         PageFrameNumber = MiRemoveZeroPage(Color);
 292     }
 293
 294     /* Initialize it */
 295     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
 296
 297     /* Release PFN lock if needed */
 298     if (HaveLock) KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 299
 300     //
 301     // Increment demand zero faults
 302     //
 303     InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
 304
 305     /* Zero the page if need be */
 306     if (NeedZero) MiZeroPfn(PageFrameNumber);
 307
 308     /* Build the PTE */
 309     if (PointerPte <= MiHighestUserPte)
 310     {
 311         /* For user mode */
 312         MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 313                                   PointerPte,
 314                                   PointerPte->u.Soft.Protection,
 315                                   PageFrameNumber);
 316     }
 317     else
 318     {
 319         /* For kernel mode */
 320         MI_MAKE_HARDWARE_PTE(&TempPte,
 321                              PointerPte,
 322                              PointerPte->u.Soft.Protection,
 323                              PageFrameNumber);
 324     }
 325
 326     /* Set it dirty if it's a writable page */
 327     if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
 328
 329     /* Write it */
 330     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 331
 332     //
 333     // It's all good now
 334     //
 335     DPRINT("Paged pool page has now been paged in\n");
 336     return STATUS_PAGE_FAULT_DEMAND_ZERO;
 337 }
 338
 339 NTSTATUS
 340 NTAPI
 341 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
 342                         IN PVOID Address,
 343                         IN PMMPTE PointerPte,
 344                         IN PMMPTE PointerProtoPte,
 345                         IN KIRQL OldIrql,
 346                         IN PMMPFN Pfn1)
 347 {
 348     MMPTE TempPte;
 349     PMMPTE OriginalPte;
 350     ULONG_PTR Protection;
 351     PFN_NUMBER PageFrameIndex;
 352
 353     /* Must be called with an valid prototype PTE, with the PFN lock held */
 354     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 355     ASSERT(PointerProtoPte->u.Hard.Valid == 1);
 356
 357     /* Get the page */
 358     PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
 359
 360     /* Get the PFN entry and set it as a prototype PTE */
 361     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 362     Pfn1->u3.e1.PrototypePte = 1;
 363
 364     /* FIXME: Increment the share count for the page table */
 365
 366     /* Check where we should be getting the protection information from */
 367     if (PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
 368     {
 369         /* Get the protection from the PTE, there's no real Proto PTE data */
 370         Protection = PointerPte->u.Soft.Protection;
 371     }
 372     else
 373     {
 374         /* Get the protection from the original PTE link */
 375         OriginalPte = &Pfn1->OriginalPte;
 376         Protection = OriginalPte->u.Soft.Protection;
 377     }
 378
 379     /* Release the PFN lock */
 380     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 381
 382     /* Remove caching bits */
 383     Protection &= ~(MM_NOCACHE | MM_NOACCESS);
 384
 385     /* Check if this is a kernel or user address */
 386     if (Address < MmSystemRangeStart)
 387     {
 388         /* Build the user PTE */
 389         MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, Protection, PageFrameIndex);
 390     }
 391     else
 392     {
 393         /* Build the kernel PTE */
 394         MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, Protection, PageFrameIndex);
 395     }
 396
 397     /* Write the PTE */
 398     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 399
 400     /* Return success */
 401     return STATUS_SUCCESS;
 402 }
 403
 404 NTSTATUS
 405 NTAPI
 406 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
 407                        IN PVOID Address,
 408                        IN PMMPTE PointerPte,
 409                        IN PMMPTE PointerProtoPte,
 410                        IN OUT PMMPFN *OutPfn,
 411                        OUT PVOID *PageFileData,
 412                        OUT PMMPTE PteValue,
 413                        IN PEPROCESS Process,
 414                        IN KIRQL OldIrql,
 415                        IN PVOID TrapInformation)
 416 {
 417     MMPTE TempPte;
 418     PMMPFN Pfn1;
 419     PFN_NUMBER PageFrameIndex;
 420     NTSTATUS Status;
 421
 422     /* Must be called with an invalid, prototype PTE, with the PFN lock held */
 423     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 424     ASSERT(PointerPte->u.Hard.Valid == 0);
 425     ASSERT(PointerPte->u.Soft.Prototype == 1);
 426
 427     /* Read the prototype PTE and check if it's valid */
 428     TempPte = *PointerProtoPte;
 429     if (TempPte.u.Hard.Valid == 1)
 430     {
 431         /* One more user of this mapped page */
 432         PageFrameIndex = PFN_FROM_PTE(&TempPte);
 433         Pfn1 = MiGetPfnEntry(PageFrameIndex);
 434         Pfn1->u2.ShareCount++;
 435
 436         /* Call it a transition */
 437         InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
 438
 439         /* Complete the prototype PTE fault -- this will release the PFN lock */
 440         return MiCompleteProtoPteFault(StoreInstruction,
 441                                        Address,
 442                                        PointerPte,
 443                                        PointerProtoPte,
 444                                        OldIrql,
 445                                        NULL);
 446     }
 447
 448     /* Make sure there's some protection mask */
 449     if (TempPte.u.Long == 0)
 450     {
 451         /* Release the lock */
 452         DPRINT1("Access on reserved section?\n");
 453         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 454         return STATUS_ACCESS_VIOLATION;
 455     }
 456
 457     /* This is the only thing we support right now */
 458     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 459     ASSERT(TempPte.u.Proto.ReadOnly == 0);
 460     ASSERT(PointerPte > MiHighestUserPte);
 461     ASSERT(TempPte.u.Soft.Prototype == 0);
 462     ASSERT(TempPte.u.Soft.Transition == 0);
 463
 464     /* Resolve the demand zero fault */
 465     Status = MiResolveDemandZeroFault(Address, PointerProtoPte, Process, OldIrql);
 466     ASSERT(NT_SUCCESS(Status));
 467
 468     /* Complete the prototype PTE fault -- this will release the PFN lock */
 469     ASSERT(PointerPte->u.Hard.Valid == 0);
 470     return MiCompleteProtoPteFault(StoreInstruction,
 471                                    Address,
 472                                    PointerPte,
 473                                    PointerProtoPte,
 474                                    OldIrql,
 475                                    NULL);
 476 }
 477
 478 NTSTATUS
 479 NTAPI
 480 MiDispatchFault(IN BOOLEAN StoreInstruction,
 481                 IN PVOID Address,
 482                 IN PMMPTE PointerPte,
 483                 IN PMMPTE PointerProtoPte,
 484                 IN BOOLEAN Recursive,
 485                 IN PEPROCESS Process,
 486                 IN PVOID TrapInformation,
 487                 IN PVOID Vad)
 488 {
 489     MMPTE TempPte;
 490     KIRQL OldIrql, LockIrql;
 491     NTSTATUS Status;
 492     PMMPTE SuperProtoPte;
 493     DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
 494              Address,
 495              Process);
 496
 497     //
 498     // Make sure APCs are off and we're not at dispatch
 499     //
 500     OldIrql = KeGetCurrentIrql();
 501     ASSERT(OldIrql <= APC_LEVEL);
 502     ASSERT(KeAreAllApcsDisabled() == TRUE);
 503
 504     //
 505     // Grab a copy of the PTE
 506     //
 507     TempPte = *PointerPte;
 508
 509     /* Do we have a prototype PTE? */
 510     if (PointerProtoPte)
 511     {
 512         /* This should never happen */
 513         ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
 514
 515         /* Check if this is a kernel-mode address */
 516         SuperProtoPte = MiAddressToPte(PointerProtoPte);
 517         if (Address >= MmSystemRangeStart)
 518         {
 519             /* Lock the PFN database */
 520             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 521
 522             /* Has the PTE been made valid yet? */
 523             if (!SuperProtoPte->u.Hard.Valid)
 524             {
 525                 UNIMPLEMENTED;
 526                 while (TRUE);
 527             }
 528             else
 529             {
 530                 /* Resolve the fault -- this will release the PFN lock */
 531                 ASSERT(PointerPte->u.Hard.Valid == 0);
 532                 Status = MiResolveProtoPteFault(StoreInstruction,
 533                                                 Address,
 534                                                 PointerPte,
 535                                                 PointerProtoPte,
 536                                                 NULL,
 537                                                 NULL,
 538                                                 NULL,
 539                                                 Process,
 540                                                 LockIrql,
 541                                                 TrapInformation);
 542                 ASSERT(Status == STATUS_SUCCESS);
 543
 544                 /* Complete this as a transition fault */
 545                 ASSERT(OldIrql == KeGetCurrentIrql());
 546                 ASSERT(OldIrql <= APC_LEVEL);
 547                 ASSERT(KeAreAllApcsDisabled() == TRUE);
 548                 return Status;
 549             }
 550         }
 551         else
 552         {
 553             /* We currently only handle very limited paths */
 554             ASSERT(PointerPte->u.Soft.Prototype == 1);
 555             ASSERT(PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED);
 556
 557             /* Lock the PFN database */
 558             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 559
 560             /* For our current usage, this should be true */
 561             ASSERT(SuperProtoPte->u.Hard.Valid == 1);
 562             ASSERT(TempPte.u.Hard.Valid == 0);
 563
 564             /* Resolve the fault -- this will release the PFN lock */
 565             Status = MiResolveProtoPteFault(StoreInstruction,
 566                                             Address,
 567                                             PointerPte,
 568                                             PointerProtoPte,
 569                                             NULL,
 570                                             NULL,
 571                                             NULL,
 572                                             Process,
 573                                             LockIrql,
 574                                             TrapInformation);
 575             ASSERT(Status == STATUS_SUCCESS);
 576
 577             /* Complete this as a transition fault */
 578             ASSERT(OldIrql == KeGetCurrentIrql());
 579             ASSERT(OldIrql <= APC_LEVEL);
 580             ASSERT(KeAreAllApcsDisabled() == TRUE);
 581             return STATUS_PAGE_FAULT_TRANSITION;
 582         }
 583     }
 584
 585     //
 586     // The PTE must be invalid
 587     //
 588     ASSERT(TempPte.u.Hard.Valid == 0);
 589
 590     /* Check if the PTE is completely empty */
 591     if (TempPte.u.Long == 0)
 592     {
 593         KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 594                      (ULONG_PTR)Address,
 595                      StoreInstruction,
 596                      (ULONG_PTR)TrapInformation,
 597                      2);
 598     }
 599
 600     //
 601     // No prototype, transition or page file software PTEs in ARM3 yet
 602     //
 603     ASSERT(TempPte.u.Soft.Prototype == 0);
 604     ASSERT(TempPte.u.Soft.Transition == 0);
 605     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 606
 607     //
 608     // If we got this far, the PTE can only be a demand zero PTE, which is what
 609     // we want. Go handle it!
 610     //
 611     Status = MiResolveDemandZeroFault(Address,
 612                                       PointerPte,
 613                                       Process,
 614                                       MM_NOIRQL);
 615     ASSERT(KeAreAllApcsDisabled() == TRUE);
 616     if (NT_SUCCESS(Status))
 617     {
 618         //
 619         // Make sure we're returning in a sane state and pass the status down
 620         //
 621         ASSERT(OldIrql == KeGetCurrentIrql());
 622         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 623         return Status;
 624     }
 625
 626     //
 627     // Generate an access fault
 628     //
 629     return STATUS_ACCESS_VIOLATION;
 630 }
 631
 632 NTSTATUS
 633 NTAPI
 634 MmArmAccessFault(IN BOOLEAN StoreInstruction,
 635                  IN PVOID Address,
 636                  IN KPROCESSOR_MODE Mode,
 637                  IN PVOID TrapInformation)
 638 {
 639     KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
 640     PMMPTE PointerPte, ProtoPte = NULL;
 641     PMMPDE PointerPde;
 642     MMPTE TempPte;
 643     PETHREAD CurrentThread;
 644     PEPROCESS CurrentProcess;
 645     NTSTATUS Status;
 646     PMMSUPPORT WorkingSet;
 647     ULONG ProtectionCode;
 648     PMMVAD Vad;
 649     PFN_NUMBER PageFrameIndex;
 650     ULONG Color;
 651     DPRINT("ARM3 FAULT AT: %p\n", Address);
 652
 653     //
 654     // Get the PTE and PDE
 655     //
 656     PointerPte = MiAddressToPte(Address);
 657     PointerPde = MiAddressToPde(Address);
 658 #if (_MI_PAGING_LEVELS >= 3)
 659     /* We need the PPE and PXE addresses */
 660     ASSERT(FALSE);
 661 #endif
 662
 663     //
 664     // Check for dispatch-level snafu
 665     //
 666     if (OldIrql > APC_LEVEL)
 667     {
 668         //
 669         // There are some special cases where this is okay, but not in ARM3 yet
 670         //
 671         DbgPrint("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
 672                  Address,
 673                  OldIrql);
 674         ASSERT(OldIrql <= APC_LEVEL);
 675     }
 676
 677     //
 678     // Check for kernel fault
 679     //
 680     while (Address >= MmSystemRangeStart)
 681     {
 682         //
 683         // What are you even DOING here?
 684         //
 685         if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
 686
 687 #if (_MI_PAGING_LEVELS >= 3)
 688         /* Need to check PXE and PDE validity */
 689         ASSERT(FALSE);
 690 #endif
 691
 692         //
 693         // Is the PDE valid?
 694         //
 695         if (PointerPde->u.Hard.Valid == 0)
 696         {
 697             //
 698             // Debug spew (eww!)
 699             //
 700             DPRINT("Invalid PDE\n");
 701 #if (_MI_PAGING_LEVELS == 2)
 702             //
 703             // Handle mapping in "Special" PDE directoreis
 704             //
 705             MiCheckPdeForPagedPool(Address);
 706 #endif
 707             //
 708             // Now we SHOULD be good
 709             //
 710             if (PointerPde->u.Hard.Valid == 0)
 711             {
 712                 //
 713                 // FIXFIX: Do the S-LIST hack
 714                 //
 715
 716                 //
 717                 // Kill the system
 718                 //
 719                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 720                              (ULONG_PTR)Address,
 721                              StoreInstruction,
 722                              (ULONG_PTR)TrapInformation,
 723                              2);
 724             }
 725         }
 726
 727         //
 728         // The PDE is valid, so read the PTE
 729         //
 730         TempPte = *PointerPte;
 731         if (TempPte.u.Hard.Valid == 1)
 732         {
 733             //
 734             // Only two things can go wrong here:
 735             // Executing NX page (we couldn't care less)
 736             // Writing to a read-only page (the stuff ARM3 works with is write,
 737             // so again, moot point).
 738             //
 739
 740             //
 741             // Otherwise, the PDE was probably invalid, and all is good now
 742             //
 743             return STATUS_SUCCESS;
 744         }
 745
 746         //
 747         // Check for a fault on the page table or hyperspace itself
 748         //
 749         if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
 750         {
 751 #if (_MI_PAGING_LEVELS == 2)
 752             /* Could be paged pool access from a new process -- synchronize the page directories */
 753             if (MiCheckPdeForPagedPool(Address) == STATUS_WAIT_1)
 754             {
 755                 DPRINT1("PAGE TABLES FAULTED IN!\n");
 756                 return STATUS_SUCCESS;
 757             }
 758 #endif
 759             /* Otherwise this could be a commit of a virtual address */
 760             break;
 761         }
 762
 763         /* In this path, we are using the system working set */
 764         CurrentThread = PsGetCurrentThread();
 765         WorkingSet = &MmSystemCacheWs;
 766
 767         /* Acquire it */
 768         KeRaiseIrql(APC_LEVEL, &LockIrql);
 769         MiLockWorkingSet(CurrentThread, WorkingSet);
 770
 771         //
 772         // Re-read PTE now that the IRQL has been raised
 773         //
 774         TempPte = *PointerPte;
 775         if (TempPte.u.Hard.Valid == 1)
 776         {
 777             //
 778             // Only two things can go wrong here:
 779             // Executing NX page (we couldn't care less)
 780             // Writing to a read-only page (the stuff ARM3 works with is write,
 781             // so again, moot point.
 782             //
 783
 784             /* Release the working set */
 785             MiUnlockWorkingSet(CurrentThread, WorkingSet);
 786             KeLowerIrql(LockIrql);
 787
 788             //
 789             // Otherwise, the PDE was probably invalid, and all is good now
 790             //
 791             return STATUS_SUCCESS;
 792         }
 793
 794         /* Check one kind of prototype PTE */
 795         if (TempPte.u.Soft.Prototype)
 796         {
 797             /* Make sure protected pool is on, and that this is a pool address */
 798             if ((MmProtectFreedNonPagedPool) &&
 799                 (((Address >= MmNonPagedPoolStart) &&
 800                   (Address < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
 801                                      MmSizeOfNonPagedPoolInBytes))) ||
 802                  ((Address >= MmNonPagedPoolExpansionStart) &&
 803                   (Address < MmNonPagedPoolEnd))))
 804             {
 805                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
 806                 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL,
 807                              (ULONG_PTR)Address,
 808                              StoreInstruction,
 809                              Mode,
 810                              4);
 811             }
 812
 813             /* Get the prototype PTE! */
 814             ProtoPte = MiProtoPteToPte(&TempPte);
 815         }
 816         else
 817         {
 818             //
 819             // We don't implement transition PTEs
 820             //
 821             ASSERT(TempPte.u.Soft.Transition == 0);
 822
 823             /* Check for no-access PTE */
 824             if (TempPte.u.Soft.Protection == MM_NOACCESS)
 825             {
 826                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
 827                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 828                              (ULONG_PTR)Address,
 829                              StoreInstruction,
 830                              (ULONG_PTR)TrapInformation,
 831                              1);
 832             }
 833         }
 834
 835         /* Check for demand page */
 836         if ((StoreInstruction) && !(ProtoPte) && !(TempPte.u.Hard.Valid))
 837         {
 838             /* Get the protection code */
 839             if (!(TempPte.u.Soft.Protection & MM_READWRITE))
 840             {
 841                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
 842                 KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
 843                              (ULONG_PTR)Address,
 844                              TempPte.u.Long,
 845                              (ULONG_PTR)TrapInformation,
 846                              14);
 847             }
 848         }
 849
 850         //
 851         // Now do the real fault handling
 852         //
 853         Status = MiDispatchFault(StoreInstruction,
 854                                  Address,
 855                                  PointerPte,
 856                                  ProtoPte,
 857                                  FALSE,
 858                                  NULL,
 859                                  TrapInformation,
 860                                  NULL);
 861
 862         /* Release the working set */
 863         ASSERT(KeAreAllApcsDisabled() == TRUE);
 864         MiUnlockWorkingSet(CurrentThread, WorkingSet);
 865         KeLowerIrql(LockIrql);
 866
 867         //
 868         // We are done!
 869         //
 870         DPRINT("Fault resolved with status: %lx\n", Status);
 871         return Status;
 872     }
 873
 874     /* This is a user fault */
 875     CurrentThread = PsGetCurrentThread();
 876     CurrentProcess = PsGetCurrentProcess();
 877
 878     /* Lock the working set */
 879     MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
 880
 881 #if (_MI_PAGING_LEVELS >= 3)
 882     /* Need to check/handle PPE and PXE validity too */
 883     ASSERT(FALSE);
 884 #endif
 885
 886     /* First things first, is the PDE valid? */
 887     ASSERT(PointerPde->u.Hard.LargePage == 0);
 888     if (PointerPde->u.Hard.Valid == 0)
 889     {
 890         /* Right now, we only handle scenarios where the PDE is totally empty */
 891         ASSERT(PointerPde->u.Long == 0);
 892
 893         /* Check if this address range belongs to a valid allocation (VAD) */
 894         MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 895
 896         /* Right now, we expect a valid protection mask on the VAD */
 897         ASSERT(ProtectionCode != MM_NOACCESS);
 898
 899         /* Make the PDE demand-zero */
 900         MI_WRITE_INVALID_PDE(PointerPde, DemandZeroPde);
 901
 902         /* And go dispatch the fault on the PDE. This should handle the demand-zero */
 903 #if MI_TRACE_PFNS
 904         UserPdeFault = TRUE;
 905 #endif
 906         Status = MiDispatchFault(TRUE,
 907                                  PointerPte,
 908                                  (PMMPTE)PointerPde,
 909                                  NULL,
 910                                  FALSE,
 911                                  PsGetCurrentProcess(),
 912                                  TrapInformation,
 913                                  NULL);
 914 #if MI_TRACE_PFNS
 915         UserPdeFault = FALSE;
 916 #endif
 917         /* We should come back with APCs enabled, and with a valid PDE */
 918         ASSERT(KeAreAllApcsDisabled() == TRUE);
 919 #if (_MI_PAGING_LEVELS >= 3)
 920         /* Need to check/handle PPE and PXE validity too */
 921         ASSERT(FALSE);
 922 #endif
 923         ASSERT(PointerPde->u.Hard.Valid == 1);
 924     }
 925
 926     /* Now capture the PTE. Ignore virtual faults for now */
 927     TempPte = *PointerPte;
 928     ASSERT(TempPte.u.Hard.Valid == 0);
 929
 930     /* Quick check for demand-zero */
 931     if (TempPte.u.Long == (MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS))
 932     {
 933         /* Resolve the fault */
 934         MiResolveDemandZeroFault(Address,
 935                                  PointerPte,
 936                                  CurrentProcess,
 937                                  MM_NOIRQL);
 938
 939         /* Return the status */
 940         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 941         return STATUS_PAGE_FAULT_DEMAND_ZERO;
 942     }
 943
 944     /* Get protection and check if it's a prototype PTE */
 945     ProtectionCode = (ULONG)TempPte.u.Soft.Protection;
 946     ASSERT(TempPte.u.Soft.Prototype == 0);
 947
 948     /* Check for non-demand zero PTE */
 949     if (TempPte.u.Long != 0)
 950     {
 951         /* This is a page fault, check for valid protection */
 952         ASSERT(ProtectionCode != 0x100);
 953
 954         /* FIXME: Run MiAccessCheck */
 955
 956         /* Dispatch the fault */
 957         Status = MiDispatchFault(StoreInstruction,
 958                                  Address,
 959                                  PointerPte,
 960                                  NULL,
 961                                  FALSE,
 962                                  PsGetCurrentProcess(),
 963                                  TrapInformation,
 964                                  NULL);
 965
 966         /* Return the status */
 967         ASSERT(NT_SUCCESS(Status));
 968         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 969         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 970         return Status;
 971     }
 972
 973     /* Check if this address range belongs to a valid allocation (VAD) */
 974     ASSERT(TempPte.u.Long == 0);
 975     ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 976     if (ProtectionCode == MM_NOACCESS)
 977     {
 978         /* This is a bogus VA */
 979         Status = STATUS_ACCESS_VIOLATION;
 980
 981         /* Could be a not-yet-mapped paged pool page table */
 982 #if (_MI_PAGING_LEVELS == 2)
 983         MiCheckPdeForPagedPool(Address);
 984 #endif
 985         /* See if that fixed it */
 986         if (PointerPte->u.Hard.Valid == 1) Status = STATUS_SUCCESS;
 987
 988         /* Return the status */
 989         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 990         return Status;
 991     }
 992
 993     /* Is this a user address? */
 994     if (Address <= MM_HIGHEST_USER_ADDRESS)
 995     {
 996         /* Add an additional page table reference */
 997         MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]++;
 998         ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT);
 999     }
1000
1001     /* Did we get a prototype PTE back? */
1002     if (!ProtoPte)
1003     {
1004         /* No, create a new PTE. First, write the protection */
1005         PointerPte->u.Soft.Protection = ProtectionCode;
1006
1007         /* Lock the PFN database since we're going to grab a page */
1008         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1009
1010         /* Try to get a zero page */
1011         MI_SET_USAGE(MI_USAGE_PEB_TEB);
1012         MI_SET_PROCESS2(CurrentProcess->ImageFileName);
1013         Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
1014         PageFrameIndex = MiRemoveZeroPageSafe(Color);
1015         if (!PageFrameIndex)
1016         {
1017             /* Grab a page out of there. Later we should grab a colored zero page */
1018             PageFrameIndex = MiRemoveAnyPage(Color);
1019             ASSERT(PageFrameIndex);
1020
1021             /* Release the lock since we need to do some zeroing */
1022             KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1023
1024             /* Zero out the page, since it's for user-mode */
1025             MiZeroPfn(PageFrameIndex);
1026
1027             /* Grab the lock again so we can initialize the PFN entry */
1028             OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1029         }
1030
1031         /* Initialize the PFN entry now */
1032         MiInitializePfn(PageFrameIndex, PointerPte, 1);
1033
1034         /* And we're done with the lock */
1035         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1036
1037         /* One more demand-zero fault */
1038         InterlockedIncrement(&KeGetCurrentPrcb()->MmDemandZeroCount);
1039
1040         /* Was the fault on an actual user page, or a kernel page for the user? */
1041         if (PointerPte <= MiHighestUserPte)
1042         {
1043             /* User fault, build a user PTE */
1044             MI_MAKE_HARDWARE_PTE_USER(&TempPte,
1045                                       PointerPte,
1046                                       PointerPte->u.Soft.Protection,
1047                                       PageFrameIndex);
1048         }
1049         else
1050         {
1051             /* Session, kernel, or user PTE, figure it out and build it */
1052             MI_MAKE_HARDWARE_PTE(&TempPte,
1053                                  PointerPte,
1054                                  PointerPte->u.Soft.Protection,
1055                                  PageFrameIndex);
1056         }
1057
1058         /* Write the dirty bit for writeable pages */
1059         if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
1060
1061         /* And now write down the PTE, making the address valid */
1062         MI_WRITE_VALID_PTE(PointerPte, TempPte);
1063
1064         /* Demand zero */
1065         Status = STATUS_PAGE_FAULT_DEMAND_ZERO;
1066     }
1067     else
1068     {
1069         /* No guard page support yet */
1070         ASSERT((ProtectionCode & MM_DECOMMIT) == 0);
1071         ASSERT(ProtectionCode != 0x100);
1072
1073         /* Write the prototype PTE */
1074         TempPte = PrototypePte;
1075         TempPte.u.Soft.Protection = ProtectionCode;
1076         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
1077
1078         /* Handle the fault */
1079         Status = MiDispatchFault(StoreInstruction,
1080                                  Address,
1081                                  PointerPte,
1082                                  ProtoPte,
1083                                  FALSE,
1084                                  CurrentProcess,
1085                                  TrapInformation,
1086                                  Vad);
1087         ASSERT(Status == STATUS_PAGE_FAULT_TRANSITION);
1088         ASSERT(PointerPte->u.Hard.Valid == 1);
1089         ASSERT(PointerPte->u.Hard.PageFrameNumber != 0);
1090     }
1091
1092     /* Release the working set */
1093     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
1094     return Status;
1095 }
1096
1097 /* EOF */