reactos/ntoskrnl/mm/ARM3/pagfault.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pagfault.c
   5  * PURPOSE:         ARM Memory Manager Page Fault Handling
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include "../ARM3/miarm.h"
  17
  18 /* GLOBALS ********************************************************************/
  19
  20 #if MI_TRACE_PFNS
  21 BOOLEAN UserPdeFault = FALSE;
  22 #endif
  23
  24 /* PRIVATE FUNCTIONS **********************************************************/
  25
  26 PMMPTE
  27 NTAPI
  28 MiCheckVirtualAddress(IN PVOID VirtualAddress,
  29                       OUT PULONG ProtectCode,
  30                       OUT PMMVAD *ProtoVad)
  31 {
  32     PMMVAD Vad;
  33     PMMPTE PointerPte;
  34
  35     /* No prototype/section support for now */
  36     *ProtoVad = NULL;
  37
  38     /* Check if this is a page table address */
  39     if (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress))
  40     {
  41         /* This should never happen, as these addresses are handled by the double-maping */
  42         if (((PMMPTE)VirtualAddress >= MiAddressToPte(MmPagedPoolStart)) &&
  43             ((PMMPTE)VirtualAddress <= MmPagedPoolInfo.LastPteForPagedPool))
  44         {
  45             /* Fail such access */
  46             *ProtectCode = MM_NOACCESS;
  47             return NULL;
  48         }
  49
  50         /* Return full access rights */
  51         *ProtectCode = MM_READWRITE;
  52         return NULL;
  53     }
  54
  55     /* Should not be a session address */
  56     ASSERT(MI_IS_SESSION_ADDRESS(VirtualAddress) == FALSE);
  57
  58     /* Special case for shared data */
  59     if (PAGE_ALIGN(VirtualAddress) == (PVOID)MM_SHARED_USER_DATA_VA)
  60     {
  61         /* It's a read-only page */
  62         *ProtectCode = MM_READONLY;
  63         return MmSharedUserDataPte;
  64     }
  65
  66     /* Find the VAD, it might not exist if the address is bogus */
  67     Vad = MiLocateAddress(VirtualAddress);
  68     if (!Vad)
  69     {
  70         /* Bogus virtual address */
  71         *ProtectCode = MM_NOACCESS;
  72         return NULL;
  73     }
  74
  75     /* This must be a VM VAD */
  76     ASSERT(Vad->u.VadFlags.VadType == VadNone);
  77
  78     /* Check if it's a section, or just an allocation */
  79     if (Vad->u.VadFlags.PrivateMemory == TRUE)
  80     {
  81         /* This must be a TEB/PEB VAD */
  82         ASSERT(Vad->u.VadFlags.MemCommit == TRUE);
  83         *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
  84         return NULL;
  85     }
  86     else
  87     {
  88         /* Return the proto VAD */
  89         ASSERT(Vad->u2.VadFlags2.ExtendableFile == 0);
  90         *ProtoVad = Vad;
  91
  92         /* Get the prototype PTE for this page */
  93         PointerPte = (((ULONG_PTR)VirtualAddress >> PAGE_SHIFT) - Vad->StartingVpn) + Vad->FirstPrototypePte;
  94         ASSERT(PointerPte <= Vad->LastContiguousPte);
  95         ASSERT(PointerPte != NULL);
  96
  97         /* Return the Prototype PTE and the protection for the page mapping */
  98         *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
  99         return PointerPte;
 100     }
 101 }
 102
 103 #if (_MI_PAGING_LEVELS == 2)
 104 BOOLEAN
 105 FORCEINLINE
 106 MiSynchronizeSystemPde(PMMPDE PointerPde)
 107 {
 108     MMPDE SystemPde;
 109     ULONG Index;
 110
 111     /* Get the Index from the PDE */
 112     Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
 113
 114     /* Copy the PDE from the double-mapped system page directory */
 115     SystemPde = MmSystemPagePtes[Index];
 116     *PointerPde = SystemPde;
 117
 118     /* Make sure we re-read the PDE and PTE */
 119     KeMemoryBarrierWithoutFence();
 120
 121     /* Return, if we had success */
 122     return (BOOLEAN)SystemPde.u.Hard.Valid;
 123 }
 124
 125 NTSTATUS
 126 FASTCALL
 127 MiCheckPdeForPagedPool(IN PVOID Address)
 128 {
 129     PMMPDE PointerPde;
 130     NTSTATUS Status = STATUS_SUCCESS;
 131
 132     /* No session support in ReactOS yet */
 133     ASSERT(MI_IS_SESSION_ADDRESS(Address) == FALSE);
 134     ASSERT(MI_IS_SESSION_PTE(Address) == FALSE);
 135
 136     //
 137     // Check if this is a fault while trying to access the page table itself
 138     //
 139     if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
 140     {
 141         //
 142         // Send a hint to the page fault handler that this is only a valid fault
 143         // if we already detected this was access within the page table range
 144         //
 145         PointerPde = (PMMPDE)MiAddressToPte(Address);
 146         Status = STATUS_WAIT_1;
 147     }
 148     else if (Address < MmSystemRangeStart)
 149     {
 150         //
 151         // This is totally illegal
 152         //
 153         return STATUS_ACCESS_VIOLATION;
 154     }
 155     else
 156     {
 157         //
 158         // Get the PDE for the address
 159         //
 160         PointerPde = MiAddressToPde(Address);
 161     }
 162
 163     //
 164     // Check if it's not valid
 165     //
 166     if (PointerPde->u.Hard.Valid == 0)
 167     {
 168 #ifdef _M_AMD64
 169         ASSERT(FALSE);
 170 #else
 171         //
 172         // Copy it from our double-mapped system page directory
 173         //
 174         InterlockedExchangePte(PointerPde,
 175                                MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)].u.Long);
 176 #endif
 177     }
 178
 179     //
 180     // Return status
 181     //
 182     return Status;
 183 }
 184 #endif
 185
 186 VOID
 187 NTAPI
 188 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
 189 {
 190     PMMPTE ZeroPte;
 191     MMPTE TempPte;
 192     PMMPFN Pfn1;
 193     PVOID ZeroAddress;
 194
 195     /* Get the PFN for this page */
 196     Pfn1 = MiGetPfnEntry(PageFrameNumber);
 197     ASSERT(Pfn1);
 198
 199     /* Grab a system PTE we can use to zero the page */
 200     ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
 201     ASSERT(ZeroPte);
 202
 203     /* Initialize the PTE for it */
 204     TempPte = ValidKernelPte;
 205     TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
 206
 207     /* Setup caching */
 208     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 209     {
 210         /* Write combining, no caching */
 211         MI_PAGE_DISABLE_CACHE(&TempPte);
 212         MI_PAGE_WRITE_COMBINED(&TempPte);
 213     }
 214     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 215     {
 216         /* Write through, no caching */
 217         MI_PAGE_DISABLE_CACHE(&TempPte);
 218         MI_PAGE_WRITE_THROUGH(&TempPte);
 219     }
 220
 221     /* Make the system PTE valid with our PFN */
 222     MI_WRITE_VALID_PTE(ZeroPte, TempPte);
 223
 224     /* Get the address it maps to, and zero it out */
 225     ZeroAddress = MiPteToAddress(ZeroPte);
 226     KeZeroPages(ZeroAddress, PAGE_SIZE);
 227
 228     /* Now get rid of it */
 229     MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
 230 }
 231
 232 NTSTATUS
 233 NTAPI
 234 MiResolveDemandZeroFault(IN PVOID Address,
 235                          IN ULONG Protection,
 236                          IN PEPROCESS Process,
 237                          IN KIRQL OldIrql)
 238 {
 239     PFN_NUMBER PageFrameNumber = 0;
 240     PMMPTE PointerPte = MiAddressToPte(Address);
 241     MMPTE TempPte;
 242     BOOLEAN NeedZero = FALSE, HaveLock = FALSE;
 243     ULONG Color;
 244     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
 245             Address,
 246             Process);
 247
 248     /* Must currently only be called by paging path */
 249     if ((Process) && (OldIrql == MM_NOIRQL))
 250     {
 251         /* Sanity check */
 252         ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
 253
 254         /* No forking yet */
 255         ASSERT(Process->ForkInProgress == NULL);
 256
 257         /* Get process color */
 258         Color = MI_GET_NEXT_PROCESS_COLOR(Process);
 259         ASSERT(Color != 0xFFFFFFFF);
 260
 261         /* We'll need a zero page */
 262         NeedZero = TRUE;
 263     }
 264     else
 265     {
 266         /* Check if we need a zero page */
 267         NeedZero = (OldIrql != MM_NOIRQL);
 268
 269         /* Get the next system page color */
 270         Color = MI_GET_NEXT_COLOR();
 271     }
 272
 273     /* Check if the PFN database should be acquired */
 274     if (OldIrql == MM_NOIRQL)
 275     {
 276         /* Acquire it and remember we should release it after */
 277         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 278         HaveLock = TRUE;
 279     }
 280
 281     /* We either manually locked the PFN DB, or already came with it locked */
 282     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 283
 284     /* Do we need a zero page? */
 285     ASSERT(PointerPte->u.Hard.Valid == 0);
 286 #if MI_TRACE_PFNS
 287     if (UserPdeFault) MI_SET_USAGE(MI_USAGE_PAGE_TABLE);
 288     if (!UserPdeFault) MI_SET_USAGE(MI_USAGE_DEMAND_ZERO);
 289 #endif
 290     if (Process) MI_SET_PROCESS2(Process->ImageFileName);
 291     if (!Process) MI_SET_PROCESS2("Kernel Demand 0");
 292     if ((NeedZero) && (Process))
 293     {
 294         /* Try to get one, if we couldn't grab a free page and zero it */
 295         PageFrameNumber = MiRemoveZeroPageSafe(Color);
 296         if (PageFrameNumber)
 297         {
 298             /* We got a genuine zero page, stop worrying about it */
 299             NeedZero = FALSE;
 300         }
 301         else
 302         {
 303             /* We'll need a free page and zero it manually */
 304             PageFrameNumber = MiRemoveAnyPage(Color);
 305         }
 306     }
 307     else if (!NeedZero)
 308     {
 309         /* Process or system doesn't want a zero page, grab anything */
 310         PageFrameNumber = MiRemoveAnyPage(Color);
 311     }
 312     else
 313     {
 314         /* System wants a zero page, obtain one */
 315         PageFrameNumber = MiRemoveZeroPage(Color);
 316     }
 317
 318     /* Initialize it */
 319     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
 320
 321     /* Increment demand zero faults */
 322     KeGetCurrentPrcb()->MmDemandZeroCount++;
 323
 324     /* Release PFN lock if needed */
 325     if (HaveLock) KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 326
 327     /* Zero the page if need be */
 328     if (NeedZero) MiZeroPfn(PageFrameNumber);
 329
 330     /* Build the PTE */
 331     if (PointerPte <= MiHighestUserPte)
 332     {
 333         /* For user mode */
 334         MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 335                                   PointerPte,
 336                                   Protection,
 337                                   PageFrameNumber);
 338     }
 339     else
 340     {
 341         /* For kernel mode */
 342         MI_MAKE_HARDWARE_PTE(&TempPte,
 343                              PointerPte,
 344                              Protection,
 345                              PageFrameNumber);
 346     }
 347
 348     /* Set it dirty if it's a writable page */
 349     if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
 350
 351     /* Write it */
 352     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 353
 354     //
 355     // It's all good now
 356     //
 357     DPRINT("Paged pool page has now been paged in\n");
 358     return STATUS_PAGE_FAULT_DEMAND_ZERO;
 359 }
 360
 361 NTSTATUS
 362 NTAPI
 363 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
 364                         IN PVOID Address,
 365                         IN PMMPTE PointerPte,
 366                         IN PMMPTE PointerProtoPte,
 367                         IN KIRQL OldIrql,
 368                         IN PMMPFN Pfn1)
 369 {
 370     MMPTE TempPte;
 371     PMMPTE OriginalPte;
 372     ULONG_PTR Protection;
 373     PFN_NUMBER PageFrameIndex;
 374
 375     /* Must be called with an valid prototype PTE, with the PFN lock held */
 376     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 377     ASSERT(PointerProtoPte->u.Hard.Valid == 1);
 378
 379     /* Get the page */
 380     PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
 381
 382     /* Get the PFN entry and set it as a prototype PTE */
 383     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 384     Pfn1->u3.e1.PrototypePte = 1;
 385
 386     /* FIXME: Increment the share count for the page table */
 387
 388     /* Check where we should be getting the protection information from */
 389     if (PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
 390     {
 391         /* Get the protection from the PTE, there's no real Proto PTE data */
 392         Protection = PointerPte->u.Soft.Protection;
 393     }
 394     else
 395     {
 396         /* Get the protection from the original PTE link */
 397         OriginalPte = &Pfn1->OriginalPte;
 398         Protection = OriginalPte->u.Soft.Protection;
 399     }
 400
 401     /* Release the PFN lock */
 402     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 403
 404     /* Remove caching bits */
 405     Protection &= ~(MM_NOCACHE | MM_NOACCESS);
 406
 407     /* Check if this is a kernel or user address */
 408     if (Address < MmSystemRangeStart)
 409     {
 410         /* Build the user PTE */
 411         MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, Protection, PageFrameIndex);
 412     }
 413     else
 414     {
 415         /* Build the kernel PTE */
 416         MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, Protection, PageFrameIndex);
 417     }
 418
 419     /* Write the PTE */
 420     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 421
 422     /* Return success */
 423     return STATUS_SUCCESS;
 424 }
 425
 426 NTSTATUS
 427 NTAPI
 428 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
 429                        IN PVOID Address,
 430                        IN PMMPTE PointerPte,
 431                        IN PMMPTE PointerProtoPte,
 432                        IN OUT PMMPFN *OutPfn,
 433                        OUT PVOID *PageFileData,
 434                        OUT PMMPTE PteValue,
 435                        IN PEPROCESS Process,
 436                        IN KIRQL OldIrql,
 437                        IN PVOID TrapInformation)
 438 {
 439     MMPTE TempPte;
 440     PMMPFN Pfn1;
 441     PFN_NUMBER PageFrameIndex;
 442     NTSTATUS Status;
 443
 444     /* Must be called with an invalid, prototype PTE, with the PFN lock held */
 445     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 446     ASSERT(PointerPte->u.Hard.Valid == 0);
 447     ASSERT(PointerPte->u.Soft.Prototype == 1);
 448
 449     /* Read the prototype PTE and check if it's valid */
 450     TempPte = *PointerProtoPte;
 451     if (TempPte.u.Hard.Valid == 1)
 452     {
 453         /* One more user of this mapped page */
 454         PageFrameIndex = PFN_FROM_PTE(&TempPte);
 455         Pfn1 = MiGetPfnEntry(PageFrameIndex);
 456         Pfn1->u2.ShareCount++;
 457
 458         /* Call it a transition */
 459         InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
 460
 461         /* Complete the prototype PTE fault -- this will release the PFN lock */
 462         return MiCompleteProtoPteFault(StoreInstruction,
 463                                        Address,
 464                                        PointerPte,
 465                                        PointerProtoPte,
 466                                        OldIrql,
 467                                        NULL);
 468     }
 469
 470     /* Make sure there's some protection mask */
 471     if (TempPte.u.Long == 0)
 472     {
 473         /* Release the lock */
 474         DPRINT1("Access on reserved section?\n");
 475         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 476         return STATUS_ACCESS_VIOLATION;
 477     }
 478
 479     /* This is the only thing we support right now */
 480     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 481     ASSERT(TempPte.u.Proto.ReadOnly == 0);
 482     ASSERT(PointerPte > MiHighestUserPte);
 483     ASSERT(TempPte.u.Soft.Prototype == 0);
 484     ASSERT(TempPte.u.Soft.Transition == 0);
 485
 486     /* Resolve the demand zero fault */
 487     Status = MiResolveDemandZeroFault(Address,
 488                                       (ULONG)PointerProtoPte->u.Soft.Protection,
 489                                       Process,
 490                                       OldIrql);
 491     ASSERT(NT_SUCCESS(Status));
 492
 493     /* Complete the prototype PTE fault -- this will release the PFN lock */
 494     ASSERT(PointerPte->u.Hard.Valid == 0);
 495     return MiCompleteProtoPteFault(StoreInstruction,
 496                                    Address,
 497                                    PointerPte,
 498                                    PointerProtoPte,
 499                                    OldIrql,
 500                                    NULL);
 501 }
 502
 503 NTSTATUS
 504 NTAPI
 505 MiDispatchFault(IN BOOLEAN StoreInstruction,
 506                 IN PVOID Address,
 507                 IN PMMPTE PointerPte,
 508                 IN PMMPTE PointerProtoPte,
 509                 IN BOOLEAN Recursive,
 510                 IN PEPROCESS Process,
 511                 IN PVOID TrapInformation,
 512                 IN PVOID Vad)
 513 {
 514     MMPTE TempPte;
 515     KIRQL OldIrql, LockIrql;
 516     NTSTATUS Status;
 517     PMMPTE SuperProtoPte;
 518     DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
 519              Address,
 520              Process);
 521
 522     /* Make sure the addresses are ok */
 523     ASSERT(PointerPte == MiAddressToPte(Address));
 524
 525     //
 526     // Make sure APCs are off and we're not at dispatch
 527     //
 528     OldIrql = KeGetCurrentIrql();
 529     ASSERT(OldIrql <= APC_LEVEL);
 530     ASSERT(KeAreAllApcsDisabled() == TRUE);
 531
 532     //
 533     // Grab a copy of the PTE
 534     //
 535     TempPte = *PointerPte;
 536
 537     /* Do we have a prototype PTE? */
 538     if (PointerProtoPte)
 539     {
 540         /* This should never happen */
 541         ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
 542
 543         /* Check if this is a kernel-mode address */
 544         SuperProtoPte = MiAddressToPte(PointerProtoPte);
 545         if (Address >= MmSystemRangeStart)
 546         {
 547             /* Lock the PFN database */
 548             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 549
 550             /* Has the PTE been made valid yet? */
 551             if (!SuperProtoPte->u.Hard.Valid)
 552             {
 553                 UNIMPLEMENTED;
 554                 while (TRUE);
 555             }
 556             else
 557             {
 558                 /* Resolve the fault -- this will release the PFN lock */
 559                 ASSERT(PointerPte->u.Hard.Valid == 0);
 560                 Status = MiResolveProtoPteFault(StoreInstruction,
 561                                                 Address,
 562                                                 PointerPte,
 563                                                 PointerProtoPte,
 564                                                 NULL,
 565                                                 NULL,
 566                                                 NULL,
 567                                                 Process,
 568                                                 LockIrql,
 569                                                 TrapInformation);
 570                 ASSERT(Status == STATUS_SUCCESS);
 571
 572                 /* Complete this as a transition fault */
 573                 ASSERT(OldIrql == KeGetCurrentIrql());
 574                 ASSERT(OldIrql <= APC_LEVEL);
 575                 ASSERT(KeAreAllApcsDisabled() == TRUE);
 576                 return Status;
 577             }
 578         }
 579         else
 580         {
 581             /* We currently only handle very limited paths */
 582             ASSERT(PointerPte->u.Soft.Prototype == 1);
 583             ASSERT(PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED);
 584
 585             /* Lock the PFN database */
 586             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 587
 588             /* For our current usage, this should be true */
 589             ASSERT(SuperProtoPte->u.Hard.Valid == 1);
 590             ASSERT(TempPte.u.Hard.Valid == 0);
 591
 592             /* Resolve the fault -- this will release the PFN lock */
 593             Status = MiResolveProtoPteFault(StoreInstruction,
 594                                             Address,
 595                                             PointerPte,
 596                                             PointerProtoPte,
 597                                             NULL,
 598                                             NULL,
 599                                             NULL,
 600                                             Process,
 601                                             LockIrql,
 602                                             TrapInformation);
 603             ASSERT(Status == STATUS_SUCCESS);
 604
 605             /* Complete this as a transition fault */
 606             ASSERT(OldIrql == KeGetCurrentIrql());
 607             ASSERT(OldIrql <= APC_LEVEL);
 608             ASSERT(KeAreAllApcsDisabled() == TRUE);
 609             return STATUS_PAGE_FAULT_TRANSITION;
 610         }
 611     }
 612
 613     //
 614     // The PTE must be invalid
 615     //
 616     ASSERT(TempPte.u.Hard.Valid == 0);
 617
 618     /* Check if the PTE is completely empty */
 619     if (TempPte.u.Long == 0)
 620     {
 621         KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 622                      (ULONG_PTR)Address,
 623                      StoreInstruction,
 624                      (ULONG_PTR)TrapInformation,
 625                      2);
 626     }
 627
 628     //
 629     // No prototype, transition or page file software PTEs in ARM3 yet
 630     //
 631     ASSERT(TempPte.u.Soft.Prototype == 0);
 632     ASSERT(TempPte.u.Soft.Transition == 0);
 633     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 634
 635     //
 636     // If we got this far, the PTE can only be a demand zero PTE, which is what
 637     // we want. Go handle it!
 638     //
 639     Status = MiResolveDemandZeroFault(Address,
 640                                       (ULONG)PointerPte->u.Soft.Protection,
 641                                       Process,
 642                                       MM_NOIRQL);
 643     ASSERT(KeAreAllApcsDisabled() == TRUE);
 644     if (NT_SUCCESS(Status))
 645     {
 646         //
 647         // Make sure we're returning in a sane state and pass the status down
 648         //
 649         ASSERT(OldIrql == KeGetCurrentIrql());
 650         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 651         return Status;
 652     }
 653
 654     //
 655     // Generate an access fault
 656     //
 657     return STATUS_ACCESS_VIOLATION;
 658 }
 659
 660 NTSTATUS
 661 NTAPI
 662 MmArmAccessFault(IN BOOLEAN StoreInstruction,
 663                  IN PVOID Address,
 664                  IN KPROCESSOR_MODE Mode,
 665                  IN PVOID TrapInformation)
 666 {
 667     KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
 668     PMMPTE ProtoPte = NULL;
 669     PMMPTE PointerPte = MiAddressToPte(Address);
 670     PMMPDE PointerPde = MiAddressToPde(Address);
 671 #if (_MI_PAGING_LEVELS >= 3)
 672     PMMPDE PointerPpe = MiAddressToPpe(Address);
 673 #if (_MI_PAGING_LEVELS == 4)
 674     PMMPDE PointerPxe = MiAddressToPxe(Address);
 675 #endif
 676 #endif
 677     MMPTE TempPte;
 678     PETHREAD CurrentThread;
 679     PEPROCESS CurrentProcess;
 680     NTSTATUS Status;
 681     PMMSUPPORT WorkingSet;
 682     ULONG ProtectionCode;
 683     PMMVAD Vad;
 684     PFN_NUMBER PageFrameIndex;
 685     ULONG Color;
 686
 687     DPRINT("ARM3 FAULT AT: %p\n", Address);
 688
 689     /* Check for page fault on high IRQL */
 690     if (OldIrql > APC_LEVEL)
 691     {
 692         // There are some special cases where this is okay, but not in ARM3 yet
 693         DbgPrint("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
 694                  Address,
 695                  OldIrql);
 696         ASSERT(OldIrql <= APC_LEVEL);
 697     }
 698
 699     /* Check for kernel fault address */
 700     while (Address >= MmSystemRangeStart)
 701     {
 702         /* Bail out, if the fault came from user mode */
 703         if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
 704
 705         /* PXEs and PPEs for kernel mode are mapped for everything we need */
 706 #if (_MI_PAGING_LEVELS >= 3)
 707         if (
 708 #if (_MI_PAGING_LEVELS == 4)
 709             (PointerPxe->u.Hard.Valid == 0) ||
 710 #endif
 711             (PointerPpe->u.Hard.Valid == 0))
 712         {
 713             /* The address is not from any pageable area! */
 714             KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 715                          (ULONG_PTR)Address,
 716                          StoreInstruction,
 717                          (ULONG_PTR)TrapInformation,
 718                          2);
 719         }
 720 #endif
 721
 722         /* Check if the PDE is invalid */
 723         if (PointerPde->u.Hard.Valid == 0)
 724         {
 725 #if (_MI_PAGING_LEVELS == 2)
 726             /* Sync this PDE and check, if that made it valid */
 727             if (!MiSynchronizeSystemPde(PointerPde))
 728 #endif
 729             {
 730                 /* PDE (still) not valid, kill the system */
 731                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 732                              (ULONG_PTR)Address,
 733                              StoreInstruction,
 734                              (ULONG_PTR)TrapInformation,
 735                              2);
 736             }
 737         }
 738
 739         /* The PDE is valid, so read the PTE */
 740         TempPte = *PointerPte;
 741         if (TempPte.u.Hard.Valid == 1)
 742         {
 743             //
 744             // Only two things can go wrong here:
 745             // Executing NX page (we couldn't care less)
 746             // Writing to a read-only page (the stuff ARM3 works with is write,
 747             // so again, moot point).
 748             //
 749
 750             //
 751             // Otherwise, the PDE was probably invalid, and all is good now
 752             //
 753             return STATUS_SUCCESS;
 754         }
 755
 756         // Check for a fault on the page table or hyperspace
 757         if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
 758         {
 759 #if (_MI_PAGING_LEVELS == 2)
 760             /* Could be paged pool access from a new process -- synchronize the page directories */
 761             if (MiCheckPdeForPagedPool(Address) == STATUS_WAIT_1)
 762             {
 763                 DPRINT1("PAGE TABLES FAULTED IN!\n");
 764                 return STATUS_SUCCESS;
 765             }
 766 #endif
 767             /* Otherwise this could be a commit of a virtual address */
 768             break;
 769         }
 770
 771         /* In this path, we are using the system working set */
 772         CurrentThread = PsGetCurrentThread();
 773         WorkingSet = &MmSystemCacheWs;
 774
 775         /* Acquire it */
 776         KeRaiseIrql(APC_LEVEL, &LockIrql);
 777         MiLockWorkingSet(CurrentThread, WorkingSet);
 778
 779         /* Re-read PTE now that we own the lock */
 780         TempPte = *PointerPte;
 781         if (TempPte.u.Hard.Valid == 1)
 782         {
 783             // Only two things can go wrong here:
 784             // Executing NX page (we couldn't care less)
 785             // Writing to a read-only page (the stuff ARM3 works with is write,
 786             // so again, moot point).
 787             ASSERT(TempPte.u.Hard.Write == 1);
 788
 789             /* Release the working set */
 790             MiUnlockWorkingSet(CurrentThread, WorkingSet);
 791             KeLowerIrql(LockIrql);
 792
 793             // Otherwise, the PDE was probably invalid, and all is good now
 794             return STATUS_SUCCESS;
 795         }
 796
 797         /* Check one kind of prototype PTE */
 798         if (TempPte.u.Soft.Prototype)
 799         {
 800             /* Make sure protected pool is on, and that this is a pool address */
 801             if ((MmProtectFreedNonPagedPool) &&
 802                 (((Address >= MmNonPagedPoolStart) &&
 803                   (Address < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
 804                                      MmSizeOfNonPagedPoolInBytes))) ||
 805                  ((Address >= MmNonPagedPoolExpansionStart) &&
 806                   (Address < MmNonPagedPoolEnd))))
 807             {
 808                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
 809                 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL,
 810                              (ULONG_PTR)Address,
 811                              StoreInstruction,
 812                              Mode,
 813                              4);
 814             }
 815
 816             /* Get the prototype PTE! */
 817             ProtoPte = MiProtoPteToPte(&TempPte);
 818         }
 819         else
 820         {
 821             /* We don't implement transition PTEs */
 822             ASSERT(TempPte.u.Soft.Transition == 0);
 823
 824             /* Check for no-access PTE */
 825             if (TempPte.u.Soft.Protection == MM_NOACCESS)
 826             {
 827                 /* Bugcheck the system! */
 828                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 829                              (ULONG_PTR)Address,
 830                              StoreInstruction,
 831                              (ULONG_PTR)TrapInformation,
 832                              1);
 833             }
 834
 835             /* Check for demand page */
 836             if ((StoreInstruction) && !(TempPte.u.Hard.Valid))
 837             {
 838                 /* Get the protection code */
 839                 if (!(TempPte.u.Soft.Protection & MM_READWRITE))
 840                 {
 841                     /* Bugcheck the system! */
 842                     KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
 843                                  (ULONG_PTR)Address,
 844                                  TempPte.u.Long,
 845                                  (ULONG_PTR)TrapInformation,
 846                                  14);
 847                 }
 848             }
 849         }
 850
 851         /* Now do the real fault handling */
 852         Status = MiDispatchFault(StoreInstruction,
 853                                  Address,
 854                                  PointerPte,
 855                                  ProtoPte,
 856                                  FALSE,
 857                                  NULL,
 858                                  TrapInformation,
 859                                  NULL);
 860
 861         /* Release the working set */
 862         ASSERT(KeAreAllApcsDisabled() == TRUE);
 863         MiUnlockWorkingSet(CurrentThread, WorkingSet);
 864         KeLowerIrql(LockIrql);
 865
 866         /* We are done! */
 867         DPRINT("Fault resolved with status: %lx\n", Status);
 868         return Status;
 869     }
 870
 871 #if (_MI_PAGING_LEVELS == 4)
 872     /* On these systems we have PXEs and PPEs ready for everything we need */
 873     if (PointerPxe->u.Hard.Valid == 0) return STATUS_ACCESS_VIOLATION;
 874 #endif
 875 #if (_MI_PAGING_LEVELS >= 3)
 876     if (PointerPpe->u.Hard.Valid == 0) return STATUS_ACCESS_VIOLATION;
 877 #endif
 878
 879     /* This is a user fault (<- And this is a lie!) */
 880     CurrentThread = PsGetCurrentThread();
 881     CurrentProcess = PsGetCurrentProcess();
 882
 883     /* Lock the working set */
 884     MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
 885
 886     /* First things first, is the PDE valid? */
 887     ASSERT(PointerPde->u.Hard.LargePage == 0);
 888     if (PointerPde->u.Hard.Valid == 0)
 889     {
 890         /* Right now, we only handle scenarios where the PDE is totally empty */
 891         ASSERT(PointerPde->u.Long == 0);
 892
 893         /* Check if this address range belongs to a valid allocation (VAD) */
 894         MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 895
 896         /* Right now, we expect a valid protection mask on the VAD */
 897         ASSERT(ProtectionCode != MM_NOACCESS);
 898
 899         /* And go dispatch the fault on the PDE. This should handle the demand-zero */
 900 #if MI_TRACE_PFNS
 901         UserPdeFault = TRUE;
 902 #endif
 903         /* Resolve a demand zero fault */
 904         Status = MiResolveDemandZeroFault(PointerPte,
 905                                           MM_READWRITE,
 906                                           CurrentProcess,
 907                                           MM_NOIRQL);
 908 #if MI_TRACE_PFNS
 909         UserPdeFault = FALSE;
 910 #endif
 911         /* We should come back with APCs enabled, and with a valid PDE */
 912         ASSERT(KeAreAllApcsDisabled() == TRUE);
 913         ASSERT(PointerPde->u.Hard.Valid == 1);
 914     }
 915
 916     /* Now capture the PTE. Ignore virtual faults for now */
 917     TempPte = *PointerPte;
 918     ASSERT(TempPte.u.Hard.Valid == 0);
 919
 920     /* Quick check for demand-zero */
 921     if (TempPte.u.Long == (MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS))
 922     {
 923         /* Resolve the fault */
 924         MiResolveDemandZeroFault(Address,
 925                                  (ULONG)PointerPte->u.Soft.Protection,
 926                                  CurrentProcess,
 927                                  MM_NOIRQL);
 928
 929         /* Return the status */
 930         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 931         return STATUS_PAGE_FAULT_DEMAND_ZERO;
 932     }
 933
 934     /* Make sure it's not a prototype PTE */
 935     ASSERT(TempPte.u.Soft.Prototype == 0);
 936
 937     /* Check for non-demand zero PTE */
 938     if (TempPte.u.Long != 0)
 939     {
 940         /* This is a page fault, check for valid protection */
 941         ASSERT(TempPte.u.Soft.Protection != 0x100);
 942
 943         /* FIXME: Run MiAccessCheck */
 944
 945         /* Dispatch the fault */
 946         Status = MiDispatchFault(StoreInstruction,
 947                                  Address,
 948                                  PointerPte,
 949                                  NULL,
 950                                  FALSE,
 951                                  PsGetCurrentProcess(),
 952                                  TrapInformation,
 953                                  NULL);
 954
 955         /* Return the status */
 956         ASSERT(NT_SUCCESS(Status));
 957         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 958         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 959         return Status;
 960     }
 961
 962     /* Check if this address range belongs to a valid allocation (VAD) */
 963     ASSERT(TempPte.u.Long == 0);
 964     ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 965     if (ProtectionCode == MM_NOACCESS)
 966     {
 967         /* This is a bogus VA */
 968         Status = STATUS_ACCESS_VIOLATION;
 969
 970         /* Could be a not-yet-mapped paged pool page table */
 971 #if (_MI_PAGING_LEVELS == 2)
 972         MiCheckPdeForPagedPool(Address);
 973 #endif
 974         /* See if that fixed it */
 975         if (PointerPte->u.Hard.Valid == 1) Status = STATUS_SUCCESS;
 976
 977         /* Return the status */
 978         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 979         return Status;
 980     }
 981
 982     /* Is this a user address? */
 983     if (Address <= MM_HIGHEST_USER_ADDRESS)
 984     {
 985         /* Add an additional page table reference */
 986         MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]++;
 987         ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT);
 988     }
 989
 990     /* Did we get a prototype PTE back? */
 991     if (!ProtoPte)
 992     {
 993         /* No, create a new PTE. First, write the protection */
 994         PointerPte->u.Soft.Protection = ProtectionCode;
 995
 996         /* Lock the PFN database since we're going to grab a page */
 997         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 998
 999         /* Try to get a zero page */
1000         MI_SET_USAGE(MI_USAGE_PEB_TEB);
1001         MI_SET_PROCESS2(CurrentProcess->ImageFileName);
1002         Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
1003         PageFrameIndex = MiRemoveZeroPageSafe(Color);
1004         if (!PageFrameIndex)
1005         {
1006             /* Grab a page out of there. Later we should grab a colored zero page */
1007             PageFrameIndex = MiRemoveAnyPage(Color);
1008             ASSERT(PageFrameIndex);
1009
1010             /* Release the lock since we need to do some zeroing */
1011             KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1012
1013             /* Zero out the page, since it's for user-mode */
1014             MiZeroPfn(PageFrameIndex);
1015
1016             /* Grab the lock again so we can initialize the PFN entry */
1017             OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1018         }
1019
1020         /* Initialize the PFN entry now */
1021         MiInitializePfn(PageFrameIndex, PointerPte, 1);
1022
1023         /* One more demand-zero fault */
1024         KeGetCurrentPrcb()->MmDemandZeroCount++;
1025
1026         /* And we're done with the lock */
1027         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1028
1029
1030         /* Was the fault on an actual user page, or a kernel page for the user? */
1031         if (PointerPte <= MiHighestUserPte)
1032         {
1033             /* User fault, build a user PTE */
1034             MI_MAKE_HARDWARE_PTE_USER(&TempPte,
1035                                       PointerPte,
1036                                       PointerPte->u.Soft.Protection,
1037                                       PageFrameIndex);
1038         }
1039         else
1040         {
1041             /* Session, kernel, or user PTE, figure it out and build it */
1042             MI_MAKE_HARDWARE_PTE(&TempPte,
1043                                  PointerPte,
1044                                  PointerPte->u.Soft.Protection,
1045                                  PageFrameIndex);
1046         }
1047
1048         /* Write the dirty bit for writeable pages */
1049         if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
1050
1051         /* And now write down the PTE, making the address valid */
1052         MI_WRITE_VALID_PTE(PointerPte, TempPte);
1053
1054         /* Demand zero */
1055         Status = STATUS_PAGE_FAULT_DEMAND_ZERO;
1056     }
1057     else
1058     {
1059         /* No guard page support yet */
1060         ASSERT((ProtectionCode & MM_DECOMMIT) == 0);
1061         ASSERT(ProtectionCode != 0x100);
1062
1063         /* Write the prototype PTE */
1064         TempPte = PrototypePte;
1065         TempPte.u.Soft.Protection = ProtectionCode;
1066         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
1067
1068         /* Handle the fault */
1069         Status = MiDispatchFault(StoreInstruction,
1070                                  Address,
1071                                  PointerPte,
1072                                  ProtoPte,
1073                                  FALSE,
1074                                  CurrentProcess,
1075                                  TrapInformation,
1076                                  Vad);
1077         ASSERT(Status == STATUS_PAGE_FAULT_TRANSITION);
1078         ASSERT(PointerPte->u.Hard.Valid == 1);
1079         ASSERT(PointerPte->u.Hard.PageFrameNumber != 0);
1080     }
1081
1082     /* Release the working set */
1083     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
1084     return Status;
1085 }
1086
1087 /* EOF */