reactos/ntoskrnl/mm/ARM3/pagfault.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pagfault.c
   5  * PURPOSE:         ARM Memory Manager Page Fault Handling
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include "../ARM3/miarm.h"
  17
  18 /* GLOBALS ********************************************************************/
  19
  20 #if MI_TRACE_PFNS
  21 BOOLEAN UserPdeFault = FALSE;
  22 #endif
  23
  24 /* PRIVATE FUNCTIONS **********************************************************/
  25
  26 PMMPTE
  27 NTAPI
  28 MiCheckVirtualAddress(IN PVOID VirtualAddress,
  29                       OUT PULONG ProtectCode,
  30                       OUT PMMVAD *ProtoVad)
  31 {
  32     PMMVAD Vad;
  33     PMMPTE PointerPte;
  34
  35     /* No prototype/section support for now */
  36     *ProtoVad = NULL;
  37
  38     /* Check if this is a page table address */
  39     if (MI_IS_PAGE_TABLE_ADDRESS(VirtualAddress))
  40     {
  41         /* This should never happen, as these addresses are handled by the double-maping */
  42         if (((PMMPTE)VirtualAddress >= MiAddressToPte(MmPagedPoolStart)) &&
  43             ((PMMPTE)VirtualAddress <= MmPagedPoolInfo.LastPteForPagedPool))
  44         {
  45             /* Fail such access */
  46             *ProtectCode = MM_NOACCESS;
  47             return NULL;
  48         }
  49
  50         /* Return full access rights */
  51         *ProtectCode = MM_READWRITE;
  52         return NULL;
  53     }
  54
  55     /* Should not be a session address */
  56     ASSERT(MI_IS_SESSION_ADDRESS(VirtualAddress) == FALSE);
  57
  58     /* Special case for shared data */
  59     if (PAGE_ALIGN(VirtualAddress) == (PVOID)MM_SHARED_USER_DATA_VA)
  60     {
  61         /* It's a read-only page */
  62         *ProtectCode = MM_READONLY;
  63         return MmSharedUserDataPte;
  64     }
  65
  66     /* Find the VAD, it might not exist if the address is bogus */
  67     Vad = MiLocateAddress(VirtualAddress);
  68     if (!Vad)
  69     {
  70         /* Bogus virtual address */
  71         *ProtectCode = MM_NOACCESS;
  72         return NULL;
  73     }
  74
  75     /* This must be a VM VAD */
  76     ASSERT(Vad->u.VadFlags.VadType == VadNone);
  77
  78     /* Check if it's a section, or just an allocation */
  79     if (Vad->u.VadFlags.PrivateMemory == TRUE)
  80     {
  81         /* This must be a TEB/PEB VAD */
  82         ASSERT(Vad->u.VadFlags.MemCommit == TRUE);
  83         *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
  84         return NULL;
  85     }
  86     else
  87     {
  88         /* Return the proto VAD */
  89         ASSERT(Vad->u2.VadFlags2.ExtendableFile == 0);
  90         *ProtoVad = Vad;
  91
  92         /* Get the prototype PTE for this page */
  93         PointerPte = (((ULONG_PTR)VirtualAddress >> PAGE_SHIFT) - Vad->StartingVpn) + Vad->FirstPrototypePte;
  94         ASSERT(PointerPte <= Vad->LastContiguousPte);
  95         ASSERT(PointerPte != NULL);
  96
  97         /* Return the Prototype PTE and the protection for the page mapping */
  98         *ProtectCode = (ULONG)Vad->u.VadFlags.Protection;
  99         return PointerPte;
 100     }
 101 }
 102
 103 #if (_MI_PAGING_LEVELS == 2)
 104 BOOLEAN
 105 FORCEINLINE
 106 MiSynchronizeSystemPde(PMMPDE PointerPde)
 107 {
 108     MMPDE SystemPde;
 109     ULONG Index;
 110
 111     /* Get the Index from the PDE */
 112     Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE);
 113
 114     /* Copy the PDE from the double-mapped system page directory */
 115     SystemPde = MmSystemPagePtes[Index];
 116     *PointerPde = SystemPde;
 117
 118     /* Make sure we re-read the PDE and PTE */
 119     KeMemoryBarrierWithoutFence();
 120
 121     /* Return, if we had success */
 122     return (BOOLEAN)SystemPde.u.Hard.Valid;
 123 }
 124
 125 NTSTATUS
 126 FASTCALL
 127 MiCheckPdeForPagedPool(IN PVOID Address)
 128 {
 129     PMMPDE PointerPde;
 130     NTSTATUS Status = STATUS_SUCCESS;
 131
 132     /* No session support in ReactOS yet */
 133     ASSERT(MI_IS_SESSION_ADDRESS(Address) == FALSE);
 134     ASSERT(MI_IS_SESSION_PTE(Address) == FALSE);
 135
 136     //
 137     // Check if this is a fault while trying to access the page table itself
 138     //
 139     if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
 140     {
 141         //
 142         // Send a hint to the page fault handler that this is only a valid fault
 143         // if we already detected this was access within the page table range
 144         //
 145         PointerPde = (PMMPDE)MiAddressToPte(Address);
 146         Status = STATUS_WAIT_1;
 147     }
 148     else if (Address < MmSystemRangeStart)
 149     {
 150         //
 151         // This is totally illegal
 152         //
 153         return STATUS_ACCESS_VIOLATION;
 154     }
 155     else
 156     {
 157         //
 158         // Get the PDE for the address
 159         //
 160         PointerPde = MiAddressToPde(Address);
 161     }
 162
 163     //
 164     // Check if it's not valid
 165     //
 166     if (PointerPde->u.Hard.Valid == 0)
 167     {
 168 #ifdef _M_AMD64
 169         ASSERT(FALSE);
 170 #else
 171         //
 172         // Copy it from our double-mapped system page directory
 173         //
 174         InterlockedExchangePte(PointerPde,
 175                                MmSystemPagePtes[((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE)].u.Long);
 176 #endif
 177     }
 178
 179     //
 180     // Return status
 181     //
 182     return Status;
 183 }
 184 #endif
 185
 186 VOID
 187 NTAPI
 188 MiZeroPfn(IN PFN_NUMBER PageFrameNumber)
 189 {
 190     PMMPTE ZeroPte;
 191     MMPTE TempPte;
 192     PMMPFN Pfn1;
 193     PVOID ZeroAddress;
 194
 195     /* Get the PFN for this page */
 196     Pfn1 = MiGetPfnEntry(PageFrameNumber);
 197     ASSERT(Pfn1);
 198
 199     /* Grab a system PTE we can use to zero the page */
 200     ZeroPte = MiReserveSystemPtes(1, SystemPteSpace);
 201     ASSERT(ZeroPte);
 202
 203     /* Initialize the PTE for it */
 204     TempPte = ValidKernelPte;
 205     TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
 206
 207     /* Setup caching */
 208     if (Pfn1->u3.e1.CacheAttribute == MiWriteCombined)
 209     {
 210         /* Write combining, no caching */
 211         MI_PAGE_DISABLE_CACHE(&TempPte);
 212         MI_PAGE_WRITE_COMBINED(&TempPte);
 213     }
 214     else if (Pfn1->u3.e1.CacheAttribute == MiNonCached)
 215     {
 216         /* Write through, no caching */
 217         MI_PAGE_DISABLE_CACHE(&TempPte);
 218         MI_PAGE_WRITE_THROUGH(&TempPte);
 219     }
 220
 221     /* Make the system PTE valid with our PFN */
 222     MI_WRITE_VALID_PTE(ZeroPte, TempPte);
 223
 224     /* Get the address it maps to, and zero it out */
 225     ZeroAddress = MiPteToAddress(ZeroPte);
 226     KeZeroPages(ZeroAddress, PAGE_SIZE);
 227
 228     /* Now get rid of it */
 229     MiReleaseSystemPtes(ZeroPte, 1, SystemPteSpace);
 230 }
 231
 232 NTSTATUS
 233 NTAPI
 234 MiResolveDemandZeroFault(IN PVOID Address,
 235                          IN ULONG Protection,
 236                          IN PEPROCESS Process,
 237                          IN KIRQL OldIrql)
 238 {
 239     PFN_NUMBER PageFrameNumber = 0;
 240     PMMPTE PointerPte = MiAddressToPte(Address);
 241     MMPTE TempPte;
 242     BOOLEAN NeedZero = FALSE, HaveLock = FALSE;
 243     ULONG Color;
 244     DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n",
 245             Address,
 246             Process);
 247
 248     /* Must currently only be called by paging path */
 249     if ((Process) && (OldIrql == MM_NOIRQL))
 250     {
 251         /* Sanity check */
 252         ASSERT(MI_IS_PAGE_TABLE_ADDRESS(PointerPte));
 253
 254         /* No forking yet */
 255         ASSERT(Process->ForkInProgress == NULL);
 256
 257         /* Get process color */
 258         Color = MI_GET_NEXT_PROCESS_COLOR(Process);
 259         ASSERT(Color != 0xFFFFFFFF);
 260
 261         /* We'll need a zero page */
 262         NeedZero = TRUE;
 263     }
 264     else
 265     {
 266         /* Check if we need a zero page */
 267         NeedZero = (OldIrql != MM_NOIRQL);
 268
 269         /* Get the next system page color */
 270         Color = MI_GET_NEXT_COLOR();
 271     }
 272
 273     /* Check if the PFN database should be acquired */
 274     if (OldIrql == MM_NOIRQL)
 275     {
 276         /* Acquire it and remember we should release it after */
 277         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 278         HaveLock = TRUE;
 279     }
 280
 281     /* We either manually locked the PFN DB, or already came with it locked */
 282     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 283
 284     /* Do we need a zero page? */
 285     ASSERT(PointerPte->u.Hard.Valid == 0);
 286 #if MI_TRACE_PFNS
 287     if (UserPdeFault) MI_SET_USAGE(MI_USAGE_PAGE_TABLE);
 288     if (!UserPdeFault) MI_SET_USAGE(MI_USAGE_DEMAND_ZERO);
 289 #endif
 290     if (Process) MI_SET_PROCESS2(Process->ImageFileName);
 291     if (!Process) MI_SET_PROCESS2("Kernel Demand 0");
 292     if ((NeedZero) && (Process))
 293     {
 294         /* Try to get one, if we couldn't grab a free page and zero it */
 295         PageFrameNumber = MiRemoveZeroPageSafe(Color);
 296         if (PageFrameNumber)
 297         {
 298             /* We got a genuine zero page, stop worrying about it */
 299             NeedZero = FALSE;
 300         }
 301         else
 302         {
 303             /* We'll need a free page and zero it manually */
 304             PageFrameNumber = MiRemoveAnyPage(Color);
 305         }
 306     }
 307     else if (!NeedZero)
 308     {
 309         /* Process or system doesn't want a zero page, grab anything */
 310         PageFrameNumber = MiRemoveAnyPage(Color);
 311     }
 312     else
 313     {
 314         /* System wants a zero page, obtain one */
 315         PageFrameNumber = MiRemoveZeroPage(Color);
 316     }
 317
 318     /* Initialize it */
 319     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
 320
 321     /* Increment demand zero faults */
 322     KeGetCurrentPrcb()->MmDemandZeroCount++;
 323
 324     /* Release PFN lock if needed */
 325     if (HaveLock) KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 326
 327     /* Zero the page if need be */
 328     if (NeedZero) MiZeroPfn(PageFrameNumber);
 329
 330     /* Build the PTE */
 331     if (PointerPte <= MiHighestUserPte)
 332     {
 333         /* For user mode */
 334         MI_MAKE_HARDWARE_PTE_USER(&TempPte,
 335                                   PointerPte,
 336                                   Protection,
 337                                   PageFrameNumber);
 338     }
 339     else
 340     {
 341         /* For kernel mode */
 342         MI_MAKE_HARDWARE_PTE(&TempPte,
 343                              PointerPte,
 344                              Protection,
 345                              PageFrameNumber);
 346     }
 347
 348     /* Set it dirty if it's a writable page */
 349     if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
 350
 351     /* Write it */
 352     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 353
 354     //
 355     // It's all good now
 356     //
 357     DPRINT("Paged pool page has now been paged in\n");
 358     return STATUS_PAGE_FAULT_DEMAND_ZERO;
 359 }
 360
 361 NTSTATUS
 362 NTAPI
 363 MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction,
 364                         IN PVOID Address,
 365                         IN PMMPTE PointerPte,
 366                         IN PMMPTE PointerProtoPte,
 367                         IN KIRQL OldIrql,
 368                         IN PMMPFN Pfn1)
 369 {
 370     MMPTE TempPte;
 371     PMMPTE OriginalPte;
 372     ULONG_PTR Protection;
 373     PFN_NUMBER PageFrameIndex;
 374
 375     /* Must be called with an valid prototype PTE, with the PFN lock held */
 376     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 377     ASSERT(PointerProtoPte->u.Hard.Valid == 1);
 378
 379     /* Get the page */
 380     PageFrameIndex = PFN_FROM_PTE(PointerProtoPte);
 381
 382     /* Get the PFN entry and set it as a prototype PTE */
 383     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 384     Pfn1->u3.e1.PrototypePte = 1;
 385
 386     /* FIXME: Increment the share count for the page table */
 387
 388     /* Check where we should be getting the protection information from */
 389     if (PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED)
 390     {
 391         /* Get the protection from the PTE, there's no real Proto PTE data */
 392         Protection = PointerPte->u.Soft.Protection;
 393     }
 394     else
 395     {
 396         /* Get the protection from the original PTE link */
 397         OriginalPte = &Pfn1->OriginalPte;
 398         Protection = OriginalPte->u.Soft.Protection;
 399     }
 400
 401     /* Release the PFN lock */
 402     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 403
 404     /* Remove caching bits */
 405     Protection &= ~(MM_NOCACHE | MM_NOACCESS);
 406
 407     /* Check if this is a kernel or user address */
 408     if (Address < MmSystemRangeStart)
 409     {
 410         /* Build the user PTE */
 411         MI_MAKE_HARDWARE_PTE_USER(&TempPte, PointerPte, Protection, PageFrameIndex);
 412     }
 413     else
 414     {
 415         /* Build the kernel PTE */
 416         MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, Protection, PageFrameIndex);
 417     }
 418
 419     /* Write the PTE */
 420     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 421
 422     /* Return success */
 423     return STATUS_SUCCESS;
 424 }
 425
 426 NTSTATUS
 427 NTAPI
 428 MiResolveProtoPteFault(IN BOOLEAN StoreInstruction,
 429                        IN PVOID Address,
 430                        IN PMMPTE PointerPte,
 431                        IN PMMPTE PointerProtoPte,
 432                        IN OUT PMMPFN *OutPfn,
 433                        OUT PVOID *PageFileData,
 434                        OUT PMMPTE PteValue,
 435                        IN PEPROCESS Process,
 436                        IN KIRQL OldIrql,
 437                        IN PVOID TrapInformation)
 438 {
 439     MMPTE TempPte;
 440     PMMPFN Pfn1;
 441     PFN_NUMBER PageFrameIndex;
 442     NTSTATUS Status;
 443
 444     /* Must be called with an invalid, prototype PTE, with the PFN lock held */
 445     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 446     ASSERT(PointerPte->u.Hard.Valid == 0);
 447     ASSERT(PointerPte->u.Soft.Prototype == 1);
 448
 449     /* Read the prototype PTE and check if it's valid */
 450     TempPte = *PointerProtoPte;
 451     if (TempPte.u.Hard.Valid == 1)
 452     {
 453         /* One more user of this mapped page */
 454         PageFrameIndex = PFN_FROM_PTE(&TempPte);
 455         Pfn1 = MiGetPfnEntry(PageFrameIndex);
 456         Pfn1->u2.ShareCount++;
 457
 458         /* Call it a transition */
 459         InterlockedIncrement(&KeGetCurrentPrcb()->MmTransitionCount);
 460
 461         /* Complete the prototype PTE fault -- this will release the PFN lock */
 462         return MiCompleteProtoPteFault(StoreInstruction,
 463                                        Address,
 464                                        PointerPte,
 465                                        PointerProtoPte,
 466                                        OldIrql,
 467                                        NULL);
 468     }
 469
 470     /* Make sure there's some protection mask */
 471     if (TempPte.u.Long == 0)
 472     {
 473         /* Release the lock */
 474         DPRINT1("Access on reserved section?\n");
 475         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 476         return STATUS_ACCESS_VIOLATION;
 477     }
 478
 479     /* This is the only thing we support right now */
 480     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 481     ASSERT(TempPte.u.Proto.ReadOnly == 0);
 482     ASSERT(PointerPte > MiHighestUserPte);
 483     ASSERT(TempPte.u.Soft.Prototype == 0);
 484     ASSERT(TempPte.u.Soft.Transition == 0);
 485
 486     /* Resolve the demand zero fault */
 487     Status = MiResolveDemandZeroFault(Address,
 488                                       (ULONG)PointerProtoPte->u.Soft.Protection,
 489                                       Process,
 490                                       OldIrql);
 491     ASSERT(NT_SUCCESS(Status));
 492
 493     /* Complete the prototype PTE fault -- this will release the PFN lock */
 494     ASSERT(PointerPte->u.Hard.Valid == 0);
 495     return MiCompleteProtoPteFault(StoreInstruction,
 496                                    Address,
 497                                    PointerPte,
 498                                    PointerProtoPte,
 499                                    OldIrql,
 500                                    NULL);
 501 }
 502
 503 NTSTATUS
 504 NTAPI
 505 MiDispatchFault(IN BOOLEAN StoreInstruction,
 506                 IN PVOID Address,
 507                 IN PMMPTE PointerPte,
 508                 IN PMMPTE PointerProtoPte,
 509                 IN BOOLEAN Recursive,
 510                 IN PEPROCESS Process,
 511                 IN PVOID TrapInformation,
 512                 IN PVOID Vad)
 513 {
 514     MMPTE TempPte;
 515     KIRQL OldIrql, LockIrql;
 516     NTSTATUS Status;
 517     PMMPTE SuperProtoPte;
 518     DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n",
 519              Address,
 520              Process);
 521
 522     /* Make sure the addresses are ok */
 523     ASSERT(PointerPte == MiAddressToPte(Address));
 524
 525     //
 526     // Make sure APCs are off and we're not at dispatch
 527     //
 528     OldIrql = KeGetCurrentIrql();
 529     ASSERT(OldIrql <= APC_LEVEL);
 530     ASSERT(KeAreAllApcsDisabled() == TRUE);
 531
 532     //
 533     // Grab a copy of the PTE
 534     //
 535     TempPte = *PointerPte;
 536
 537     /* Do we have a prototype PTE? */
 538     if (PointerProtoPte)
 539     {
 540         /* This should never happen */
 541         ASSERT(!MI_IS_PHYSICAL_ADDRESS(PointerProtoPte));
 542
 543         /* Check if this is a kernel-mode address */
 544         SuperProtoPte = MiAddressToPte(PointerProtoPte);
 545         if (Address >= MmSystemRangeStart)
 546         {
 547             /* Lock the PFN database */
 548             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 549
 550             /* Has the PTE been made valid yet? */
 551             if (!SuperProtoPte->u.Hard.Valid)
 552             {
 553                 UNIMPLEMENTED;
 554                 while (TRUE);
 555             }
 556             else
 557             {
 558                 /* Resolve the fault -- this will release the PFN lock */
 559                 ASSERT(PointerPte->u.Hard.Valid == 0);
 560                 Status = MiResolveProtoPteFault(StoreInstruction,
 561                                                 Address,
 562                                                 PointerPte,
 563                                                 PointerProtoPte,
 564                                                 NULL,
 565                                                 NULL,
 566                                                 NULL,
 567                                                 Process,
 568                                                 LockIrql,
 569                                                 TrapInformation);
 570                 ASSERT(Status == STATUS_SUCCESS);
 571
 572                 /* Complete this as a transition fault */
 573                 ASSERT(OldIrql == KeGetCurrentIrql());
 574                 ASSERT(OldIrql <= APC_LEVEL);
 575                 ASSERT(KeAreAllApcsDisabled() == TRUE);
 576                 return Status;
 577             }
 578         }
 579         else
 580         {
 581             /* We currently only handle very limited paths */
 582             ASSERT(PointerPte->u.Soft.Prototype == 1);
 583             ASSERT(PointerPte->u.Soft.PageFileHigh == MI_PTE_LOOKUP_NEEDED);
 584
 585             /* Lock the PFN database */
 586             LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 587
 588             /* For our current usage, this should be true */
 589             ASSERT(SuperProtoPte->u.Hard.Valid == 1);
 590             ASSERT(TempPte.u.Hard.Valid == 0);
 591
 592             /* Resolve the fault -- this will release the PFN lock */
 593             Status = MiResolveProtoPteFault(StoreInstruction,
 594                                             Address,
 595                                             PointerPte,
 596                                             PointerProtoPte,
 597                                             NULL,
 598                                             NULL,
 599                                             NULL,
 600                                             Process,
 601                                             LockIrql,
 602                                             TrapInformation);
 603             ASSERT(Status == STATUS_SUCCESS);
 604
 605             /* Complete this as a transition fault */
 606             ASSERT(OldIrql == KeGetCurrentIrql());
 607             ASSERT(OldIrql <= APC_LEVEL);
 608             ASSERT(KeAreAllApcsDisabled() == TRUE);
 609             return STATUS_PAGE_FAULT_TRANSITION;
 610         }
 611     }
 612
 613     //
 614     // The PTE must be invalid
 615     //
 616     ASSERT(TempPte.u.Hard.Valid == 0);
 617
 618     /* Check if the PTE is completely empty */
 619     if (TempPte.u.Long == 0)
 620     {
 621         KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 622                      (ULONG_PTR)Address,
 623                      StoreInstruction,
 624                      (ULONG_PTR)TrapInformation,
 625                      2);
 626     }
 627
 628     //
 629     // No prototype, transition or page file software PTEs in ARM3 yet
 630     //
 631     ASSERT(TempPte.u.Soft.Prototype == 0);
 632     ASSERT(TempPte.u.Soft.Transition == 0);
 633     ASSERT(TempPte.u.Soft.PageFileHigh == 0);
 634
 635     //
 636     // If we got this far, the PTE can only be a demand zero PTE, which is what
 637     // we want. Go handle it!
 638     //
 639     Status = MiResolveDemandZeroFault(Address,
 640                                       (ULONG)PointerPte->u.Soft.Protection,
 641                                       Process,
 642                                       MM_NOIRQL);
 643     ASSERT(KeAreAllApcsDisabled() == TRUE);
 644     if (NT_SUCCESS(Status))
 645     {
 646         //
 647         // Make sure we're returning in a sane state and pass the status down
 648         //
 649         ASSERT(OldIrql == KeGetCurrentIrql());
 650         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 651         return Status;
 652     }
 653
 654     //
 655     // Generate an access fault
 656     //
 657     return STATUS_ACCESS_VIOLATION;
 658 }
 659
 660 NTSTATUS
 661 NTAPI
 662 MmArmAccessFault(IN BOOLEAN StoreInstruction,
 663                  IN PVOID Address,
 664                  IN KPROCESSOR_MODE Mode,
 665                  IN PVOID TrapInformation)
 666 {
 667     KIRQL OldIrql = KeGetCurrentIrql(), LockIrql;
 668     PMMPTE ProtoPte = NULL;
 669     PMMPTE PointerPte = MiAddressToPte(Address);
 670     PMMPDE PointerPde = MiAddressToPde(Address);
 671 #if (_MI_PAGING_LEVELS >= 3)
 672     PMMPDE PointerPpe = MiAddressToPpe(Address);
 673 #if (_MI_PAGING_LEVELS == 4)
 674     PMMPDE PointerPxe = MiAddressToPxe(Address);
 675 #endif
 676 #endif
 677     MMPTE TempPte;
 678     PETHREAD CurrentThread;
 679     PEPROCESS CurrentProcess;
 680     NTSTATUS Status;
 681     PMMSUPPORT WorkingSet;
 682     ULONG ProtectionCode;
 683     PMMVAD Vad;
 684     PFN_NUMBER PageFrameIndex;
 685     ULONG Color;
 686
 687     DPRINT("ARM3 FAULT AT: %p\n", Address);
 688
 689     /* Check for page fault on high IRQL */
 690     if (OldIrql > APC_LEVEL)
 691     {
 692         // There are some special cases where this is okay, but not in ARM3 yet
 693         DbgPrint("MM:***PAGE FAULT AT IRQL > 1  Va %p, IRQL %lx\n",
 694                  Address,
 695                  OldIrql);
 696         ASSERT(OldIrql <= APC_LEVEL);
 697     }
 698
 699     /* Check for kernel fault address */
 700     if (Address >= MmSystemRangeStart)
 701     {
 702         /* Bail out, if the fault came from user mode */
 703         if (Mode == UserMode) return STATUS_ACCESS_VIOLATION;
 704
 705         /* PXEs and PPEs for kernel mode are mapped for everything we need */
 706 #if (_MI_PAGING_LEVELS >= 3)
 707         if (
 708 #if (_MI_PAGING_LEVELS == 4)
 709             (PointerPxe->u.Hard.Valid == 0) ||
 710 #endif
 711             (PointerPpe->u.Hard.Valid == 0))
 712         {
 713             /* The address is not from any pageable area! */
 714             KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 715                          (ULONG_PTR)Address,
 716                          StoreInstruction,
 717                          (ULONG_PTR)TrapInformation,
 718                          2);
 719         }
 720 #endif
 721
 722 #if (_MI_PAGING_LEVELS == 2)
 723         /* Check if we have a situation that might need synchronization
 724            of the PDE with the system page directory */
 725         if (MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address))
 726         {
 727             /* This could be a paged pool commit with an unsychronized PDE.
 728                NOTE: This way it works on x86, verify for other architectures! */
 729             if (MiSynchronizeSystemPde((PMMPDE)PointerPte)) return STATUS_SUCCESS;
 730         }
 731 #endif
 732
 733         /* Check if the PDE is invalid */
 734         if (PointerPde->u.Hard.Valid == 0)
 735         {
 736 #if (_MI_PAGING_LEVELS == 2)
 737             /* Sync this PDE and check, if that made it valid */
 738             if (!MiSynchronizeSystemPde(PointerPde))
 739 #endif
 740             {
 741                 /* PDE (still) not valid, kill the system */
 742                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 743                              (ULONG_PTR)Address,
 744                              StoreInstruction,
 745                              (ULONG_PTR)TrapInformation,
 746                              2);
 747             }
 748         }
 749
 750         /* The PDE is valid, so read the PTE */
 751         TempPte = *PointerPte;
 752         if (TempPte.u.Hard.Valid == 1)
 753         {
 754             //
 755             // Only two things can go wrong here:
 756             // Executing NX page (we couldn't care less)
 757             // Writing to a read-only page (the stuff ARM3 works with is write,
 758             // so again, moot point).
 759             //
 760
 761             //
 762             // Otherwise, the PDE was probably invalid, and all is good now
 763             //
 764             return STATUS_SUCCESS;
 765         }
 766
 767         // Check for a fault on the page table or hyperspace
 768         if (MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address))
 769         {
 770             ASSERT(TempPte.u.Long != (MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS));
 771             ASSERT(TempPte.u.Soft.Prototype == 0);
 772
 773             /* Use the process working set */
 774             CurrentProcess = (PEPROCESS)CurrentThread->Tcb.ApcState.Process;
 775             WorkingSet = &CurrentProcess->Vm;
 776         }
 777         else
 778         {
 779             /* Otherwise use the system working set */
 780             WorkingSet = &MmSystemCacheWs;
 781             CurrentProcess = NULL;
 782         }
 783
 784         /* Get the current thread */
 785         CurrentThread = PsGetCurrentThread();
 786
 787         /* Acquire the working set lock */
 788         KeRaiseIrql(APC_LEVEL, &LockIrql);
 789         MiLockWorkingSet(CurrentThread, WorkingSet);
 790
 791         /* Re-read PTE now that we own the lock */
 792         TempPte = *PointerPte;
 793         if (TempPte.u.Hard.Valid == 1)
 794         {
 795             // Only two things can go wrong here:
 796             // Executing NX page (we couldn't care less)
 797             // Writing to a read-only page (the stuff ARM3 works with is write,
 798             // so again, moot point).
 799             ASSERT(TempPte.u.Hard.Write == 1);
 800
 801             /* Release the working set */
 802             MiUnlockWorkingSet(CurrentThread, WorkingSet);
 803             KeLowerIrql(LockIrql);
 804
 805             // Otherwise, the PDE was probably invalid, and all is good now
 806             return STATUS_SUCCESS;
 807         }
 808
 809         /* Check one kind of prototype PTE */
 810         if (TempPte.u.Soft.Prototype)
 811         {
 812             /* Make sure protected pool is on, and that this is a pool address */
 813             if ((MmProtectFreedNonPagedPool) &&
 814                 (((Address >= MmNonPagedPoolStart) &&
 815                   (Address < (PVOID)((ULONG_PTR)MmNonPagedPoolStart +
 816                                      MmSizeOfNonPagedPoolInBytes))) ||
 817                  ((Address >= MmNonPagedPoolExpansionStart) &&
 818                   (Address < MmNonPagedPoolEnd))))
 819             {
 820                 /* Bad boy, bad boy, whatcha gonna do, whatcha gonna do when ARM3 comes for you! */
 821                 KeBugCheckEx(DRIVER_CAUGHT_MODIFYING_FREED_POOL,
 822                              (ULONG_PTR)Address,
 823                              StoreInstruction,
 824                              Mode,
 825                              4);
 826             }
 827
 828             /* Get the prototype PTE! */
 829             ProtoPte = MiProtoPteToPte(&TempPte);
 830         }
 831         else
 832         {
 833             /* We don't implement transition PTEs */
 834             ASSERT(TempPte.u.Soft.Transition == 0);
 835
 836             /* Check for no-access PTE */
 837             if (TempPte.u.Soft.Protection == MM_NOACCESS)
 838             {
 839                 /* Bugcheck the system! */
 840                 KeBugCheckEx(PAGE_FAULT_IN_NONPAGED_AREA,
 841                              (ULONG_PTR)Address,
 842                              StoreInstruction,
 843                              (ULONG_PTR)TrapInformation,
 844                              1);
 845             }
 846
 847             /* Check for demand page */
 848             if ((StoreInstruction) && !(TempPte.u.Hard.Valid))
 849             {
 850                 /* Get the protection code */
 851                 if (!(TempPte.u.Soft.Protection & MM_READWRITE))
 852                 {
 853                     /* Bugcheck the system! */
 854                     KeBugCheckEx(ATTEMPTED_WRITE_TO_READONLY_MEMORY,
 855                                  (ULONG_PTR)Address,
 856                                  TempPte.u.Long,
 857                                  (ULONG_PTR)TrapInformation,
 858                                  14);
 859                 }
 860             }
 861         }
 862
 863         /* Now do the real fault handling */
 864         Status = MiDispatchFault(StoreInstruction,
 865                                  Address,
 866                                  PointerPte,
 867                                  ProtoPte,
 868                                  FALSE,
 869                                  CurrentProcess,
 870                                  TrapInformation,
 871                                  NULL);
 872
 873         /* Release the working set */
 874         ASSERT(KeAreAllApcsDisabled() == TRUE);
 875         MiUnlockWorkingSet(CurrentThread, WorkingSet);
 876         KeLowerIrql(LockIrql);
 877
 878         /* We are done! */
 879         DPRINT("Fault resolved with status: %lx\n", Status);
 880         return Status;
 881     }
 882
 883     /* This is a user fault */
 884     CurrentThread = PsGetCurrentThread();
 885     CurrentProcess = (PEPROCESS)CurrentThread->Tcb.ApcState.Process;
 886
 887     /* Lock the working set */
 888     MiLockProcessWorkingSet(CurrentProcess, CurrentThread);
 889
 890 #if (_MI_PAGING_LEVELS == 2)
 891     ASSERT(PointerPde->u.Hard.LargePage == 0);
 892 #endif
 893
 894     /* Check if this address range belongs to a valid allocation (VAD) */
 895     ProtoPte = MiCheckVirtualAddress(Address, &ProtectionCode, &Vad);
 896     if (ProtectionCode == MM_NOACCESS)
 897     {
 898         /* This is a bogus VA */
 899         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 900         return STATUS_ACCESS_VIOLATION;
 901     }
 902
 903 #if (_MI_PAGING_LEVELS == 4)
 904     /* On these systems we have PXEs and PPEs ready for everything we need */
 905     if (PointerPxe->u.Hard.Valid == 0) return STATUS_ACCESS_VIOLATION;
 906 #endif
 907
 908 #if (_MI_PAGING_LEVELS >= 3)
 909     if (PointerPpe->u.Hard.Valid == 0) return STATUS_ACCESS_VIOLATION;
 910 #endif
 911
 912
 913     /* First things first, is the PDE valid? */
 914     if (PointerPde->u.Hard.Valid == 0)
 915     {
 916         /* Right now, we only handle scenarios where the PDE is totally empty */
 917         ASSERT(PointerPde->u.Long == 0);
 918
 919         /* Right now, we expect a valid protection mask on the VAD */
 920         ASSERT(ProtectionCode != MM_NOACCESS);
 921
 922         /* And go dispatch the fault on the PDE. This should handle the demand-zero */
 923 #if MI_TRACE_PFNS
 924         UserPdeFault = TRUE;
 925 #endif
 926         /* Resolve a demand zero fault */
 927         Status = MiResolveDemandZeroFault(PointerPte,
 928                                           MM_READWRITE,
 929                                           CurrentProcess,
 930                                           MM_NOIRQL);
 931 #if MI_TRACE_PFNS
 932         UserPdeFault = FALSE;
 933 #endif
 934         /* We should come back with APCs enabled, and with a valid PDE */
 935         ASSERT(KeAreAllApcsDisabled() == TRUE);
 936         ASSERT(PointerPde->u.Hard.Valid == 1);
 937     }
 938
 939     /* Now capture the PTE. Ignore virtual faults for now */
 940     TempPte = *PointerPte;
 941     ASSERT(TempPte.u.Hard.Valid == 0);
 942
 943     /* Quick check for demand-zero */
 944     if (TempPte.u.Long == (MM_READWRITE << MM_PTE_SOFTWARE_PROTECTION_BITS))
 945     {
 946         /* Resolve the fault */
 947         MI_WRITE_INVALID_PDE(PointerPde, DemandZeroPde);
 948         MiResolveDemandZeroFault(Address,
 949                                  (ULONG)PointerPte->u.Soft.Protection,
 950                                  CurrentProcess,
 951                                  MM_NOIRQL);
 952
 953         /* Return the status */
 954         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 955         return STATUS_PAGE_FAULT_DEMAND_ZERO;
 956     }
 957
 958     /* Make sure it's not a prototype PTE */
 959     ASSERT(TempPte.u.Soft.Prototype == 0);
 960
 961     /* Check for non-demand zero PTE */
 962     if (TempPte.u.Long != 0)
 963     {
 964         /* This is a page fault */
 965
 966         /* FIXME: Run MiAccessCheck */
 967
 968         /* Dispatch the fault */
 969         Status = MiDispatchFault(StoreInstruction,
 970                                  Address,
 971                                  PointerPte,
 972                                  NULL,
 973                                  FALSE,
 974                                  PsGetCurrentProcess(),
 975                                  TrapInformation,
 976                                  NULL);
 977
 978         /* Return the status */
 979         ASSERT(NT_SUCCESS(Status));
 980         ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 981         MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
 982         return Status;
 983     }
 984
 985
 986     {
 987         /* Add an additional page table reference */
 988         MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]++;
 989         ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT);
 990     }
 991
 992     /* Did we get a prototype PTE back? */
 993     if (!ProtoPte)
 994     {
 995         /* No, create a new PTE. First, write the protection */
 996         PointerPte->u.Soft.Protection = ProtectionCode;
 997
 998         /* Lock the PFN database since we're going to grab a page */
 999         OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1000
1001         /* Try to get a zero page */
1002         MI_SET_USAGE(MI_USAGE_PEB_TEB);
1003         MI_SET_PROCESS2(CurrentProcess->ImageFileName);
1004         Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess);
1005         PageFrameIndex = MiRemoveZeroPageSafe(Color);
1006         if (!PageFrameIndex)
1007         {
1008             /* Grab a page out of there. Later we should grab a colored zero page */
1009             PageFrameIndex = MiRemoveAnyPage(Color);
1010             ASSERT(PageFrameIndex);
1011
1012             /* Release the lock since we need to do some zeroing */
1013             KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1014
1015             /* Zero out the page, since it's for user-mode */
1016             MiZeroPfn(PageFrameIndex);
1017
1018             /* Grab the lock again so we can initialize the PFN entry */
1019             OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1020         }
1021
1022         /* Initialize the PFN entry now */
1023         MiInitializePfn(PageFrameIndex, PointerPte, 1);
1024
1025         /* One more demand-zero fault */
1026         KeGetCurrentPrcb()->MmDemandZeroCount++;
1027
1028         /* And we're done with the lock */
1029         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1030
1031         /* User fault, build a user PTE */
1032         MI_MAKE_HARDWARE_PTE_USER(&TempPte,
1033                                   PointerPte,
1034                                   PointerPte->u.Soft.Protection,
1035                                   PageFrameIndex);
1036
1037         /* Write the dirty bit for writeable pages */
1038         if (MI_IS_PAGE_WRITEABLE(&TempPte)) MI_MAKE_DIRTY_PAGE(&TempPte);
1039
1040         /* And now write down the PTE, making the address valid */
1041         MI_WRITE_VALID_PTE(PointerPte, TempPte);
1042
1043         /* Demand zero */
1044         Status = STATUS_PAGE_FAULT_DEMAND_ZERO;
1045     }
1046     else
1047     {
1048         /* No guard page support yet */
1049         ASSERT((ProtectionCode & MM_DECOMMIT) == 0);
1050         ASSERT(ProtectionCode != 0x100);
1051
1052         /* Write the prototype PTE */
1053         TempPte = PrototypePte;
1054         TempPte.u.Soft.Protection = ProtectionCode;
1055         MI_WRITE_INVALID_PTE(PointerPte, TempPte);
1056
1057         /* Handle the fault */
1058         Status = MiDispatchFault(StoreInstruction,
1059                                  Address,
1060                                  PointerPte,
1061                                  ProtoPte,
1062                                  FALSE,
1063                                  CurrentProcess,
1064                                  TrapInformation,
1065                                  Vad);
1066         ASSERT(Status == STATUS_PAGE_FAULT_TRANSITION);
1067         ASSERT(PointerPte->u.Hard.Valid == 1);
1068         ASSERT(PointerPte->u.Hard.PageFrameNumber != 0);
1069     }
1070
1071     /* Release the working set */
1072     MiUnlockProcessWorkingSet(CurrentProcess, CurrentThread);
1073     return Status;
1074 }
1075
1076 /* EOF */