ntoskrnl/mm/rmap.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/rmap.c
   5  * PURPOSE:         Kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #include <cache/section/newmm.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 /* TYPES ********************************************************************/
  18
  19 /* GLOBALS ******************************************************************/
  20
  21 static NPAGED_LOOKASIDE_LIST RmapLookasideList;
  22
  23 /* FUNCTIONS ****************************************************************/
  24
  25 _IRQL_requires_max_(DISPATCH_LEVEL)
  26 static
  27 VOID
  28 NTAPI
  29 RmapListFree(
  30     _In_ __drv_freesMem(Mem) PVOID P)
  31 {
  32     ExFreePoolWithTag(P, TAG_RMAP);
  33 }
  34
  35 CODE_SEG("INIT")
  36 VOID
  37 NTAPI
  38 MmInitializeRmapList(VOID)
  39 {
  40     ExInitializeNPagedLookasideList (&RmapLookasideList,
  41                                      NULL,
  42                                      RmapListFree,
  43                                      0,
  44                                      sizeof(MM_RMAP_ENTRY),
  45                                      TAG_RMAP,
  46                                      50);
  47 }
  48
  49 NTSTATUS
  50 NTAPI
  51 MmPageOutPhysicalAddress(PFN_NUMBER Page)
  52 {
  53     PMM_RMAP_ENTRY entry;
  54     PMEMORY_AREA MemoryArea;
  55     PMMSUPPORT AddressSpace;
  56     PVOID Address = NULL;
  57     PEPROCESS Process = NULL;
  58     NTSTATUS Status = STATUS_SUCCESS;
  59     PMM_SECTION_SEGMENT Segment;
  60     LARGE_INTEGER SegmentOffset;
  61     KIRQL OldIrql;
  62
  63 GetEntry:
  64     OldIrql = MiAcquirePfnLock();
  65
  66     entry = MmGetRmapListHeadPage(Page);
  67
  68     while (entry && RMAP_IS_SEGMENT(entry->Address))
  69         entry = entry->Next;
  70
  71     if (entry == NULL)
  72     {
  73         MiReleasePfnLock(OldIrql);
  74         goto WriteSegment;
  75     }
  76
  77     Process = entry->Process;
  78     Address = entry->Address;
  79
  80     if ((((ULONG_PTR)Address) & 0xFFF) != 0)
  81     {
  82         KeBugCheck(MEMORY_MANAGEMENT);
  83     }
  84
  85     /* This is for user-mode address only */
  86     ASSERT(Address < MmSystemRangeStart);
  87
  88     if (!ExAcquireRundownProtection(&Process->RundownProtect))
  89     {
  90         MiReleasePfnLock(OldIrql);
  91         return STATUS_PROCESS_IS_TERMINATING;
  92     }
  93
  94     Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode);
  95     MiReleasePfnLock(OldIrql);
  96     if (!NT_SUCCESS(Status))
  97     {
  98         ExReleaseRundownProtection(&Process->RundownProtect);
  99         return Status;
 100     }
 101     AddressSpace = &Process->Vm;
 102
 103     MmLockAddressSpace(AddressSpace);
 104
 105     MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
 106     if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
 107     {
 108         MmUnlockAddressSpace(AddressSpace);
 109         ExReleaseRundownProtection(&Process->RundownProtect);
 110         ObDereferenceObject(Process);
 111         goto GetEntry;
 112     }
 113
 114
 115     /* Attach to it, if needed */
 116     ASSERT(PsGetCurrentProcess() == PsInitialSystemProcess);
 117     if (Process != PsInitialSystemProcess)
 118         KeAttachProcess(&Process->Pcb);
 119
 120     if (MmGetPfnForProcess(Process, Address) != Page)
 121     {
 122         /* This changed in the short window where we didn't have any locks */
 123         if (Process != PsInitialSystemProcess)
 124             KeDetachProcess();
 125         MmUnlockAddressSpace(AddressSpace);
 126         ExReleaseRundownProtection(&Process->RundownProtect);
 127         ObDereferenceObject(Process);
 128         goto GetEntry;
 129     }
 130
 131     if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW)
 132     {
 133         ULONG_PTR Entry;
 134         BOOLEAN Dirty;
 135         PFN_NUMBER MapPage;
 136         LARGE_INTEGER Offset;
 137         BOOLEAN Released;
 138
 139         Offset.QuadPart = MemoryArea->SectionData.ViewOffset +
 140                  ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea));
 141
 142         Segment = MemoryArea->SectionData.Segment;
 143
 144         MmLockSectionSegment(Segment);
 145
 146         Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
 147         if (Entry && MM_IS_WAIT_PTE(Entry))
 148         {
 149             /* The segment is being read or something. Give up */
 150             MmUnlockSectionSegment(Segment);
 151             if (Process != PsInitialSystemProcess)
 152                 KeDetachProcess();
 153             MmUnlockAddressSpace(AddressSpace);
 154             ExReleaseRundownProtection(&Process->RundownProtect);
 155             ObDereferenceObject(Process);
 156             return(STATUS_UNSUCCESSFUL);
 157         }
 158
 159         /* Delete this virtual mapping in the process */
 160         MmDeleteRmap(Page, Process, Address);
 161         MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage);
 162
 163         /* We checked this earlier */
 164         ASSERT(MapPage == Page);
 165
 166         if (Page != PFN_FROM_SSE(Entry))
 167         {
 168             SWAPENTRY SwapEntry;
 169
 170             /* This page is private to the process */
 171             MmUnlockSectionSegment(Segment);
 172
 173             /* Check if we should write it back to the page file */
 174             SwapEntry = MmGetSavedSwapEntryPage(Page);
 175
 176             if ((SwapEntry == 0) && Dirty)
 177             {
 178                 /* We don't have a Swap entry, yet the page is dirty. Get one */
 179                 SwapEntry = MmAllocSwapPage();
 180                 if (!SwapEntry)
 181                 {
 182                     PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea),
 183                             &MemoryArea->SectionData.RegionListHead,
 184                             Address, NULL);
 185
 186                     /* We can't, so let this page in the Process VM */
 187                     MmCreateVirtualMapping(Process, Address, Region->Protect, Page);
 188                     MmInsertRmap(Page, Process, Address);
 189                     MmSetDirtyPage(Process, Address);
 190
 191                     MmUnlockAddressSpace(AddressSpace);
 192                     if (Process != PsInitialSystemProcess)
 193                         KeDetachProcess();
 194                     ExReleaseRundownProtection(&Process->RundownProtect);
 195                     ObDereferenceObject(Process);
 196
 197                     return STATUS_UNSUCCESSFUL;
 198                 }
 199             }
 200
 201             if (Dirty)
 202             {
 203                 SWAPENTRY Dummy;
 204
 205                 /* Put a wait entry into the process and unlock */
 206                 MmCreatePageFileMapping(Process, Address, MM_WAIT_ENTRY);
 207                 MmUnlockAddressSpace(AddressSpace);
 208
 209                 Status = MmWriteToSwapPage(SwapEntry, Page);
 210
 211                 MmLockAddressSpace(AddressSpace);
 212                 MmDeletePageFileMapping(Process, Address, &Dummy);
 213                 ASSERT(Dummy == MM_WAIT_ENTRY);
 214
 215                 if (!NT_SUCCESS(Status))
 216                 {
 217                     /* We failed at saving the content of this page. Keep it in */
 218                     PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea),
 219                             &MemoryArea->SectionData.RegionListHead,
 220                             Address, NULL);
 221
 222                     /* This Swap Entry is useless to us */
 223                     MmSetSavedSwapEntryPage(Page, 0);
 224                     MmFreeSwapPage(SwapEntry);
 225
 226                     /* We can't, so let this page in the Process VM */
 227                     MmCreateVirtualMapping(Process, Address, Region->Protect, Page);
 228                     MmInsertRmap(Page, Process, Address);
 229                     MmSetDirtyPage(Process, Address);
 230
 231                     MmUnlockAddressSpace(AddressSpace);
 232                     if (Process != PsInitialSystemProcess)
 233                         KeDetachProcess();
 234                     ExReleaseRundownProtection(&Process->RundownProtect);
 235                     ObDereferenceObject(Process);
 236
 237                     return STATUS_UNSUCCESSFUL;
 238                 }
 239             }
 240
 241             if (SwapEntry)
 242             {
 243                 /* Keep this in the process VM */
 244                 MmCreatePageFileMapping(Process, Address, SwapEntry);
 245                 MmSetSavedSwapEntryPage(Page, 0);
 246             }
 247
 248             /* We can finally let this page go */
 249             MmUnlockAddressSpace(AddressSpace);
 250             if (Process != PsInitialSystemProcess)
 251                 KeDetachProcess();
 252 #if DBG
 253             OldIrql = MiAcquirePfnLock();
 254             ASSERT(MmGetRmapListHeadPage(Page) == NULL);
 255             MiReleasePfnLock(OldIrql);
 256 #endif
 257             MmReleasePageMemoryConsumer(MC_USER, Page);
 258
 259             ExReleaseRundownProtection(&Process->RundownProtect);
 260             ObDereferenceObject(Process);
 261
 262             return STATUS_SUCCESS;
 263         }
 264
 265         /* One less mapping referencing this segment */
 266         Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, TRUE, NULL);
 267
 268         MmUnlockSectionSegment(Segment);
 269         if (Process != PsInitialSystemProcess)
 270             KeDetachProcess();
 271         MmUnlockAddressSpace(AddressSpace);
 272
 273         ExReleaseRundownProtection(&Process->RundownProtect);
 274         ObDereferenceObject(Process);
 275
 276         if (Released) return STATUS_SUCCESS;
 277     }
 278 #ifdef NEWCC
 279     else if (Type == MEMORY_AREA_CACHE)
 280     {
 281         /* NEWCC does locking itself */
 282         MmUnlockAddressSpace(AddressSpace);
 283         Status = MmpPageOutPhysicalAddress(Page);
 284     }
 285 #endif
 286     else
 287     {
 288         KeBugCheck(MEMORY_MANAGEMENT);
 289     }
 290
 291 WriteSegment:
 292     /* Now write this page to file, if needed */
 293     Segment = MmGetSectionAssociation(Page, &SegmentOffset);
 294     if (Segment)
 295     {
 296         BOOLEAN Released;
 297
 298         MmLockSectionSegment(Segment);
 299
 300         Released = MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE);
 301
 302         MmUnlockSectionSegment(Segment);
 303
 304         MmDereferenceSegment(Segment);
 305
 306         if (Released)
 307         {
 308             return STATUS_SUCCESS;
 309         }
 310     }
 311
 312     /* If we are here, then we didn't release the page */
 313     return STATUS_UNSUCCESSFUL;
 314 }
 315
 316 VOID
 317 NTAPI
 318 MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process,
 319              PVOID Address)
 320 {
 321     PMM_RMAP_ENTRY current_entry;
 322     PMM_RMAP_ENTRY new_entry;
 323     ULONG PrevSize;
 324     KIRQL OldIrql;
 325
 326     if (!RMAP_IS_SEGMENT(Address))
 327         Address = (PVOID)PAGE_ROUND_DOWN(Address);
 328
 329     new_entry = ExAllocateFromNPagedLookasideList(&RmapLookasideList);
 330     if (new_entry == NULL)
 331     {
 332         KeBugCheck(MEMORY_MANAGEMENT);
 333     }
 334     new_entry->Address = Address;
 335     new_entry->Process = (PEPROCESS)Process;
 336 #if DBG
 337     new_entry->Caller = _ReturnAddress();
 338 #endif
 339
 340     if (
 341         !RMAP_IS_SEGMENT(Address) &&
 342         MmGetPfnForProcess(Process, Address) != Page)
 343     {
 344         DPRINT1("Insert rmap (%d, 0x%.8X) 0x%.8X which doesn't match physical "
 345                 "address 0x%.8X\n", Process ? Process->UniqueProcessId : 0,
 346                 Address,
 347                 MmGetPfnForProcess(Process, Address) << PAGE_SHIFT,
 348                 Page << PAGE_SHIFT);
 349         KeBugCheck(MEMORY_MANAGEMENT);
 350     }
 351
 352     OldIrql = MiAcquirePfnLock();
 353     current_entry = MmGetRmapListHeadPage(Page);
 354
 355     PMM_RMAP_ENTRY previous_entry = NULL;
 356     /* Keep the list sorted */
 357     while (current_entry && (current_entry->Address < Address))
 358     {
 359         previous_entry = current_entry;
 360         current_entry = current_entry->Next;
 361     }
 362
 363     /* In case of clash in the address, sort by process */
 364     if (current_entry && (current_entry->Address == Address))
 365     {
 366         while (current_entry && (current_entry->Process < Process))
 367         {
 368             previous_entry = current_entry;
 369             current_entry = current_entry->Next;
 370         }
 371     }
 372
 373     if (current_entry && (current_entry->Address == Address) && (current_entry->Process == Process))
 374     {
 375 #if DBG
 376         DbgPrint("MmInsertRmap tries to add a second rmap entry for address %p\n", current_entry->Address);
 377         DbgPrint("    current caller  %p\n", new_entry->Caller);
 378         DbgPrint("    previous caller %p\n", current_entry->Caller);
 379 #endif
 380         KeBugCheck(MEMORY_MANAGEMENT);
 381     }
 382
 383     new_entry->Next = current_entry;
 384     if (previous_entry)
 385         previous_entry->Next = new_entry;
 386     else
 387         MmSetRmapListHeadPage(Page, new_entry);
 388
 389     MiReleasePfnLock(OldIrql);
 390
 391     if (!RMAP_IS_SEGMENT(Address))
 392     {
 393         ASSERT(Process != NULL);
 394         PrevSize = InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, PAGE_SIZE);
 395         if (PrevSize >= Process->Vm.PeakWorkingSetSize)
 396         {
 397             Process->Vm.PeakWorkingSetSize = PrevSize + PAGE_SIZE;
 398         }
 399     }
 400 }
 401
 402 VOID
 403 NTAPI
 404 MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process,
 405              PVOID Address)
 406 {
 407     PMM_RMAP_ENTRY current_entry, previous_entry;
 408     KIRQL OldIrql;
 409
 410     OldIrql = MiAcquirePfnLock();
 411     previous_entry = NULL;
 412     current_entry = MmGetRmapListHeadPage(Page);
 413
 414     while (current_entry != NULL)
 415     {
 416         if (current_entry->Process == (PEPROCESS)Process &&
 417                 current_entry->Address == Address)
 418         {
 419             if (previous_entry == NULL)
 420             {
 421                 MmSetRmapListHeadPage(Page, current_entry->Next);
 422             }
 423             else
 424             {
 425                 previous_entry->Next = current_entry->Next;
 426             }
 427             MiReleasePfnLock(OldIrql);
 428
 429             ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
 430             if (!RMAP_IS_SEGMENT(Address))
 431             {
 432                 ASSERT(Process != NULL);
 433                 (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE);
 434             }
 435             return;
 436         }
 437         previous_entry = current_entry;
 438         current_entry = current_entry->Next;
 439     }
 440     KeBugCheck(MEMORY_MANAGEMENT);
 441 }
 442
 443 /*
 444
 445 Return the process pointer given when a previous call to MmInsertRmap was
 446 called with a process and address pointer that conform to the segment rmap
 447 schema.  In short, this requires the address part to be 0xffffff00 + n
 448 where n is between 0 and 255.  When such an rmap exists, it specifies a
 449 segment rmap in which the process part is a pointer to a slice of a section
 450 page table, and the low 8 bits of the address represent a page index in the
 451 page table slice.  Together, this information is used by
 452 MmGetSectionAssociation to determine which page entry points to this page in
 453 the segment page table.
 454
 455 */
 456
 457 PVOID
 458 NTAPI
 459 MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset)
 460 {
 461     PCACHE_SECTION_PAGE_TABLE Result = NULL;
 462     PMM_RMAP_ENTRY current_entry;//, previous_entry;
 463     KIRQL OldIrql = MiAcquirePfnLock();
 464
 465     //previous_entry = NULL;
 466     current_entry = MmGetRmapListHeadPage(Page);
 467     while (current_entry != NULL)
 468     {
 469         if (RMAP_IS_SEGMENT(current_entry->Address))
 470         {
 471             Result = (PCACHE_SECTION_PAGE_TABLE)current_entry->Process;
 472             *RawOffset = (ULONG_PTR)current_entry->Address & ~RMAP_SEGMENT_MASK;
 473             if (*Result->Segment->Flags & MM_SEGMENT_INDELETE)
 474             {
 475                 MiReleasePfnLock(OldIrql);
 476                 return NULL;
 477             }
 478
 479             InterlockedIncrement64(Result->Segment->ReferenceCount);
 480             MiReleasePfnLock(OldIrql);
 481             return Result;
 482         }
 483         //previous_entry = current_entry;
 484         current_entry = current_entry->Next;
 485     }
 486     MiReleasePfnLock(OldIrql);
 487     return NULL;
 488 }
 489
 490 /*
 491
 492 Remove the section rmap associated with the indicated page, if it exists.
 493
 494 */
 495
 496 VOID
 497 NTAPI
 498 MmDeleteSectionAssociation(PFN_NUMBER Page)
 499 {
 500     PMM_RMAP_ENTRY current_entry, previous_entry;
 501     KIRQL OldIrql = MiAcquirePfnLock();
 502
 503     previous_entry = NULL;
 504     current_entry = MmGetRmapListHeadPage(Page);
 505     while (current_entry != NULL)
 506     {
 507         if (RMAP_IS_SEGMENT(current_entry->Address))
 508         {
 509             if (previous_entry == NULL)
 510             {
 511                 MmSetRmapListHeadPage(Page, current_entry->Next);
 512             }
 513             else
 514             {
 515                 previous_entry->Next = current_entry->Next;
 516             }
 517             MiReleasePfnLock(OldIrql);
 518             ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
 519             return;
 520         }
 521         previous_entry = current_entry;
 522         current_entry = current_entry->Next;
 523     }
 524     MiReleasePfnLock(OldIrql);
 525 }