reactos/ntoskrnl/mm/rmap.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/rmap.c
   5  * PURPOSE:         Kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #include "../cache/section/newmm.h"
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #if defined (ALLOC_PRAGMA)
  18 #pragma alloc_text(INIT, MmInitializeRmapList)
  19 #endif
  20
  21 /* TYPES ********************************************************************/
  22
  23 /* GLOBALS ******************************************************************/
  24
  25 static NPAGED_LOOKASIDE_LIST RmapLookasideList;
  26 FAST_MUTEX RmapListLock;
  27
  28 /* FUNCTIONS ****************************************************************/
  29
  30 _IRQL_requires_max_(DISPATCH_LEVEL)
  31 static
  32 VOID
  33 NTAPI
  34 RmapListFree(
  35     _In_ __drv_freesMem(Mem) PVOID P)
  36 {
  37     ExFreePoolWithTag(P, TAG_RMAP);
  38 }
  39
  40 VOID
  41 INIT_FUNCTION
  42 NTAPI
  43 MmInitializeRmapList(VOID)
  44 {
  45    ExInitializeFastMutex(&RmapListLock);
  46    ExInitializeNPagedLookasideList (&RmapLookasideList,
  47                                     NULL,
  48                                     RmapListFree,
  49                                     0,
  50                                     sizeof(MM_RMAP_ENTRY),
  51                                     TAG_RMAP,
  52                                     50);
  53 }
  54
  55 NTSTATUS
  56 NTAPI
  57 MmPageOutPhysicalAddress(PFN_NUMBER Page)
  58 {
  59    PMM_RMAP_ENTRY entry;
  60    PMEMORY_AREA MemoryArea;
  61    PMMSUPPORT AddressSpace;
  62    ULONG Type;
  63    PVOID Address;
  64    PEPROCESS Process;
  65    ULONGLONG Offset;
  66    NTSTATUS Status = STATUS_SUCCESS;
  67
  68    ExAcquireFastMutex(&RmapListLock);
  69    entry = MmGetRmapListHeadPage(Page);
  70
  71 #ifdef NEWCC
  72    // Special case for NEWCC: we can have a page that's only in a segment
  73    // page table
  74    if (entry && RMAP_IS_SEGMENT(entry->Address) && entry->Next == NULL)
  75    {
  76        /* NEWCC does locking itself */
  77        ExReleaseFastMutex(&RmapListLock);
  78        return MmpPageOutPhysicalAddress(Page);
  79    }
  80 #endif
  81
  82    while (entry && RMAP_IS_SEGMENT(entry->Address))
  83        entry = entry->Next;
  84
  85    if (entry == NULL)
  86    {
  87       ExReleaseFastMutex(&RmapListLock);
  88       return(STATUS_UNSUCCESSFUL);
  89    }
  90
  91    Process = entry->Process;
  92
  93    Address = entry->Address;
  94
  95    if ((((ULONG_PTR)Address) & 0xFFF) != 0)
  96    {
  97       KeBugCheck(MEMORY_MANAGEMENT);
  98    }
  99
 100    if (Address < MmSystemRangeStart)
 101    {
 102        if (!ExAcquireRundownProtection(&Process->RundownProtect))
 103        {
 104           ExReleaseFastMutex(&RmapListLock);
 105           return STATUS_PROCESS_IS_TERMINATING;
 106        }
 107
 108       Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode);
 109       ExReleaseFastMutex(&RmapListLock);
 110       if (!NT_SUCCESS(Status))
 111       {
 112          ExReleaseRundownProtection(&Process->RundownProtect);
 113          return Status;
 114       }
 115       AddressSpace = &Process->Vm;
 116    }
 117    else
 118    {
 119       ExReleaseFastMutex(&RmapListLock);
 120       AddressSpace = MmGetKernelAddressSpace();
 121    }
 122
 123    MmLockAddressSpace(AddressSpace);
 124    MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
 125    if (MemoryArea == NULL || MemoryArea->DeleteInProgress)
 126    {
 127       MmUnlockAddressSpace(AddressSpace);
 128       if (Address < MmSystemRangeStart)
 129       {
 130          ExReleaseRundownProtection(&Process->RundownProtect);
 131          ObDereferenceObject(Process);
 132       }
 133       return(STATUS_UNSUCCESSFUL);
 134    }
 135    Type = MemoryArea->Type;
 136    if (Type == MEMORY_AREA_SECTION_VIEW)
 137    {
 138       ULONG_PTR Entry;
 139       Offset = MemoryArea->Data.SectionData.ViewOffset.QuadPart +
 140                ((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress);
 141
 142       MmLockSectionSegment(MemoryArea->Data.SectionData.Segment);
 143
 144       /*
 145        * Get or create a pageop
 146        */
 147       Entry = MmGetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment,
 148                                            (PLARGE_INTEGER)&Offset);
 149       if (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY)
 150       {
 151          MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment);
 152          MmUnlockAddressSpace(AddressSpace);
 153          if (Address < MmSystemRangeStart)
 154          {
 155             ExReleaseRundownProtection(&Process->RundownProtect);
 156             ObDereferenceObject(Process);
 157          }
 158          return(STATUS_UNSUCCESSFUL);
 159       }
 160
 161       MmSetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));
 162
 163       /*
 164        * Release locks now we have a page op.
 165        */
 166       MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment);
 167       MmUnlockAddressSpace(AddressSpace);
 168
 169       /*
 170        * Do the actual page out work.
 171        */
 172       Status = MmPageOutSectionView(AddressSpace, MemoryArea, Address, Entry);
 173    }
 174    else if (Type == MEMORY_AREA_CACHE)
 175    {
 176        /* NEWCC does locking itself */
 177        MmUnlockAddressSpace(AddressSpace);
 178        Status = MmpPageOutPhysicalAddress(Page);
 179    }
 180    else
 181    {
 182       KeBugCheck(MEMORY_MANAGEMENT);
 183    }
 184
 185    if (Address < MmSystemRangeStart)
 186    {
 187       ExReleaseRundownProtection(&Process->RundownProtect);
 188       ObDereferenceObject(Process);
 189    }
 190    return(Status);
 191 }
 192
 193 VOID
 194 NTAPI
 195 MmSetCleanAllRmaps(PFN_NUMBER Page)
 196 {
 197    PMM_RMAP_ENTRY current_entry;
 198
 199    ExAcquireFastMutex(&RmapListLock);
 200    current_entry = MmGetRmapListHeadPage(Page);
 201    if (current_entry == NULL)
 202    {
 203       DPRINT1("MmIsDirtyRmap: No rmaps.\n");
 204       KeBugCheck(MEMORY_MANAGEMENT);
 205    }
 206    while (current_entry != NULL)
 207    {
 208       if (!RMAP_IS_SEGMENT(current_entry->Address))
 209          MmSetCleanPage(current_entry->Process, current_entry->Address);
 210       current_entry = current_entry->Next;
 211    }
 212    ExReleaseFastMutex(&RmapListLock);
 213 }
 214
 215 VOID
 216 NTAPI
 217 MmSetDirtyAllRmaps(PFN_NUMBER Page)
 218 {
 219    PMM_RMAP_ENTRY current_entry;
 220
 221    ExAcquireFastMutex(&RmapListLock);
 222    current_entry = MmGetRmapListHeadPage(Page);
 223    if (current_entry == NULL)
 224    {
 225       DPRINT1("MmIsDirtyRmap: No rmaps.\n");
 226       KeBugCheck(MEMORY_MANAGEMENT);
 227    }
 228    while (current_entry != NULL)
 229    {
 230       if (!RMAP_IS_SEGMENT(current_entry->Address))
 231          MmSetDirtyPage(current_entry->Process, current_entry->Address);
 232       current_entry = current_entry->Next;
 233    }
 234    ExReleaseFastMutex(&RmapListLock);
 235 }
 236
 237 BOOLEAN
 238 NTAPI
 239 MmIsDirtyPageRmap(PFN_NUMBER Page)
 240 {
 241    PMM_RMAP_ENTRY current_entry;
 242
 243    ExAcquireFastMutex(&RmapListLock);
 244    current_entry = MmGetRmapListHeadPage(Page);
 245    if (current_entry == NULL)
 246    {
 247       ExReleaseFastMutex(&RmapListLock);
 248       return(FALSE);
 249    }
 250    while (current_entry != NULL)
 251    {
 252       if (
 253           !RMAP_IS_SEGMENT(current_entry->Address) &&
 254           MmIsDirtyPage(current_entry->Process, current_entry->Address))
 255       {
 256          ExReleaseFastMutex(&RmapListLock);
 257          return(TRUE);
 258       }
 259       current_entry = current_entry->Next;
 260    }
 261    ExReleaseFastMutex(&RmapListLock);
 262    return(FALSE);
 263 }
 264
 265 VOID
 266 NTAPI
 267 MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process,
 268              PVOID Address)
 269 {
 270    PMM_RMAP_ENTRY current_entry;
 271    PMM_RMAP_ENTRY new_entry;
 272    ULONG PrevSize;
 273    if (!RMAP_IS_SEGMENT(Address))
 274       Address = (PVOID)PAGE_ROUND_DOWN(Address);
 275
 276    new_entry = ExAllocateFromNPagedLookasideList(&RmapLookasideList);
 277    if (new_entry == NULL)
 278    {
 279       KeBugCheck(MEMORY_MANAGEMENT);
 280    }
 281    new_entry->Address = Address;
 282    new_entry->Process = (PEPROCESS)Process;
 283 #if DBG
 284 #ifdef __GNUC__
 285    new_entry->Caller = __builtin_return_address(0);
 286 #else
 287    new_entry->Caller = _ReturnAddress();
 288 #endif
 289 #endif
 290
 291    if (
 292        !RMAP_IS_SEGMENT(Address) &&
 293        MmGetPfnForProcess(Process, Address) != Page)
 294    {
 295       DPRINT1("Insert rmap (%d, 0x%.8X) 0x%.8X which doesn't match physical "
 296               "address 0x%.8X\n", Process ? Process->UniqueProcessId : 0,
 297               Address,
 298               MmGetPfnForProcess(Process, Address) << PAGE_SHIFT,
 299               Page << PAGE_SHIFT);
 300       KeBugCheck(MEMORY_MANAGEMENT);
 301    }
 302
 303    ExAcquireFastMutex(&RmapListLock);
 304    current_entry = MmGetRmapListHeadPage(Page);
 305    new_entry->Next = current_entry;
 306 #if DBG
 307    while (current_entry)
 308    {
 309       if (current_entry->Address == new_entry->Address && current_entry->Process == new_entry->Process)
 310       {
 311           DbgPrint("MmInsertRmap tries to add a second rmap entry for address %p\n    current caller ",
 312                    current_entry->Address);
 313           DbgPrint("%p", new_entry->Caller);
 314           DbgPrint("\n    previous caller ");
 315           DbgPrint("%p", current_entry->Caller);
 316           DbgPrint("\n");
 317           KeBugCheck(MEMORY_MANAGEMENT);
 318       }
 319       current_entry = current_entry->Next;
 320    }
 321 #endif
 322    MmSetRmapListHeadPage(Page, new_entry);
 323    ExReleaseFastMutex(&RmapListLock);
 324    if (!RMAP_IS_SEGMENT(Address))
 325    {
 326       if (Process == NULL)
 327       {
 328          Process = PsInitialSystemProcess;
 329       }
 330       if (Process)
 331       {
 332          PrevSize = InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, PAGE_SIZE);
 333          if (PrevSize >= Process->Vm.PeakWorkingSetSize)
 334          {
 335             Process->Vm.PeakWorkingSetSize = PrevSize + PAGE_SIZE;
 336          }
 337       }
 338    }
 339 }
 340
 341 VOID
 342 NTAPI
 343 MmDeleteAllRmaps(PFN_NUMBER Page, PVOID Context,
 344                  VOID (*DeleteMapping)(PVOID Context, PEPROCESS Process,
 345                                        PVOID Address))
 346 {
 347    PMM_RMAP_ENTRY current_entry;
 348    PMM_RMAP_ENTRY previous_entry;
 349    PEPROCESS Process;
 350
 351    ExAcquireFastMutex(&RmapListLock);
 352    current_entry = MmGetRmapListHeadPage(Page);
 353    if (current_entry == NULL)
 354    {
 355       DPRINT1("MmDeleteAllRmaps: No rmaps.\n");
 356       KeBugCheck(MEMORY_MANAGEMENT);
 357    }
 358    MmSetRmapListHeadPage(Page, NULL);
 359    ExReleaseFastMutex(&RmapListLock);
 360
 361    while (current_entry != NULL)
 362    {
 363       previous_entry = current_entry;
 364       current_entry = current_entry->Next;
 365       if (!RMAP_IS_SEGMENT(previous_entry->Address))
 366       {
 367          if (DeleteMapping)
 368          {
 369             DeleteMapping(Context, previous_entry->Process,
 370                           previous_entry->Address);
 371          }
 372          Process = previous_entry->Process;
 373          ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry);
 374          if (Process == NULL)
 375          {
 376             Process = PsInitialSystemProcess;
 377          }
 378          if (Process)
 379          {
 380             (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE);
 381          }
 382       }
 383       else
 384       {
 385          ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry);
 386       }
 387    }
 388 }
 389
 390 VOID
 391 NTAPI
 392 MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process,
 393              PVOID Address)
 394 {
 395    PMM_RMAP_ENTRY current_entry, previous_entry;
 396
 397    ExAcquireFastMutex(&RmapListLock);
 398    previous_entry = NULL;
 399    current_entry = MmGetRmapListHeadPage(Page);
 400
 401    while (current_entry != NULL)
 402    {
 403       if (current_entry->Process == (PEPROCESS)Process &&
 404             current_entry->Address == Address)
 405       {
 406          if (previous_entry == NULL)
 407          {
 408             MmSetRmapListHeadPage(Page, current_entry->Next);
 409          }
 410          else
 411          {
 412             previous_entry->Next = current_entry->Next;
 413          }
 414          ExReleaseFastMutex(&RmapListLock);
 415          ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
 416          if (!RMAP_IS_SEGMENT(Address))
 417          {
 418             if (Process == NULL)
 419             {
 420                Process = PsInitialSystemProcess;
 421             }
 422             if (Process)
 423             {
 424                (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE);
 425             }
 426          }
 427          return;
 428       }
 429       previous_entry = current_entry;
 430       current_entry = current_entry->Next;
 431    }
 432    KeBugCheck(MEMORY_MANAGEMENT);
 433 }
 434
 435 /*
 436
 437 Return the process pointer given when a previous call to MmInsertRmap was
 438 called with a process and address pointer that conform to the segment rmap
 439 schema.  In short, this requires the address part to be 0xffffff00 + n
 440 where n is between 0 and 255.  When such an rmap exists, it specifies a
 441 segment rmap in which the process part is a pointer to a slice of a section
 442 page table, and the low 8 bits of the address represent a page index in the
 443 page table slice.  Together, this information is used by
 444 MmGetSectionAssociation to determine which page entry points to this page in
 445 the segment page table.
 446
 447 */
 448
 449 PVOID
 450 NTAPI
 451 MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset)
 452 {
 453    PCACHE_SECTION_PAGE_TABLE Result = NULL;
 454    PMM_RMAP_ENTRY current_entry, previous_entry;
 455
 456    ExAcquireFastMutex(&RmapListLock);
 457    previous_entry = NULL;
 458    current_entry = MmGetRmapListHeadPage(Page);
 459    while (current_entry != NULL)
 460    {
 461       if (RMAP_IS_SEGMENT(current_entry->Address))
 462       {
 463          Result = (PCACHE_SECTION_PAGE_TABLE)current_entry->Process;
 464          *RawOffset = (ULONG_PTR)current_entry->Address & ~RMAP_SEGMENT_MASK;
 465          InterlockedIncrementUL(&Result->Segment->ReferenceCount);
 466          ExReleaseFastMutex(&RmapListLock);
 467          return Result;
 468       }
 469       previous_entry = current_entry;
 470       current_entry = current_entry->Next;
 471    }
 472    ExReleaseFastMutex(&RmapListLock);
 473    return NULL;
 474 }
 475
 476 /*
 477
 478 Remove the section rmap associated with the indicated page, if it exists.
 479
 480 */
 481
 482 VOID
 483 NTAPI
 484 MmDeleteSectionAssociation(PFN_NUMBER Page)
 485 {
 486    PMM_RMAP_ENTRY current_entry, previous_entry;
 487
 488    ExAcquireFastMutex(&RmapListLock);
 489    previous_entry = NULL;
 490    current_entry = MmGetRmapListHeadPage(Page);
 491    while (current_entry != NULL)
 492    {
 493       if (RMAP_IS_SEGMENT(current_entry->Address))
 494       {
 495          if (previous_entry == NULL)
 496          {
 497             MmSetRmapListHeadPage(Page, current_entry->Next);
 498          }
 499          else
 500          {
 501             previous_entry->Next = current_entry->Next;
 502          }
 503          ExReleaseFastMutex(&RmapListLock);
 504          ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry);
 505          return;
 506       }
 507       previous_entry = current_entry;
 508       current_entry = current_entry->Next;
 509    }
 510    ExReleaseFastMutex(&RmapListLock);
 511 }