ntoskrnl/mm/freelist.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/freelist.c
   5  * PURPOSE:         Handle the list of free physical pages
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Robert Bergkvist
   9  */
  10
  11 /* INCLUDES ****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #if defined (ALLOC_PRAGMA)
  18 #pragma alloc_text(INIT, MmInitializePageList)
  19 #endif
  20
  21 #define MODULE_INVOLVED_IN_ARM3
  22 #include "ARM3/miarm.h"
  23
  24 /* GLOBALS ****************************************************************/
  25
  26 // ReactOS to NT Physical Page Descriptor Entry Legacy Mapping Definitions
  27 #define PHYSICAL_PAGE        MMPFN
  28 #define PPHYSICAL_PAGE       PMMPFN
  29
  30 PPHYSICAL_PAGE MmPfnDatabase;
  31
  32 PFN_NUMBER MmAvailablePages;
  33 PFN_NUMBER MmResidentAvailablePages;
  34 PFN_NUMBER MmResidentAvailableAtInit;
  35
  36 SIZE_T MmTotalCommitLimit;
  37 SIZE_T MmTotalCommittedPages;
  38 SIZE_T MmSharedCommit;
  39 SIZE_T MmDriverCommit;
  40 SIZE_T MmProcessCommit;
  41 SIZE_T MmPagedPoolCommit;
  42 SIZE_T MmPeakCommitment;
  43 SIZE_T MmtotalCommitLimitMaximum;
  44
  45 static RTL_BITMAP MiUserPfnBitMap;
  46
  47 /* FUNCTIONS *************************************************************/
  48
  49 VOID
  50 NTAPI
  51 MiInitializeUserPfnBitmap(VOID)
  52 {
  53     PVOID Bitmap;
  54
  55     /* Allocate enough buffer for the PFN bitmap and align it on 32-bits */
  56     Bitmap = ExAllocatePoolWithTag(NonPagedPool,
  57                                    (((MmHighestPhysicalPage + 1) + 31) / 32) * 4,
  58                                    '  mM');
  59     ASSERT(Bitmap);
  60
  61     /* Initialize it and clear all the bits to begin with */
  62     RtlInitializeBitMap(&MiUserPfnBitMap,
  63                         Bitmap,
  64                         MmHighestPhysicalPage + 1);
  65     RtlClearAllBits(&MiUserPfnBitMap);
  66 }
  67
  68 PFN_NUMBER
  69 NTAPI
  70 MmGetLRUFirstUserPage(VOID)
  71 {
  72     ULONG Position;
  73     KIRQL OldIrql;
  74
  75     /* Find the first user page */
  76     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
  77     Position = RtlFindSetBits(&MiUserPfnBitMap, 1, 0);
  78     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
  79     if (Position == 0xFFFFFFFF) return 0;
  80
  81     /* Return it */
  82     return Position;
  83 }
  84
  85 VOID
  86 NTAPI
  87 MmInsertLRULastUserPage(PFN_NUMBER Pfn)
  88 {
  89     KIRQL OldIrql;
  90
  91     /* Set the page as a user page */
  92     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
  93     RtlSetBit(&MiUserPfnBitMap, Pfn);
  94     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
  95 }
  96
  97 PFN_NUMBER
  98 NTAPI
  99 MmGetLRUNextUserPage(PFN_NUMBER PreviousPfn)
 100 {
 101     ULONG Position;
 102     KIRQL OldIrql;
 103
 104     /* Find the next user page */
 105     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 106     Position = RtlFindSetBits(&MiUserPfnBitMap, 1, PreviousPfn + 1);
 107     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 108     if (Position == 0xFFFFFFFF) return 0;
 109
 110     /* Return it */
 111     return Position;
 112 }
 113
 114 VOID
 115 NTAPI
 116 MmRemoveLRUUserPage(PFN_NUMBER Page)
 117 {
 118     /* Unset the page as a user page */
 119     RtlClearBit(&MiUserPfnBitMap, Page);
 120 }
 121
 122 BOOLEAN
 123 NTAPI
 124 MiIsPfnFree(IN PMMPFN Pfn1)
 125 {
 126     /* Must be a free or zero page, with no references, linked */
 127     return ((Pfn1->u3.e1.PageLocation <= StandbyPageList) &&
 128             (Pfn1->u1.Flink) &&
 129             (Pfn1->u2.Blink) &&
 130             !(Pfn1->u3.e2.ReferenceCount));
 131 }
 132
 133 BOOLEAN
 134 NTAPI
 135 MiIsPfnInUse(IN PMMPFN Pfn1)
 136 {
 137     /* Standby list or higher, unlinked, and with references */
 138     return !MiIsPfnFree(Pfn1);
 139 }
 140
 141 PMDL
 142 NTAPI
 143 MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress,
 144                       IN PHYSICAL_ADDRESS HighAddress,
 145                       IN PHYSICAL_ADDRESS SkipBytes,
 146                       IN SIZE_T TotalBytes,
 147                       IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
 148                       IN ULONG MdlFlags)
 149 {
 150     PMDL Mdl;
 151     PFN_NUMBER PageCount, LowPage, HighPage, SkipPages, PagesFound = 0, Page;
 152     PPFN_NUMBER MdlPage, LastMdlPage;
 153     KIRQL OldIrql;
 154     PPHYSICAL_PAGE Pfn1;
 155     INT LookForZeroedPages;
 156     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 157
 158     //
 159     // Convert the low address into a PFN
 160     //
 161     LowPage = (PFN_NUMBER)(LowAddress.QuadPart >> PAGE_SHIFT);
 162
 163     //
 164     // Convert, and normalize, the high address into a PFN
 165     //
 166     HighPage = (PFN_NUMBER)(HighAddress.QuadPart >> PAGE_SHIFT);
 167     if (HighPage > MmHighestPhysicalPage) HighPage = MmHighestPhysicalPage;
 168
 169     //
 170     // Validate skipbytes and convert them into pages
 171     //
 172     if (BYTE_OFFSET(SkipBytes.LowPart)) return NULL;
 173     SkipPages = (PFN_NUMBER)(SkipBytes.QuadPart >> PAGE_SHIFT);
 174
 175     /* This isn't supported at all */
 176     if (SkipPages) DPRINT1("WARNING: Caller requesting SkipBytes, MDL might be mismatched\n");
 177
 178     //
 179     // Now compute the number of pages the MDL will cover
 180     //
 181     PageCount = (PFN_NUMBER)ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, TotalBytes);
 182     do
 183     {
 184         //
 185         // Try creating an MDL for these many pages
 186         //
 187         Mdl = MmCreateMdl(NULL, NULL, PageCount << PAGE_SHIFT);
 188         if (Mdl) break;
 189
 190         //
 191         // This function is not required to return the amount of pages requested
 192         // In fact, it can return as little as 1 page, and callers are supposed
 193         // to deal with this scenario. So re-attempt the allocation with less
 194         // pages than before, and see if it worked this time.
 195         //
 196         PageCount -= (PageCount >> 4);
 197     } while (PageCount);
 198
 199     //
 200     // Wow, not even a single page was around!
 201     //
 202     if (!Mdl) return NULL;
 203
 204     //
 205     // This is where the page array starts....
 206     //
 207     MdlPage = (PPFN_NUMBER)(Mdl + 1);
 208
 209     //
 210     // Lock the PFN database
 211     //
 212     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 213
 214     //
 215     // Are we looking for any pages, without discriminating?
 216     //
 217     if ((LowPage == 0) && (HighPage == MmHighestPhysicalPage))
 218     {
 219         //
 220         // Well then, let's go shopping
 221         //
 222         while (PagesFound < PageCount)
 223         {
 224             /* Grab a page */
 225             Page = MiRemoveAnyPage(0);
 226             if (Page == 0)
 227             {
 228                 /* This is not good... hopefully we have at least SOME pages */
 229                 ASSERT(PagesFound);
 230                 break;
 231             }
 232
 233             /* Grab the page entry for it */
 234             Pfn1 = MiGetPfnEntry(Page);
 235
 236             //
 237             // Make sure it's really free
 238             //
 239             ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 240
 241             //
 242             // Allocate it and mark it
 243             //
 244             Pfn1->u3.e1.StartOfAllocation = 1;
 245             Pfn1->u3.e1.EndOfAllocation = 1;
 246             Pfn1->u3.e2.ReferenceCount = 1;
 247
 248             //
 249             // Save it into the MDL
 250             //
 251             *MdlPage++ = MiGetPfnEntryIndex(Pfn1);
 252             PagesFound++;
 253         }
 254     }
 255     else
 256     {
 257         //
 258         // You want specific range of pages. We'll do this in two runs
 259         //
 260         for (LookForZeroedPages = 1; LookForZeroedPages >= 0; LookForZeroedPages--)
 261         {
 262             //
 263             // Scan the range you specified
 264             //
 265             for (Page = LowPage; Page < HighPage; Page++)
 266             {
 267                 //
 268                 // Get the PFN entry for this page
 269                 //
 270                 Pfn1 = MiGetPfnEntry(Page);
 271                 ASSERT(Pfn1);
 272
 273                 //
 274                 // Make sure it's free and if this is our first pass, zeroed
 275                 //
 276                 if (MiIsPfnInUse(Pfn1)) continue;
 277                 if ((Pfn1->u3.e1.PageLocation == ZeroedPageList) != LookForZeroedPages) continue;
 278
 279                 /* Remove the page from the free or zero list */
 280                 ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
 281                 MiUnlinkFreeOrZeroedPage(Pfn1);
 282
 283                 //
 284                 // Sanity checks
 285                 //
 286                 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 287
 288                 //
 289                 // Now setup the page and mark it
 290                 //
 291                 Pfn1->u3.e2.ReferenceCount = 1;
 292                 Pfn1->u2.ShareCount = 1;
 293                 MI_SET_PFN_DELETED(Pfn1);
 294                 Pfn1->u4.PteFrame = 0x1FFEDCB;
 295                 Pfn1->u3.e1.StartOfAllocation = 1;
 296                 Pfn1->u3.e1.EndOfAllocation = 1;
 297                 Pfn1->u4.VerifierAllocation = 0;
 298
 299                 //
 300                 // Save this page into the MDL
 301                 //
 302                 *MdlPage++ = Page;
 303                 if (++PagesFound == PageCount) break;
 304             }
 305
 306             //
 307             // If the first pass was enough, don't keep going, otherwise, go again
 308             //
 309             if (PagesFound == PageCount) break;
 310         }
 311     }
 312
 313     //
 314     // Now release the PFN count
 315     //
 316     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 317
 318     //
 319     // We might've found less pages, but not more ;-)
 320     //
 321     if (PagesFound != PageCount) ASSERT(PagesFound < PageCount);
 322     if (!PagesFound)
 323     {
 324         //
 325         // If we didn' tfind any pages at all, fail
 326         //
 327         DPRINT1("NO MDL PAGES!\n");
 328         ExFreePool(Mdl);
 329         return NULL;
 330     }
 331
 332     //
 333     // Write out how many pages we found
 334     //
 335     Mdl->ByteCount = (ULONG)(PagesFound << PAGE_SHIFT);
 336
 337     //
 338     // Terminate the MDL array if there's certain missing pages
 339     //
 340     if (PagesFound != PageCount) *MdlPage = LIST_HEAD;
 341
 342     //
 343     // Now go back and loop over all the MDL pages
 344     //
 345     MdlPage = (PPFN_NUMBER)(Mdl + 1);
 346     LastMdlPage = MdlPage + PagesFound;
 347     while (MdlPage < LastMdlPage)
 348     {
 349         //
 350         // Check if we've reached the end
 351         //
 352         Page = *MdlPage++;
 353         if (Page == LIST_HEAD) break;
 354
 355         //
 356         // Get the PFN entry for the page and check if we should zero it out
 357         //
 358         Pfn1 = MiGetPfnEntry(Page);
 359         ASSERT(Pfn1);
 360         if (Pfn1->u3.e1.PageLocation != ZeroedPageList) MiZeroPhysicalPage(Page);
 361         Pfn1->u3.e1.PageLocation = ActiveAndValid;
 362     }
 363
 364     //
 365     // We're done, mark the pages as locked (should we lock them, though???)
 366     //
 367     Mdl->Process = NULL;
 368     Mdl->MdlFlags |= MDL_PAGES_LOCKED;
 369     return Mdl;
 370 }
 371
 372 VOID
 373 NTAPI
 374 MmDumpPfnDatabase(VOID)
 375 {
 376     /* Pretty useless for now, to be improved later */
 377     return;
 378 }
 379
 380 VOID
 381 NTAPI
 382 MmSetRmapListHeadPage(PFN_NUMBER Pfn, PMM_RMAP_ENTRY ListHead)
 383 {
 384     KIRQL oldIrql;
 385     PMMPFN Pfn1;
 386
 387     oldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 388     Pfn1 = MiGetPfnEntry(Pfn);
 389     ASSERT(Pfn1);
 390     ASSERT_IS_ROS_PFN(Pfn1);
 391
 392     if (ListHead)
 393     {
 394         /* Should not be trying to insert an RMAP for a non-active page */
 395         ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
 396
 397         /* Set the list head address */
 398         MI_GET_ROS_DATA(Pfn1)->RmapListHead = ListHead;
 399     }
 400     else
 401     {
 402         /* ReactOS semantics dictate the page is STILL active right now */
 403         ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
 404
 405         /* In this case, the RMAP is actually being removed, so clear field */
 406         MI_GET_ROS_DATA(Pfn1)->RmapListHead = NULL;
 407
 408         /* ReactOS semantics will now release the page, which will make it free and enter a colored list */
 409     }
 410
 411     KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql);
 412 }
 413
 414 PMM_RMAP_ENTRY
 415 NTAPI
 416 MmGetRmapListHeadPage(PFN_NUMBER Pfn)
 417 {
 418     KIRQL oldIrql;
 419     PMM_RMAP_ENTRY ListHead;
 420     PMMPFN Pfn1;
 421
 422     /* Lock PFN database */
 423     oldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 424
 425     /* Get the entry */
 426     Pfn1 = MiGetPfnEntry(Pfn);
 427     ASSERT(Pfn1);
 428     ASSERT_IS_ROS_PFN(Pfn1);
 429
 430     /* Get the list head */
 431     ListHead = MI_GET_ROS_DATA(Pfn1)->RmapListHead;
 432
 433     /* Should not have an RMAP for a non-active page */
 434     ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
 435
 436     /* Release PFN database and return rmap list head */
 437     KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql);
 438     return ListHead;
 439 }
 440
 441 VOID
 442 NTAPI
 443 MmSetSavedSwapEntryPage(PFN_NUMBER Pfn,  SWAPENTRY SwapEntry)
 444 {
 445    KIRQL oldIrql;
 446    PPHYSICAL_PAGE Page;
 447
 448    Page = MiGetPfnEntry(Pfn);
 449    ASSERT(Page);
 450    ASSERT_IS_ROS_PFN(Page);
 451
 452    oldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 453    MI_GET_ROS_DATA(Page)->SwapEntry = SwapEntry;
 454    KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql);
 455 }
 456
 457 SWAPENTRY
 458 NTAPI
 459 MmGetSavedSwapEntryPage(PFN_NUMBER Pfn)
 460 {
 461    SWAPENTRY SwapEntry;
 462    KIRQL oldIrql;
 463    PPHYSICAL_PAGE Page;
 464
 465    Page = MiGetPfnEntry(Pfn);
 466    ASSERT(Page);
 467    ASSERT_IS_ROS_PFN(Page);
 468
 469    oldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 470    SwapEntry = MI_GET_ROS_DATA(Page)->SwapEntry;
 471    KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql);
 472
 473    return(SwapEntry);
 474 }
 475
 476 VOID
 477 NTAPI
 478 MmReferencePage(PFN_NUMBER Pfn)
 479 {
 480    PPHYSICAL_PAGE Page;
 481
 482    DPRINT("MmReferencePage(PysicalAddress %x)\n", Pfn << PAGE_SHIFT);
 483
 484    if (Pfn == 0 || Pfn > MmHighestPhysicalPage)
 485    {
 486       return;
 487    }
 488
 489    Page = MiGetPfnEntry(Pfn);
 490    ASSERT(Page);
 491    ASSERT_IS_ROS_PFN(Page);
 492
 493    Page->u3.e2.ReferenceCount++;
 494 }
 495
 496 ULONG
 497 NTAPI
 498 MmGetReferenceCountPage(PFN_NUMBER Pfn)
 499 {
 500    KIRQL oldIrql;
 501    ULONG RCount;
 502    PPHYSICAL_PAGE Page;
 503
 504    DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
 505
 506    oldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 507    Page = MiGetPfnEntry(Pfn);
 508    ASSERT(Page);
 509    ASSERT_IS_ROS_PFN(Page);
 510
 511    RCount = Page->u3.e2.ReferenceCount;
 512
 513    KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql);
 514    return(RCount);
 515 }
 516
 517 BOOLEAN
 518 NTAPI
 519 MmIsPageInUse(PFN_NUMBER Pfn)
 520 {
 521     return MiIsPfnInUse(MiGetPfnEntry(Pfn));
 522 }
 523
 524 VOID
 525 NTAPI
 526 MmDereferencePage(PFN_NUMBER Pfn)
 527 {
 528    PPHYSICAL_PAGE Page;
 529    DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
 530
 531    Page = MiGetPfnEntry(Pfn);
 532    ASSERT(Page);
 533    ASSERT_IS_ROS_PFN(Page);
 534
 535    Page->u3.e2.ReferenceCount--;
 536    if (Page->u3.e2.ReferenceCount == 0)
 537    {
 538         /* Mark the page temporarily as valid, we're going to make it free soon */
 539         Page->u3.e1.PageLocation = ActiveAndValid;
 540
 541         /* It's not a ROS PFN anymore */
 542         Page->u4.AweAllocation = FALSE;
 543         ExFreePool(MI_GET_ROS_DATA(Page));
 544         Page->RosMmData = 0;
 545
 546         /* Bring it back into the free list */
 547         DPRINT("Legacy free: %lx\n", Pfn);
 548         MiInsertPageInFreeList(Pfn);
 549    }
 550 }
 551
 552 PFN_NUMBER
 553 NTAPI
 554 MmAllocPage(ULONG Type)
 555 {
 556    PFN_NUMBER PfnOffset;
 557    PMMPFN Pfn1;
 558
 559    PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
 560
 561    if (!PfnOffset)
 562    {
 563        DPRINT1("MmAllocPage(): Out of memory\n");
 564        return 0;
 565    }
 566
 567    DPRINT("Legacy allocate: %lx\n", PfnOffset);
 568    Pfn1 = MiGetPfnEntry(PfnOffset);
 569    Pfn1->u3.e2.ReferenceCount = 1;
 570    Pfn1->u3.e1.PageLocation = ActiveAndValid;
 571
 572    /* This marks the PFN as a ReactOS PFN */
 573    Pfn1->u4.AweAllocation = TRUE;
 574
 575    /* Allocate the extra ReactOS Data and zero it out */
 576    Pfn1->RosMmData = (LONG)ExAllocatePoolWithTag(NonPagedPool, sizeof(MMROSPFN), 'RsPf');
 577    ASSERT(MI_GET_ROS_DATA(Pfn1) != NULL);
 578    ASSERT_IS_ROS_PFN(Pfn1);
 579    MI_GET_ROS_DATA(Pfn1)->SwapEntry = 0;
 580    MI_GET_ROS_DATA(Pfn1)->RmapListHead = NULL;
 581
 582    return PfnOffset;
 583 }
 584
 585 /* EOF */