ntoskrnl/mm/freelist.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/freelist.c
   5  * PURPOSE:         Handle the list of free physical pages
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Robert Bergkvist
   9  */
  10
  11 /* INCLUDES ****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #define MODULE_INVOLVED_IN_ARM3
  18 #include "ARM3/miarm.h"
  19
  20 #define ASSERT_IS_ROS_PFN(x) ASSERT(MI_IS_ROS_PFN(x) == TRUE);
  21
  22 /* GLOBALS ****************************************************************/
  23
  24 PMMPFN MmPfnDatabase;
  25
  26 PFN_NUMBER MmAvailablePages;
  27 PFN_NUMBER MmResidentAvailablePages;
  28 PFN_NUMBER MmResidentAvailableAtInit;
  29
  30 SIZE_T MmTotalCommittedPages;
  31 SIZE_T MmSharedCommit;
  32 SIZE_T MmDriverCommit;
  33 SIZE_T MmProcessCommit;
  34 SIZE_T MmPagedPoolCommit;
  35 SIZE_T MmPeakCommitment;
  36 SIZE_T MmtotalCommitLimitMaximum;
  37
  38 PMMPFN FirstUserLRUPfn;
  39 PMMPFN LastUserLRUPfn;
  40
  41 /* FUNCTIONS *************************************************************/
  42
  43 PFN_NUMBER
  44 NTAPI
  45 MmGetLRUFirstUserPage(VOID)
  46 {
  47     PFN_NUMBER Page;
  48     KIRQL OldIrql;
  49
  50     /* Find the first user page */
  51     OldIrql = MiAcquirePfnLock();
  52
  53     if (FirstUserLRUPfn == NULL)
  54     {
  55         MiReleasePfnLock(OldIrql);
  56         return 0;
  57     }
  58
  59     Page = MiGetPfnEntryIndex(FirstUserLRUPfn);
  60     MmReferencePage(Page);
  61
  62     MiReleasePfnLock(OldIrql);
  63
  64     return Page;
  65 }
  66
  67 static
  68 VOID
  69 MmInsertLRULastUserPage(PFN_NUMBER Page)
  70 {
  71     MI_ASSERT_PFN_LOCK_HELD();
  72
  73     PMMPFN Pfn = MiGetPfnEntry(Page);
  74
  75     if (FirstUserLRUPfn == NULL)
  76         FirstUserLRUPfn = Pfn;
  77
  78     Pfn->PreviousLRU = LastUserLRUPfn;
  79
  80     if (LastUserLRUPfn != NULL)
  81         LastUserLRUPfn->NextLRU = Pfn;
  82     LastUserLRUPfn = Pfn;
  83 }
  84
  85 static
  86 VOID
  87 MmRemoveLRUUserPage(PFN_NUMBER Page)
  88 {
  89     MI_ASSERT_PFN_LOCK_HELD();
  90
  91     /* Unset the page as a user page */
  92     ASSERT(Page != 0);
  93
  94     PMMPFN Pfn = MiGetPfnEntry(Page);
  95
  96     ASSERT_IS_ROS_PFN(Pfn);
  97
  98     if (Pfn->PreviousLRU)
  99     {
 100         ASSERT(Pfn->PreviousLRU->NextLRU == Pfn);
 101         Pfn->PreviousLRU->NextLRU = Pfn->NextLRU;
 102     }
 103     else
 104     {
 105         ASSERT(FirstUserLRUPfn == Pfn);
 106         FirstUserLRUPfn = Pfn->NextLRU;
 107     }
 108
 109     if (Pfn->NextLRU)
 110     {
 111         ASSERT(Pfn->NextLRU->PreviousLRU == Pfn);
 112         Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU;
 113     }
 114     else
 115     {
 116         ASSERT(Pfn == LastUserLRUPfn);
 117         LastUserLRUPfn = Pfn->PreviousLRU;
 118     }
 119
 120     Pfn->PreviousLRU = Pfn->NextLRU = NULL;
 121 }
 122
 123 PFN_NUMBER
 124 NTAPI
 125 MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast)
 126 {
 127     PFN_NUMBER Page = 0;
 128     KIRQL OldIrql;
 129
 130     /* Find the next user page */
 131     OldIrql = MiAcquirePfnLock();
 132
 133     PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage);
 134     PMMPFN NextPfn = PreviousPfn->NextLRU;
 135
 136     /*
 137      * Move this one at the end of the list.
 138      * It may be freed by MmDereferencePage below.
 139      * If it's not, then it means it is still hanging in some process address space.
 140      * This avoids paging-out e.g. ntdll early just because it's mapped first time.
 141      */
 142     if ((MoveToLast) && (MmGetReferenceCountPage(PreviousPage) > 1))
 143     {
 144         MmRemoveLRUUserPage(PreviousPage);
 145         MmInsertLRULastUserPage(PreviousPage);
 146     }
 147
 148     if (NextPfn)
 149     {
 150         Page = MiGetPfnEntryIndex(NextPfn);
 151         MmReferencePage(Page);
 152     }
 153
 154     MmDereferencePage(PreviousPage);
 155
 156     MiReleasePfnLock(OldIrql);
 157
 158     return Page;
 159 }
 160
 161 BOOLEAN
 162 NTAPI
 163 MiIsPfnFree(IN PMMPFN Pfn1)
 164 {
 165     /* Must be a free or zero page, with no references, linked */
 166     return ((Pfn1->u3.e1.PageLocation <= StandbyPageList) &&
 167             (Pfn1->u1.Flink) &&
 168             (Pfn1->u2.Blink) &&
 169             !(Pfn1->u3.e2.ReferenceCount));
 170 }
 171
 172 BOOLEAN
 173 NTAPI
 174 MiIsPfnInUse(IN PMMPFN Pfn1)
 175 {
 176     /* Standby list or higher, unlinked, and with references */
 177     return !MiIsPfnFree(Pfn1);
 178 }
 179
 180 PMDL
 181 NTAPI
 182 MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress,
 183                       IN PHYSICAL_ADDRESS HighAddress,
 184                       IN PHYSICAL_ADDRESS SkipBytes,
 185                       IN SIZE_T TotalBytes,
 186                       IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
 187                       IN ULONG MdlFlags)
 188 {
 189     PMDL Mdl;
 190     PFN_NUMBER PageCount, LowPage, HighPage, SkipPages, PagesFound = 0, Page;
 191     PPFN_NUMBER MdlPage, LastMdlPage;
 192     KIRQL OldIrql;
 193     PMMPFN Pfn1;
 194     INT LookForZeroedPages;
 195     ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
 196     DPRINT1("ARM3-DEBUG: Being called with %I64x %I64x %I64x %lx %d %lu\n", LowAddress, HighAddress, SkipBytes, TotalBytes, CacheAttribute, MdlFlags);
 197
 198     //
 199     // Convert the low address into a PFN
 200     //
 201     LowPage = (PFN_NUMBER)(LowAddress.QuadPart >> PAGE_SHIFT);
 202
 203     //
 204     // Convert, and normalize, the high address into a PFN
 205     //
 206     HighPage = (PFN_NUMBER)(HighAddress.QuadPart >> PAGE_SHIFT);
 207     if (HighPage > MmHighestPhysicalPage) HighPage = MmHighestPhysicalPage;
 208
 209     //
 210     // Validate skipbytes and convert them into pages
 211     //
 212     if (BYTE_OFFSET(SkipBytes.LowPart)) return NULL;
 213     SkipPages = (PFN_NUMBER)(SkipBytes.QuadPart >> PAGE_SHIFT);
 214
 215     /* This isn't supported at all */
 216     if (SkipPages) DPRINT1("WARNING: Caller requesting SkipBytes, MDL might be mismatched\n");
 217
 218     //
 219     // Now compute the number of pages the MDL will cover
 220     //
 221     PageCount = (PFN_NUMBER)ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, TotalBytes);
 222     do
 223     {
 224         //
 225         // Try creating an MDL for these many pages
 226         //
 227         Mdl = MmCreateMdl(NULL, NULL, PageCount << PAGE_SHIFT);
 228         if (Mdl) break;
 229
 230         //
 231         // This function is not required to return the amount of pages requested
 232         // In fact, it can return as little as 1 page, and callers are supposed
 233         // to deal with this scenario. So re-attempt the allocation with less
 234         // pages than before, and see if it worked this time.
 235         //
 236         PageCount -= (PageCount >> 4);
 237     } while (PageCount);
 238
 239     //
 240     // Wow, not even a single page was around!
 241     //
 242     if (!Mdl) return NULL;
 243
 244     //
 245     // This is where the page array starts....
 246     //
 247     MdlPage = (PPFN_NUMBER)(Mdl + 1);
 248
 249     //
 250     // Lock the PFN database
 251     //
 252     OldIrql = MiAcquirePfnLock();
 253
 254     //
 255     // Are we looking for any pages, without discriminating?
 256     //
 257     if ((LowPage == 0) && (HighPage == MmHighestPhysicalPage))
 258     {
 259         //
 260         // Well then, let's go shopping
 261         //
 262         while (PagesFound < PageCount)
 263         {
 264             /* Grab a page */
 265             MI_SET_USAGE(MI_USAGE_MDL);
 266             MI_SET_PROCESS2("Kernel");
 267
 268             /* FIXME: This check should be smarter */
 269             Page = 0;
 270             if (MmAvailablePages != 0)
 271                 Page = MiRemoveAnyPage(0);
 272
 273             if (Page == 0)
 274             {
 275                 /* This is not good... hopefully we have at least SOME pages */
 276                 ASSERT(PagesFound);
 277                 break;
 278             }
 279
 280             /* Grab the page entry for it */
 281             Pfn1 = MiGetPfnEntry(Page);
 282
 283             //
 284             // Make sure it's really free
 285             //
 286             ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 287
 288             /* Now setup the page and mark it */
 289             Pfn1->u3.e2.ReferenceCount = 1;
 290             Pfn1->u2.ShareCount = 1;
 291             MI_SET_PFN_DELETED(Pfn1);
 292             Pfn1->u4.PteFrame = 0x1FFEDCB;
 293             Pfn1->u3.e1.StartOfAllocation = 1;
 294             Pfn1->u3.e1.EndOfAllocation = 1;
 295             Pfn1->u4.VerifierAllocation = 0;
 296
 297             //
 298             // Save it into the MDL
 299             //
 300             *MdlPage++ = MiGetPfnEntryIndex(Pfn1);
 301             PagesFound++;
 302         }
 303     }
 304     else
 305     {
 306         //
 307         // You want specific range of pages. We'll do this in two runs
 308         //
 309         for (LookForZeroedPages = 1; LookForZeroedPages >= 0; LookForZeroedPages--)
 310         {
 311             //
 312             // Scan the range you specified
 313             //
 314             for (Page = LowPage; Page < HighPage; Page++)
 315             {
 316                 //
 317                 // Get the PFN entry for this page
 318                 //
 319                 Pfn1 = MiGetPfnEntry(Page);
 320                 ASSERT(Pfn1);
 321
 322                 //
 323                 // Make sure it's free and if this is our first pass, zeroed
 324                 //
 325                 if (MiIsPfnInUse(Pfn1)) continue;
 326                 if ((Pfn1->u3.e1.PageLocation == ZeroedPageList) != LookForZeroedPages) continue;
 327
 328                 /* Remove the page from the free or zero list */
 329                 ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
 330                 MI_SET_USAGE(MI_USAGE_MDL);
 331                 MI_SET_PROCESS2("Kernel");
 332                 MiUnlinkFreeOrZeroedPage(Pfn1);
 333
 334                 //
 335                 // Sanity checks
 336                 //
 337                 ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 338
 339                 //
 340                 // Now setup the page and mark it
 341                 //
 342                 Pfn1->u3.e2.ReferenceCount = 1;
 343                 Pfn1->u2.ShareCount = 1;
 344                 MI_SET_PFN_DELETED(Pfn1);
 345                 Pfn1->u4.PteFrame = 0x1FFEDCB;
 346                 Pfn1->u3.e1.StartOfAllocation = 1;
 347                 Pfn1->u3.e1.EndOfAllocation = 1;
 348                 Pfn1->u4.VerifierAllocation = 0;
 349
 350                 //
 351                 // Save this page into the MDL
 352                 //
 353                 *MdlPage++ = Page;
 354                 if (++PagesFound == PageCount) break;
 355             }
 356
 357             //
 358             // If the first pass was enough, don't keep going, otherwise, go again
 359             //
 360             if (PagesFound == PageCount) break;
 361         }
 362     }
 363
 364     //
 365     // Now release the PFN count
 366     //
 367     MiReleasePfnLock(OldIrql);
 368
 369     //
 370     // We might've found less pages, but not more ;-)
 371     //
 372     if (PagesFound != PageCount) ASSERT(PagesFound < PageCount);
 373     if (!PagesFound)
 374     {
 375         //
 376         // If we didn' tfind any pages at all, fail
 377         //
 378         DPRINT1("NO MDL PAGES!\n");
 379         ExFreePoolWithTag(Mdl, TAG_MDL);
 380         return NULL;
 381     }
 382
 383     //
 384     // Write out how many pages we found
 385     //
 386     Mdl->ByteCount = (ULONG)(PagesFound << PAGE_SHIFT);
 387
 388     //
 389     // Terminate the MDL array if there's certain missing pages
 390     //
 391     if (PagesFound != PageCount) *MdlPage = LIST_HEAD;
 392
 393     //
 394     // Now go back and loop over all the MDL pages
 395     //
 396     MdlPage = (PPFN_NUMBER)(Mdl + 1);
 397     LastMdlPage = MdlPage + PagesFound;
 398     while (MdlPage < LastMdlPage)
 399     {
 400         //
 401         // Check if we've reached the end
 402         //
 403         Page = *MdlPage++;
 404         if (Page == LIST_HEAD) break;
 405
 406         //
 407         // Get the PFN entry for the page and check if we should zero it out
 408         //
 409         Pfn1 = MiGetPfnEntry(Page);
 410         ASSERT(Pfn1);
 411         if (Pfn1->u3.e1.PageLocation != ZeroedPageList) MiZeroPhysicalPage(Page);
 412         Pfn1->u3.e1.PageLocation = ActiveAndValid;
 413     }
 414
 415     //
 416     // We're done, mark the pages as locked
 417     //
 418     Mdl->Process = NULL;
 419     Mdl->MdlFlags |= MDL_PAGES_LOCKED;
 420     return Mdl;
 421 }
 422
 423 VOID
 424 NTAPI
 425 MmSetRmapListHeadPage(PFN_NUMBER Pfn, PMM_RMAP_ENTRY ListHead)
 426 {
 427     PMMPFN Pfn1;
 428
 429     /* PFN database must be locked */
 430     MI_ASSERT_PFN_LOCK_HELD();
 431
 432     Pfn1 = MiGetPfnEntry(Pfn);
 433     ASSERT(Pfn1);
 434     ASSERT_IS_ROS_PFN(Pfn1);
 435
 436     if (ListHead)
 437     {
 438         /* Should not be trying to insert an RMAP for a non-active page */
 439         ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
 440
 441         /* Set the list head address */
 442         Pfn1->RmapListHead = ListHead;
 443     }
 444     else
 445     {
 446         /* ReactOS semantics dictate the page is STILL active right now */
 447         ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
 448
 449         /* In this case, the RMAP is actually being removed, so clear field */
 450         Pfn1->RmapListHead = NULL;
 451
 452         /* ReactOS semantics will now release the page, which will make it free and enter a colored list */
 453     }
 454 }
 455
 456 PMM_RMAP_ENTRY
 457 NTAPI
 458 MmGetRmapListHeadPage(PFN_NUMBER Pfn)
 459 {
 460     PMMPFN Pfn1;
 461
 462     /* PFN database must be locked */
 463     MI_ASSERT_PFN_LOCK_HELD();
 464
 465     /* Get the entry */
 466     Pfn1 = MiGetPfnEntry(Pfn);
 467     ASSERT(Pfn1);
 468
 469     if (!MI_IS_ROS_PFN(Pfn1))
 470     {
 471         return NULL;
 472     }
 473
 474     /* Should not have an RMAP for a non-active page */
 475     ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
 476
 477     /* Get the list head */
 478     return Pfn1->RmapListHead;
 479 }
 480
 481 VOID
 482 NTAPI
 483 MmSetSavedSwapEntryPage(PFN_NUMBER Pfn,  SWAPENTRY SwapEntry)
 484 {
 485     KIRQL oldIrql;
 486     PMMPFN Pfn1;
 487
 488     Pfn1 = MiGetPfnEntry(Pfn);
 489     ASSERT(Pfn1);
 490     ASSERT_IS_ROS_PFN(Pfn1);
 491
 492     oldIrql = MiAcquirePfnLock();
 493     Pfn1->u1.SwapEntry = SwapEntry;
 494     MiReleasePfnLock(oldIrql);
 495 }
 496
 497 SWAPENTRY
 498 NTAPI
 499 MmGetSavedSwapEntryPage(PFN_NUMBER Pfn)
 500 {
 501     SWAPENTRY SwapEntry;
 502     KIRQL oldIrql;
 503     PMMPFN Pfn1;
 504
 505     Pfn1 = MiGetPfnEntry(Pfn);
 506     ASSERT(Pfn1);
 507     ASSERT_IS_ROS_PFN(Pfn1);
 508
 509     oldIrql = MiAcquirePfnLock();
 510     SwapEntry = Pfn1->u1.SwapEntry;
 511     MiReleasePfnLock(oldIrql);
 512
 513     return(SwapEntry);
 514 }
 515
 516 VOID
 517 NTAPI
 518 MmReferencePage(PFN_NUMBER Pfn)
 519 {
 520     PMMPFN Pfn1;
 521
 522     DPRINT("MmReferencePage(PysicalAddress %x)\n", Pfn << PAGE_SHIFT);
 523
 524     MI_ASSERT_PFN_LOCK_HELD();
 525     ASSERT(Pfn != 0);
 526     ASSERT(Pfn <= MmHighestPhysicalPage);
 527
 528     Pfn1 = MiGetPfnEntry(Pfn);
 529     ASSERT(Pfn1);
 530     ASSERT_IS_ROS_PFN(Pfn1);
 531
 532     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 533     Pfn1->u3.e2.ReferenceCount++;
 534 }
 535
 536 ULONG
 537 NTAPI
 538 MmGetReferenceCountPage(PFN_NUMBER Pfn)
 539 {
 540     KIRQL oldIrql;
 541     ULONG RCount;
 542     PMMPFN Pfn1;
 543
 544     DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
 545
 546     oldIrql = MiAcquirePfnLock();
 547     Pfn1 = MiGetPfnEntry(Pfn);
 548     ASSERT(Pfn1);
 549     ASSERT_IS_ROS_PFN(Pfn1);
 550
 551     RCount = Pfn1->u3.e2.ReferenceCount;
 552
 553     MiReleasePfnLock(oldIrql);
 554     return(RCount);
 555 }
 556
 557 BOOLEAN
 558 NTAPI
 559 MmIsPageInUse(PFN_NUMBER Pfn)
 560 {
 561     return MiIsPfnInUse(MiGetPfnEntry(Pfn));
 562 }
 563
 564 VOID
 565 NTAPI
 566 MmDereferencePage(PFN_NUMBER Pfn)
 567 {
 568     PMMPFN Pfn1;
 569     KIRQL OldIrql;
 570     DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
 571
 572     OldIrql = MiAcquirePfnLock();
 573
 574     Pfn1 = MiGetPfnEntry(Pfn);
 575     ASSERT(Pfn1);
 576     ASSERT_IS_ROS_PFN(Pfn1);
 577
 578     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 579     Pfn1->u3.e2.ReferenceCount--;
 580     if (Pfn1->u3.e2.ReferenceCount == 0)
 581     {
 582         /* Apply LRU hack */
 583         if (Pfn1->u4.MustBeCached)
 584         {
 585             MmRemoveLRUUserPage(Pfn);
 586             Pfn1->u4.MustBeCached = 0;
 587         }
 588
 589         /* Mark the page temporarily as valid, we're going to make it free soon */
 590         Pfn1->u3.e1.PageLocation = ActiveAndValid;
 591
 592         /* It's not a ROS PFN anymore */
 593         Pfn1->u4.AweAllocation = FALSE;
 594
 595         /* Bring it back into the free list */
 596         DPRINT("Legacy free: %lx\n", Pfn);
 597         MiInsertPageInFreeList(Pfn);
 598     }
 599
 600     MiReleasePfnLock(OldIrql);
 601 }
 602
 603 PFN_NUMBER
 604 NTAPI
 605 MmAllocPage(ULONG Type)
 606 {
 607     PFN_NUMBER PfnOffset;
 608     PMMPFN Pfn1;
 609     KIRQL OldIrql;
 610
 611     OldIrql = MiAcquirePfnLock();
 612
 613 #if MI_TRACE_PFNS
 614     switch(Type)
 615     {
 616     case MC_SYSTEM:
 617         MI_SET_USAGE(MI_USAGE_CACHE);
 618         break;
 619     case MC_USER:
 620         MI_SET_USAGE(MI_USAGE_SECTION);
 621         break;
 622     default:
 623         ASSERT(FALSE);
 624     }
 625 #endif
 626
 627     PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
 628     if (!PfnOffset)
 629     {
 630         KeBugCheck(NO_PAGES_AVAILABLE);
 631     }
 632
 633     DPRINT("Legacy allocate: %lx\n", PfnOffset);
 634     Pfn1 = MiGetPfnEntry(PfnOffset);
 635     Pfn1->u3.e2.ReferenceCount = 1;
 636     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 637
 638     /* This marks the PFN as a ReactOS PFN */
 639     Pfn1->u4.AweAllocation = TRUE;
 640
 641     /* Allocate the extra ReactOS Data and zero it out */
 642     Pfn1->u1.SwapEntry = 0;
 643     Pfn1->RmapListHead = NULL;
 644
 645     Pfn1->NextLRU = NULL;
 646     Pfn1->PreviousLRU = NULL;
 647
 648     if (Type == MC_USER)
 649     {
 650         Pfn1->u4.MustBeCached = 1; /* HACK again */
 651         MmInsertLRULastUserPage(PfnOffset);
 652     }
 653
 654     MiReleasePfnLock(OldIrql);
 655     return PfnOffset;
 656 }
 657
 658 /* EOF */