ntoskrnl/mm/ARM3/pfnlist.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pfnlist.c
   5  * PURPOSE:         ARM Memory Manager PFN List Manipulation
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #line 15 "ARM³::PFNLIST"
  16 #define MODULE_INVOLVED_IN_ARM3
  17 #include "../ARM3/miarm.h"
  18
  19 #if DBG
  20 #define ASSERT_LIST_INVARIANT(x) \
  21 do { \
  22         ASSERT(((x)->Total == 0 && \
  23             (x)->Flink == LIST_HEAD && \
  24                         (x)->Blink == LIST_HEAD) || \
  25                    ((x)->Total != 0 && \
  26                         (x)->Flink != LIST_HEAD && \
  27                         (x)->Blink != LIST_HEAD)); \
  28 } while (0)
  29 #else
  30 #define ASSERT_LIST_INVARIANT(x)
  31 #endif
  32
  33 /* GLOBALS ********************************************************************/
  34
  35 BOOLEAN MmDynamicPfn;
  36 BOOLEAN MmMirroring;
  37 ULONG MmSystemPageColor;
  38
  39 MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
  40 MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
  41 MMPFNLIST MmStandbyPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  42 MMPFNLIST MmModifiedPageListHead = {0, ModifiedPageList, LIST_HEAD, LIST_HEAD};
  43 MMPFNLIST MmModifiedNoWritePageListHead = {0, ModifiedNoWritePageList, LIST_HEAD, LIST_HEAD};
  44 MMPFNLIST MmBadPageListHead = {0, BadPageList, LIST_HEAD, LIST_HEAD};
  45 MMPFNLIST MmRomPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  46
  47 PMMPFNLIST MmPageLocationList[] =
  48 {
  49     &MmZeroedPageListHead,
  50     &MmFreePageListHead,
  51     &MmStandbyPageListHead,
  52     &MmModifiedPageListHead,
  53     &MmModifiedNoWritePageListHead,
  54     &MmBadPageListHead,
  55     NULL,
  56     NULL
  57 };
  58 /* FUNCTIONS ******************************************************************/
  59
  60 VOID
  61 NTAPI
  62 MiZeroPhysicalPage(IN PFN_NUMBER PageFrameIndex)
  63 {
  64     KIRQL OldIrql;
  65     PVOID VirtualAddress;
  66     PEPROCESS Process = PsGetCurrentProcess();
  67
  68     /* Map in hyperspace, then wipe it using XMMI or MEMSET */
  69     VirtualAddress = MiMapPageInHyperSpace(Process, PageFrameIndex, &OldIrql);
  70     ASSERT(VirtualAddress);
  71     KeZeroPages(VirtualAddress, PAGE_SIZE);
  72     MiUnmapPageInHyperSpace(Process, VirtualAddress, OldIrql);
  73 }
  74
  75 VOID
  76 NTAPI
  77 MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
  78 {
  79     PFN_NUMBER OldFlink, OldBlink;
  80     PMMPFNLIST ListHead;
  81     MMLISTS ListName;
  82     ULONG Color;
  83     PMMCOLOR_TABLES ColorTable;
  84     PMMPFN Pfn1;
  85
  86     /* Make sure the PFN lock is held */
  87     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
  88
  89     /* Make sure the PFN entry isn't in-use */
  90     ASSERT(Entry->u3.e1.WriteInProgress == 0);
  91     ASSERT(Entry->u3.e1.ReadInProgress == 0);
  92
  93     /* Find the list for this entry, make sure it's the free or zero list */
  94     ListHead = MmPageLocationList[Entry->u3.e1.PageLocation];
  95     ListName = ListHead->ListName;
  96     ASSERT(ListHead != NULL);
  97     ASSERT(ListName <= FreePageList);
  98     ASSERT_LIST_INVARIANT(ListHead);
  99
 100     /* Remove one count */
 101     ASSERT(ListHead->Total != 0);
 102     ListHead->Total--;
 103
 104     /* Get the forward and back pointers */
 105     OldFlink = Entry->u1.Flink;
 106     OldBlink = Entry->u2.Blink;
 107
 108     /* Check if the next entry is the list head */
 109     if (OldFlink != LIST_HEAD)
 110     {
 111         /* It is not, so set the backlink of the actual entry, to our backlink */
 112         MiGetPfnEntry(OldFlink)->u2.Blink = OldBlink;
 113     }
 114     else
 115     {
 116         /* Set the list head's backlink instead */
 117         ListHead->Blink = OldBlink;
 118     }
 119
 120     /* Check if the back entry is the list head */
 121     if (OldBlink != LIST_HEAD)
 122     {
 123         /* It is not, so set the backlink of the actual entry, to our backlink */
 124         MiGetPfnEntry(OldBlink)->u1.Flink = OldFlink;
 125     }
 126     else
 127     {
 128         /* Set the list head's backlink instead */
 129         ListHead->Flink = OldFlink;
 130     }
 131
 132     /* Get the page color */
 133     OldBlink = MiGetPfnEntryIndex(Entry);
 134     Color = OldBlink & MmSecondaryColorMask;
 135
 136     /* Get the first page on the color list */
 137     ColorTable = &MmFreePagesByColor[ListName][Color];
 138
 139     /* Check if this was was actually the head */
 140     OldFlink = ColorTable->Flink;
 141     if (OldFlink == OldBlink)
 142     {
 143         /* Make the table point to the next page this page was linking to */
 144         ColorTable->Flink = Entry->OriginalPte.u.Long;
 145         if (ColorTable->Flink != LIST_HEAD)
 146         {
 147             /* And make the previous link point to the head now */
 148             MiGetPfnEntry(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 149         }
 150         else
 151         {
 152             /* And if that page was the head, loop the list back around */
 153             ColorTable->Blink = (PVOID)LIST_HEAD;
 154         }
 155     }
 156     else
 157     {
 158         /* This page shouldn't be pointing back to the head */
 159         ASSERT(Entry->u4.PteFrame != COLORED_LIST_HEAD);
 160
 161         /* Make the back link point to whoever the next page is */
 162         Pfn1 = MiGetPfnEntry(Entry->u4.PteFrame);
 163         Pfn1->OriginalPte.u.Long = Entry->OriginalPte.u.Long;
 164
 165         /* Check if this page was pointing to the head */
 166         if (Entry->OriginalPte.u.Long != LIST_HEAD)
 167         {
 168             /* Make the back link point to the head */
 169             Pfn1 = MiGetPfnEntry(Entry->OriginalPte.u.Long);
 170             Pfn1->u4.PteFrame = Entry->u4.PteFrame;
 171         }
 172         else
 173         {
 174             /* Then the table is directly back pointing to this page now */
 175             ColorTable->Blink = Pfn1;
 176         }
 177     }
 178
 179     /* One less colored page */
 180     ASSERT(ColorTable->Count >= 1);
 181     ColorTable->Count--;
 182
 183     /* ReactOS Hack */
 184     Entry->OriginalPte.u.Long = 0;
 185
 186     /* We are not on a list anymore */
 187     Entry->u1.Flink = Entry->u2.Blink = 0;
 188     ASSERT_LIST_INVARIANT(ListHead);
 189
 190     /* See if we hit any thresholds */
 191     if (MmAvailablePages == MmHighMemoryThreshold)
 192     {
 193         /* Clear the high memory event */
 194         KeClearEvent(MiHighMemoryEvent);
 195     }
 196     else if (MmAvailablePages == MmLowMemoryThreshold)
 197     {
 198         /* Signal the low memory event */
 199         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 200     }
 201
 202     /* One less page */
 203     if (--MmAvailablePages < MmMinimumFreePages)
 204     {
 205         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 206     }
 207 }
 208
 209 PFN_NUMBER
 210 NTAPI
 211 MiRemovePageByColor(IN PFN_NUMBER PageIndex,
 212                     IN ULONG Color)
 213 {
 214     PMMPFN Pfn1;
 215     PMMPFNLIST ListHead;
 216     MMLISTS ListName;
 217     PFN_NUMBER OldFlink, OldBlink;
 218     ULONG OldColor, OldCache;
 219     PMMCOLOR_TABLES ColorTable;
 220
 221     /* Make sure PFN lock is held */
 222     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 223     ASSERT(Color < MmSecondaryColors);
 224
 225     /* Get the PFN entry */
 226     Pfn1 = MiGetPfnEntry(PageIndex);
 227     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 228     ASSERT(Pfn1->u3.e1.Rom == 0);
 229
 230     /* Capture data for later */
 231     OldColor = Pfn1->u3.e1.PageColor;
 232     OldCache = Pfn1->u3.e1.CacheAttribute;
 233
 234     /* Could be either on free or zero list */
 235     ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
 236     ASSERT_LIST_INVARIANT(ListHead);
 237     ListName = ListHead->ListName;
 238     ASSERT(ListName <= FreePageList);
 239
 240     /* Remove a page */
 241     ListHead->Total--;
 242
 243     /* Get the forward and back pointers */
 244     OldFlink = Pfn1->u1.Flink;
 245     OldBlink = Pfn1->u2.Blink;
 246
 247     /* Check if the next entry is the list head */
 248     if (OldFlink != LIST_HEAD)
 249     {
 250         /* It is not, so set the backlink of the actual entry, to our backlink */
 251         MiGetPfnEntry(OldFlink)->u2.Blink = OldBlink;
 252     }
 253     else
 254     {
 255         /* Set the list head's backlink instead */
 256         ListHead->Blink = OldFlink;
 257     }
 258
 259     /* Check if the back entry is the list head */
 260     if (OldBlink != LIST_HEAD)
 261     {
 262         /* It is not, so set the backlink of the actual entry, to our backlink */
 263         MiGetPfnEntry(OldBlink)->u1.Flink = OldFlink;
 264     }
 265     else
 266     {
 267         /* Set the list head's backlink instead */
 268         ListHead->Flink = OldFlink;
 269     }
 270
 271     /* We are not on a list anymore */
 272         ASSERT_LIST_INVARIANT(ListHead);
 273     Pfn1->u1.Flink = Pfn1->u2.Blink = 0;
 274
 275     /* Zero flags but restore color and cache */
 276     Pfn1->u3.e2.ShortFlags = 0;
 277     Pfn1->u3.e1.PageColor = OldColor;
 278     Pfn1->u3.e1.CacheAttribute = OldCache;
 279
 280     /* Get the first page on the color list */
 281     ASSERT(Color < MmSecondaryColors);
 282     ColorTable = &MmFreePagesByColor[ListName][Color];
 283     ASSERT(ColorTable->Count >= 1);
 284
 285     /* Set the forward link to whoever we were pointing to */
 286     ColorTable->Flink = Pfn1->OriginalPte.u.Long;
 287
 288     /* Get the first page on the color list */
 289     if (ColorTable->Flink == LIST_HEAD)
 290     {
 291         /* This is the beginning of the list, so set the sentinel value */
 292         ColorTable->Blink = (PVOID)LIST_HEAD;
 293     }
 294     else
 295     {
 296         /* The list is empty, so we are the first page */
 297         MiGetPfnEntry(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 298     }
 299
 300     /* One less page */
 301     ColorTable->Count--;
 302
 303     /* ReactOS Hack */
 304     Pfn1->OriginalPte.u.Long = 0;
 305
 306     /* See if we hit any thresholds */
 307     if (MmAvailablePages == MmHighMemoryThreshold)
 308     {
 309         /* Clear the high memory event */
 310         KeClearEvent(MiHighMemoryEvent);
 311     }
 312     else if (MmAvailablePages == MmLowMemoryThreshold)
 313     {
 314         /* Signal the low memory event */
 315         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 316     }
 317
 318     /* One less page */
 319     if (--MmAvailablePages < MmMinimumFreePages)
 320     {
 321         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 322     }
 323
 324     /* Return the page */
 325     return PageIndex;
 326 }
 327
 328 PFN_NUMBER
 329 NTAPI
 330 MiRemoveAnyPage(IN ULONG Color)
 331 {
 332     PFN_NUMBER PageIndex;
 333     PMMPFN Pfn1;
 334
 335     /* Make sure PFN lock is held and we have pages */
 336     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 337     ASSERT(MmAvailablePages != 0);
 338     ASSERT(Color < MmSecondaryColors);
 339 #if 0
 340     /* Check the colored free list */
 341     PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 342     DPRINT1("Found free page: %lx\n", PageIndex);
 343     if (PageIndex == LIST_HEAD)
 344     {
 345         /* Check the colored zero list */
 346         PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 347         DPRINT1("Found zero page: %lx\n", PageIndex);
 348         if (PageIndex == LIST_HEAD)
 349         {
 350 #endif
 351             /* Check the free list */
 352             ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 353             PageIndex = MmFreePageListHead.Flink;
 354             Color = PageIndex & MmSecondaryColorMask;
 355             if (PageIndex == LIST_HEAD)
 356             {
 357                 /* Check the zero list */
 358                 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 359                 PageIndex = MmZeroedPageListHead.Flink;
 360                 Color = PageIndex & MmSecondaryColorMask;
 361                 ASSERT(PageIndex != LIST_HEAD);
 362                 if (PageIndex == LIST_HEAD)
 363                 {
 364                     /* FIXME: Should check the standby list */
 365                     ASSERT(MmZeroedPageListHead.Total == 0);
 366                 }
 367             }
 368 #if 0
 369         }
 370     }
 371 #endif
 372     /* Remove the page from its list */
 373     PageIndex = MiRemovePageByColor(PageIndex, Color);
 374
 375     /* Sanity checks */
 376     Pfn1 = MiGetPfnEntry(PageIndex);
 377     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 378            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 379     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 380     ASSERT(Pfn1->u2.ShareCount == 0);
 381     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 382     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 383
 384     /* Return the page */
 385     return PageIndex;
 386 }
 387
 388 PFN_NUMBER
 389 NTAPI
 390 MiRemoveZeroPage(IN ULONG Color)
 391 {
 392     PFN_NUMBER PageIndex;
 393     PMMPFN Pfn1;
 394     BOOLEAN Zero;
 395
 396     /* Make sure PFN lock is held and we have pages */
 397     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 398     ASSERT(MmAvailablePages != 0);
 399     ASSERT(Color < MmSecondaryColors);
 400
 401     /* Check the colored zero list */
 402 #if 0
 403     PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 404     if (PageIndex == LIST_HEAD)
 405     {
 406 #endif
 407         /* Check the zero list */
 408         ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 409         PageIndex = MmZeroedPageListHead.Flink;
 410         Color = PageIndex & MmSecondaryColorMask;
 411         if (PageIndex == LIST_HEAD)
 412         {
 413             /* This means there's no zero pages, we have to look for free ones */
 414             ASSERT(MmZeroedPageListHead.Total == 0);
 415             Zero = TRUE;
 416 #if 0
 417             /* Check the colored free list */
 418             PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 419             if (PageIndex == LIST_HEAD)
 420             {
 421 #endif
 422                 /* Check the free list */
 423                 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 424                 PageIndex = MmFreePageListHead.Flink;
 425                 Color = PageIndex & MmSecondaryColorMask;
 426                 ASSERT(PageIndex != LIST_HEAD);
 427                 if (PageIndex == LIST_HEAD)
 428                 {
 429                     /* FIXME: Should check the standby list */
 430                     ASSERT(MmZeroedPageListHead.Total == 0);
 431                 }
 432 #if 0
 433             }
 434 #endif
 435         }
 436 #if 0
 437     }
 438 #endif
 439
 440     /* Sanity checks */
 441     Pfn1 = MiGetPfnEntry(PageIndex);
 442     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 443            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 444
 445     /* Remove the page from its list */
 446     PageIndex = MiRemovePageByColor(PageIndex, Color);
 447     ASSERT(Pfn1 == MiGetPfnEntry(PageIndex));
 448
 449     /* Zero it, if needed */
 450     if (Zero) MiZeroPhysicalPage(PageIndex);
 451
 452     /* Sanity checks */
 453     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 454     ASSERT(Pfn1->u2.ShareCount == 0);
 455     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 456     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 457
 458     /* Return the page */
 459     return PageIndex;
 460 }
 461
 462 VOID
 463 NTAPI
 464 MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
 465 {
 466     PMMPFNLIST ListHead;
 467     PFN_NUMBER LastPage;
 468     PMMPFN Pfn1;
 469     ULONG Color;
 470     PMMPFN Blink;
 471     PMMCOLOR_TABLES ColorTable;
 472
 473     /* Make sure the page index is valid */
 474     ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
 475     ASSERT((PageFrameIndex != 0) &&
 476            (PageFrameIndex <= MmHighestPhysicalPage) &&
 477            (PageFrameIndex >= MmLowestPhysicalPage));
 478
 479     /* Get the PFN entry */
 480     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 481
 482     /* Sanity checks that a right kind of page is being inserted here */
 483     ASSERT(Pfn1->u4.MustBeCached == 0);
 484     ASSERT(Pfn1->u3.e1.Rom != 1);
 485     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 486     ASSERT(Pfn1->u4.VerifierAllocation == 0);
 487     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 488
 489     /* Get the free page list and increment its count */
 490     ListHead = &MmFreePageListHead;
 491     ASSERT_LIST_INVARIANT(ListHead);
 492     ListHead->Total++;
 493
 494     /* Get the last page on the list */
 495     LastPage = ListHead->Blink;
 496     if (LastPage != LIST_HEAD)
 497     {
 498         /* Link us with the previous page, so we're at the end now */
 499         MiGetPfnEntry(LastPage)->u1.Flink = PageFrameIndex;
 500     }
 501     else
 502     {
 503         /* The list is empty, so we are the first page */
 504         ListHead->Flink = PageFrameIndex;
 505     }
 506
 507     /* Now make the list head point back to us (since we go at the end) */
 508     ListHead->Blink = PageFrameIndex;
 509     ASSERT_LIST_INVARIANT(ListHead);
 510
 511     /* And initialize our own list pointers */
 512     Pfn1->u1.Flink = LIST_HEAD;
 513     Pfn1->u2.Blink = LastPage;
 514
 515     /* Set the list name and default priority */
 516     Pfn1->u3.e1.PageLocation = FreePageList;
 517     Pfn1->u4.Priority = 3;
 518
 519     /* Clear some status fields */
 520     Pfn1->u4.InPageError = 0;
 521     Pfn1->u4.AweAllocation = 0;
 522
 523     /* Increase available pages */
 524     MmAvailablePages++;
 525
 526     /* Check if we've reached the configured low memory threshold */
 527     if (MmAvailablePages == MmLowMemoryThreshold)
 528     {
 529         /* Clear the event, because now we're ABOVE the threshold */
 530         KeClearEvent(MiLowMemoryEvent);
 531     }
 532     else if (MmAvailablePages == MmHighMemoryThreshold)
 533     {
 534         /* Otherwise check if we reached the high threshold and signal the event */
 535         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 536     }
 537
 538     /* Get the page color */
 539     Color = PageFrameIndex & MmSecondaryColorMask;
 540
 541     /* Get the first page on the color list */
 542     ColorTable = &MmFreePagesByColor[FreePageList][Color];
 543     if (ColorTable->Flink == LIST_HEAD)
 544     {
 545         /* The list is empty, so we are the first page */
 546         Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 547         ColorTable->Flink = PageFrameIndex;
 548     }
 549     else
 550     {
 551         /* Get the previous page */
 552         Blink = (PMMPFN)ColorTable->Blink;
 553
 554         /* Make it link to us */
 555         Pfn1->u4.PteFrame = MiGetPfnEntryIndex(Blink);
 556
 557         /* If there is an original pte, it should be an old link, NOT a ReactOS RMAP */
 558         ASSERT(Blink->u4.AweAllocation == FALSE);
 559         Blink->OriginalPte.u.Long = PageFrameIndex;
 560     }
 561
 562     /* Now initialize our own list pointers */
 563     ColorTable->Blink = Pfn1;
 564
 565     /* If there is an original pte, it should be an old link, NOT a ReactOS RMAP */
 566     ASSERT(Pfn1->u4.AweAllocation == FALSE);
 567     Pfn1->OriginalPte.u.Long = LIST_HEAD;
 568
 569     /* And increase the count in the colored list */
 570     ColorTable->Count++;
 571
 572     /* Notify zero page thread if enough pages are on the free list now */
 573     if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive))
 574     {
 575         /* Set the event */
 576         MmZeroingPageThreadActive = TRUE;
 577         KeSetEvent(&MmZeroingPageEvent, IO_NO_INCREMENT, FALSE);
 578     }
 579 }
 580
 581 /* Note: This function is hardcoded only for the zeroed page list, for now */
 582 VOID
 583 NTAPI
 584 MiInsertPageInList(IN PMMPFNLIST ListHead,
 585                    IN PFN_NUMBER PageFrameIndex)
 586 {
 587     PFN_NUMBER Flink;
 588     PMMPFN Pfn1, Pfn2;
 589     MMLISTS ListName;
 590     PMMCOLOR_TABLES ColorHead;
 591     ULONG Color;
 592
 593     /* For free pages, use MiInsertPageInFreeList */
 594     ASSERT(ListHead != &MmFreePageListHead);
 595
 596     /* Make sure the lock is held */
 597     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 598
 599     /* Make sure the PFN is valid */
 600     ASSERT((PageFrameIndex) &&
 601            (PageFrameIndex <= MmHighestPhysicalPage) &&
 602            (PageFrameIndex >= MmLowestPhysicalPage));
 603
 604     /* Page should be unused */
 605     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 606     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 607     ASSERT(Pfn1->u3.e1.Rom != 1);
 608
 609     /* Only used for zero pages in ReactOS */
 610     ListName = ListHead->ListName;
 611     ASSERT(ListName == ZeroedPageList);
 612     ListHead->Total++;
 613
 614     /* Don't handle bad pages yet yet */
 615     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 616
 617     /* Make the head of the list point to this page now */
 618     Flink = ListHead->Flink;
 619     ListHead->Flink = PageFrameIndex;
 620
 621     /* Make the page point to the previous head, and back to the list */
 622     Pfn1->u1.Flink = Flink;
 623     Pfn1->u2.Blink = LIST_HEAD;
 624
 625     /* Was the list empty? */
 626     if (Flink != LIST_HEAD)
 627     {
 628         /* It wasn't, so update the backlink of the previous head page */
 629         Pfn2 = MiGetPfnEntry(Flink);
 630         Pfn2->u2.Blink = PageFrameIndex;
 631     }
 632     else
 633     {
 634         /* It was empty, so have it loop back around to this new page */
 635         ListHead->Blink = PageFrameIndex;
 636     }
 637
 638     /* Move the page onto its new location */
 639     Pfn1->u3.e1.PageLocation = ListName;
 640
 641     /* One more page on the system */
 642     MmAvailablePages++;
 643
 644     /* Check if we've reached the configured low memory threshold */
 645     if (MmAvailablePages == MmLowMemoryThreshold)
 646     {
 647         /* Clear the event, because now we're ABOVE the threshold */
 648         KeClearEvent(MiLowMemoryEvent);
 649     }
 650     else if (MmAvailablePages == MmHighMemoryThreshold)
 651     {
 652         /* Otherwise check if we reached the high threshold and signal the event */
 653         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 654     }
 655
 656     /* Sanity checks */
 657     ASSERT(ListName == ZeroedPageList);
 658     ASSERT(Pfn1->u4.InPageError == 0);
 659
 660     /* Get the page color */
 661     Color = PageFrameIndex & MmSecondaryColorMask;
 662
 663     /* Get the list for this color */
 664     ColorHead = &MmFreePagesByColor[ZeroedPageList][Color];
 665
 666     /* Get the old head */
 667     Flink = ColorHead->Flink;
 668
 669     /* If there is an original pte, it should be an old link, NOT a ReactOS RMAP */
 670     ASSERT(Pfn1->u4.AweAllocation == FALSE);
 671
 672     /* Make this page point back to the list, and point forwards to the old head */
 673     Pfn1->OriginalPte.u.Long = Flink;
 674     Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 675
 676     /* Set the new head */
 677     ColorHead->Flink = PageFrameIndex;
 678
 679     /* Was the head empty? */
 680     if (Flink != LIST_HEAD)
 681     {
 682         /* No, so make the old head point to this page */
 683         Pfn2 = MiGetPfnEntry(Flink);
 684         Pfn2->u4.PteFrame = PageFrameIndex;
 685     }
 686     else
 687     {
 688         /* Yes, make it loop back to this page */
 689         ColorHead->Blink = (PVOID)Pfn1;
 690     }
 691
 692     /* One more paged on the colored list */
 693     ColorHead->Count++;
 694 }
 695
 696 VOID
 697 NTAPI
 698 MiInitializePfn(IN PFN_NUMBER PageFrameIndex,
 699                 IN PMMPTE PointerPte,
 700                 IN BOOLEAN Modified)
 701 {
 702     PMMPFN Pfn1;
 703     NTSTATUS Status;
 704     PMMPTE PointerPtePte;
 705     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 706
 707     /* Setup the PTE */
 708     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 709     Pfn1->PteAddress = PointerPte;
 710
 711     /* Check if this PFN is part of a valid address space */
 712     if (PointerPte->u.Hard.Valid == 1)
 713     {
 714         /* Only valid from MmCreateProcessAddressSpace path */
 715         ASSERT(PsGetCurrentProcess()->Vm.WorkingSetSize == 0);
 716
 717         /* Make this a demand zero PTE */
 718         MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
 719     }
 720     else
 721     {
 722         /* Copy the PTE data */
 723         Pfn1->OriginalPte = *PointerPte;
 724         ASSERT(!((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
 725                  (Pfn1->OriginalPte.u.Soft.Transition == 1)));
 726     }
 727
 728     /* Otherwise this is a fresh page -- set it up */
 729     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 730     Pfn1->u3.e2.ReferenceCount = 1;
 731     Pfn1->u2.ShareCount = 1;
 732     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 733     ASSERT(Pfn1->u3.e1.Rom == 0);
 734     Pfn1->u3.e1.Modified = Modified;
 735
 736     /* Get the page table for the PTE */
 737     PointerPtePte = MiAddressToPte(PointerPte);
 738     if (PointerPtePte->u.Hard.Valid == 0)
 739     {
 740         /* Make sure the PDE gets paged in properly */
 741         Status = MiCheckPdeForPagedPool(PointerPte);
 742         if (!NT_SUCCESS(Status))
 743         {
 744             /* Crash */
 745             KeBugCheckEx(MEMORY_MANAGEMENT,
 746                          0x61940,
 747                          (ULONG_PTR)PointerPte,
 748                          (ULONG_PTR)PointerPtePte->u.Long,
 749                          (ULONG_PTR)MiPteToAddress(PointerPte));
 750         }
 751     }
 752
 753     /* Get the PFN for the page table */
 754     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
 755     ASSERT(PageFrameIndex != 0);
 756     Pfn1->u4.PteFrame = PageFrameIndex;
 757
 758     /* Increase its share count so we don't get rid of it */
 759     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 760     Pfn1->u2.ShareCount++;
 761 }
 762
 763 PFN_NUMBER
 764 NTAPI
 765 MiAllocatePfn(IN PMMPTE PointerPte,
 766               IN ULONG Protection)
 767 {
 768     KIRQL OldIrql;
 769     PFN_NUMBER PageFrameIndex;
 770     MMPTE TempPte;
 771
 772     /* Sanity check that we aren't passed a valid PTE */
 773     ASSERT(PointerPte->u.Hard.Valid == 0);
 774
 775     /* Make an empty software PTE */
 776     MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
 777
 778     /* Lock the PFN database */
 779     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 780
 781     /* Check if we're running low on pages */
 782     if (MmAvailablePages < 128)
 783     {
 784         DPRINT1("Warning, running low on memory: %d pages left\n", MmAvailablePages);
 785         //MiEnsureAvailablePageOrWait(NULL, OldIrql);
 786     }
 787
 788     /* Grab a page */
 789     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 790     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 791     PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
 792
 793     /* Write the software PTE */
 794     MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 795     PointerPte->u.Soft.Protection |= Protection;
 796
 797     /* Initialize its PFN entry */
 798     MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
 799
 800     /* Release the PFN lock and return the page */
 801     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 802     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 803     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 804     return PageFrameIndex;
 805 }
 806
 807 VOID
 808 NTAPI
 809 MiDecrementShareCount(IN PMMPFN Pfn1,
 810                       IN PFN_NUMBER PageFrameIndex)
 811 {
 812     ASSERT(PageFrameIndex > 0);
 813     ASSERT(MiGetPfnEntry(PageFrameIndex) != NULL);
 814     ASSERT(Pfn1 == MiGetPfnEntry(PageFrameIndex));
 815
 816     /* Page must be in-use */
 817     if ((Pfn1->u3.e1.PageLocation != ActiveAndValid) &&
 818         (Pfn1->u3.e1.PageLocation != StandbyPageList))
 819     {
 820         /* Otherwise we have PFN corruption */
 821         KeBugCheckEx(PFN_LIST_CORRUPT,
 822                      0x99,
 823                      PageFrameIndex,
 824                      Pfn1->u3.e1.PageLocation,
 825                      0);
 826     }
 827
 828     /* Check if the share count is now 0 */
 829     ASSERT(Pfn1->u2.ShareCount < 0xF000000);
 830     if (!--Pfn1->u2.ShareCount)
 831     {
 832         /* ReactOS does not handle these */
 833         ASSERT(Pfn1->u3.e1.PrototypePte == 0);
 834
 835         /* Put the page in transition */
 836         Pfn1->u3.e1.PageLocation = TransitionPage;
 837
 838         /* PFN lock must be held */
 839         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 840
 841         /* Page should at least have one reference */
 842         ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 843         if (Pfn1->u3.e2.ReferenceCount == 1)
 844         {
 845             /* In ReactOS, this path should always be hit with a deleted PFN */
 846             ASSERT(MI_IS_PFN_DELETED(Pfn1) == TRUE);
 847
 848             /* Clear the last reference */
 849             Pfn1->u3.e2.ReferenceCount = 0;
 850
 851             /*
 852              * OriginalPte is used by AweReferenceCount in ReactOS, but either
 853              * ways we shouldn't be seeing RMAP entries at this point
 854              */
 855             ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 856             ASSERT(Pfn1->u4.AweAllocation == FALSE);
 857
 858             /* Mark the page temporarily as valid, we're going to make it free soon */
 859             Pfn1->u3.e1.PageLocation = ActiveAndValid;
 860
 861             /* Bring it back into the free list */
 862             MiInsertPageInFreeList(PageFrameIndex);
 863         }
 864         else
 865         {
 866             /* Otherwise, just drop the reference count */
 867             InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
 868         }
 869     }
 870 }
 871
 872 VOID
 873 NTAPI
 874 MiDecrementReferenceCount(IN PMMPFN Pfn1,
 875                           IN PFN_NUMBER PageFrameIndex)
 876 {
 877     /* PFN lock must be held */
 878     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 879
 880     /* Sanity checks on the page */
 881     ASSERT(PageFrameIndex < MmHighestPhysicalPage);
 882     ASSERT(Pfn1 == MiGetPfnEntry(PageFrameIndex));
 883     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 884
 885     /* Dereference the page, bail out if it's still alive */
 886     InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
 887     if (Pfn1->u3.e2.ReferenceCount) return;
 888
 889     /* Nobody should still have reference to this page */
 890     if (Pfn1->u2.ShareCount != 0)
 891     {
 892         /* Otherwise something's really wrong */
 893         KeBugCheckEx(PFN_LIST_CORRUPT, 7, PageFrameIndex, Pfn1->u2.ShareCount, 0);
 894     }
 895
 896     /* And it should be lying on some page list */
 897     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
 898
 899     /* Did someone set the delete flag? */
 900     if (MI_IS_PFN_DELETED(Pfn1))
 901     {
 902         /* Insert it into the free list, there's nothing left to do */
 903         MiInsertPageInFreeList(PageFrameIndex);
 904         return;
 905     }
 906
 907     /* We don't have a modified list yet */
 908     ASSERT(Pfn1->u3.e1.Modified == 0);
 909     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 910
 911     /* FIXME: Normally it would go on the standby list, but we're pushing it on the free list */
 912     MiInsertPageInFreeList(PageFrameIndex);
 913 }
 914
 915 VOID
 916 NTAPI
 917 MiInitializePfnForOtherProcess(IN PFN_NUMBER PageFrameIndex,
 918                                IN PMMPTE PointerPte,
 919                                IN PFN_NUMBER PteFrame)
 920 {
 921     PMMPFN Pfn1;
 922
 923     /* Setup the PTE */
 924     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 925     Pfn1->PteAddress = PointerPte;
 926
 927     /* Make this a software PTE */
 928     MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
 929
 930     /* Setup the page */
 931     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 932     Pfn1->u3.e2.ReferenceCount = 1;
 933     Pfn1->u2.ShareCount = 1;
 934     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 935     Pfn1->u3.e1.Modified = TRUE;
 936     Pfn1->u4.InPageError = FALSE;
 937
 938     /* Did we get a PFN for the page table */
 939     if (PteFrame)
 940     {
 941         /* Store it */
 942         Pfn1->u4.PteFrame = PteFrame;
 943
 944         /* Increase its share count so we don't get rid of it */
 945         Pfn1 = MiGetPfnEntry(PteFrame);
 946         Pfn1->u2.ShareCount++;
 947     }
 948 }
 949
 950 /* EOF */