reactos/ntoskrnl/mm/ARM3/pfnlist.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pfnlist.c
   5  * PURPOSE:         ARM Memory Manager PFN List Manipulation
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include "../ARM3/miarm.h"
  17
  18 #if DBG
  19 #define ASSERT_LIST_INVARIANT(x) \
  20 do { \
  21     ASSERT(((x)->Total == 0 && \
  22             (x)->Flink == LIST_HEAD && \
  23             (x)->Blink == LIST_HEAD) || \
  24            ((x)->Total != 0 && \
  25             (x)->Flink != LIST_HEAD && \
  26             (x)->Blink != LIST_HEAD)); \
  27 } while (0)
  28 #else
  29 #define ASSERT_LIST_INVARIANT(x)
  30 #endif
  31
  32 /* GLOBALS ********************************************************************/
  33
  34 BOOLEAN MmDynamicPfn;
  35 BOOLEAN MmMirroring;
  36 ULONG MmSystemPageColor;
  37
  38 MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
  39 MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
  40 MMPFNLIST MmStandbyPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  41 MMPFNLIST MmModifiedPageListHead = {0, ModifiedPageList, LIST_HEAD, LIST_HEAD};
  42 MMPFNLIST MmModifiedNoWritePageListHead = {0, ModifiedNoWritePageList, LIST_HEAD, LIST_HEAD};
  43 MMPFNLIST MmBadPageListHead = {0, BadPageList, LIST_HEAD, LIST_HEAD};
  44 MMPFNLIST MmRomPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  45
  46 PMMPFNLIST MmPageLocationList[] =
  47 {
  48     &MmZeroedPageListHead,
  49     &MmFreePageListHead,
  50     &MmStandbyPageListHead,
  51     &MmModifiedPageListHead,
  52     &MmModifiedNoWritePageListHead,
  53     &MmBadPageListHead,
  54     NULL,
  55     NULL
  56 };
  57
  58 ULONG MI_PFN_CURRENT_USAGE;
  59 CHAR MI_PFN_CURRENT_PROCESS_NAME[16] = "None yet";
  60
  61 /* FUNCTIONS ******************************************************************/
  62
  63 VOID
  64 NTAPI
  65 MiZeroPhysicalPage(IN PFN_NUMBER PageFrameIndex)
  66 {
  67     KIRQL OldIrql;
  68     PVOID VirtualAddress;
  69     PEPROCESS Process = PsGetCurrentProcess();
  70
  71     /* Map in hyperspace, then wipe it using XMMI or MEMSET */
  72     VirtualAddress = MiMapPageInHyperSpace(Process, PageFrameIndex, &OldIrql);
  73     ASSERT(VirtualAddress);
  74     KeZeroPages(VirtualAddress, PAGE_SIZE);
  75     MiUnmapPageInHyperSpace(Process, VirtualAddress, OldIrql);
  76 }
  77
  78 VOID
  79 NTAPI
  80 MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
  81 {
  82     PFN_NUMBER OldFlink, OldBlink;
  83     PMMPFNLIST ListHead;
  84     MMLISTS ListName;
  85     ULONG Color;
  86     PMMCOLOR_TABLES ColorTable;
  87     PMMPFN Pfn1;
  88
  89     /* Make sure the PFN lock is held */
  90     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
  91
  92     /* Make sure the PFN entry isn't in-use */
  93     ASSERT(Entry->u3.e1.WriteInProgress == 0);
  94     ASSERT(Entry->u3.e1.ReadInProgress == 0);
  95
  96     /* Find the list for this entry, make sure it's the free or zero list */
  97     ListHead = MmPageLocationList[Entry->u3.e1.PageLocation];
  98     ListName = ListHead->ListName;
  99     ASSERT(ListHead != NULL);
 100     ASSERT(ListName <= FreePageList);
 101     ASSERT_LIST_INVARIANT(ListHead);
 102
 103     /* Remove one count */
 104     ASSERT(ListHead->Total != 0);
 105     ListHead->Total--;
 106
 107     /* Get the forward and back pointers */
 108     OldFlink = Entry->u1.Flink;
 109     OldBlink = Entry->u2.Blink;
 110
 111     /* Check if the next entry is the list head */
 112     if (OldFlink != LIST_HEAD)
 113     {
 114         /* It is not, so set the backlink of the actual entry, to our backlink */
 115         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 116     }
 117     else
 118     {
 119         /* Set the list head's backlink instead */
 120         ListHead->Blink = OldBlink;
 121     }
 122
 123     /* Check if the back entry is the list head */
 124     if (OldBlink != LIST_HEAD)
 125     {
 126         /* It is not, so set the backlink of the actual entry, to our backlink */
 127         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 128     }
 129     else
 130     {
 131         /* Set the list head's backlink instead */
 132         ListHead->Flink = OldFlink;
 133     }
 134
 135     /* Get the page color */
 136     OldBlink = MiGetPfnEntryIndex(Entry);
 137     Color = OldBlink & MmSecondaryColorMask;
 138
 139     /* Get the first page on the color list */
 140     ColorTable = &MmFreePagesByColor[ListName][Color];
 141
 142     /* Check if this was was actually the head */
 143     OldFlink = ColorTable->Flink;
 144     if (OldFlink == OldBlink)
 145     {
 146         /* Make the table point to the next page this page was linking to */
 147         ColorTable->Flink = Entry->OriginalPte.u.Long;
 148         if (ColorTable->Flink != LIST_HEAD)
 149         {
 150             /* And make the previous link point to the head now */
 151             MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 152         }
 153         else
 154         {
 155             /* And if that page was the head, loop the list back around */
 156             ColorTable->Blink = (PVOID)LIST_HEAD;
 157         }
 158     }
 159     else
 160     {
 161         /* This page shouldn't be pointing back to the head */
 162         ASSERT(Entry->u4.PteFrame != COLORED_LIST_HEAD);
 163
 164         /* Make the back link point to whoever the next page is */
 165         Pfn1 = MI_PFN_ELEMENT(Entry->u4.PteFrame);
 166         Pfn1->OriginalPte.u.Long = Entry->OriginalPte.u.Long;
 167
 168         /* Check if this page was pointing to the head */
 169         if (Entry->OriginalPte.u.Long != LIST_HEAD)
 170         {
 171             /* Make the back link point to the head */
 172             Pfn1 = MI_PFN_ELEMENT(Entry->OriginalPte.u.Long);
 173             Pfn1->u4.PteFrame = Entry->u4.PteFrame;
 174         }
 175         else
 176         {
 177             /* Then the table is directly back pointing to this page now */
 178             ColorTable->Blink = Pfn1;
 179         }
 180     }
 181
 182     /* One less colored page */
 183     ASSERT(ColorTable->Count >= 1);
 184     ColorTable->Count--;
 185
 186     /* ReactOS Hack */
 187     Entry->OriginalPte.u.Long = 0;
 188
 189     /* We are not on a list anymore */
 190     Entry->u1.Flink = Entry->u2.Blink = 0;
 191     ASSERT_LIST_INVARIANT(ListHead);
 192
 193     /* See if we hit any thresholds */
 194     if (MmAvailablePages == MmHighMemoryThreshold)
 195     {
 196         /* Clear the high memory event */
 197         KeClearEvent(MiHighMemoryEvent);
 198     }
 199     else if (MmAvailablePages == MmLowMemoryThreshold)
 200     {
 201         /* Signal the low memory event */
 202         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 203     }
 204
 205     /* One less page */
 206     if (--MmAvailablePages < MmMinimumFreePages)
 207     {
 208         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 209
 210         DPRINT1("Running low on pages: %d remaining\n", MmAvailablePages);
 211
 212         /* Call RosMm and see if it can release any pages for us */
 213         MmRebalanceMemoryConsumers();
 214     }
 215
 216 #if MI_TRACE_PFNS
 217     ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 218     Entry->PfnUsage = MI_PFN_CURRENT_USAGE;
 219     memcpy(Entry->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 220 //    MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 221 //    memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 222 #endif
 223 }
 224
 225 PFN_NUMBER
 226 NTAPI
 227 MiRemovePageByColor(IN PFN_NUMBER PageIndex,
 228                     IN ULONG Color)
 229 {
 230     PMMPFN Pfn1;
 231     PMMPFNLIST ListHead;
 232     MMLISTS ListName;
 233     PFN_NUMBER OldFlink, OldBlink;
 234     USHORT OldColor, OldCache;
 235     PMMCOLOR_TABLES ColorTable;
 236
 237     /* Make sure PFN lock is held */
 238     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 239     ASSERT(Color < MmSecondaryColors);
 240
 241     /* Get the PFN entry */
 242     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 243     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 244     ASSERT(Pfn1->u3.e1.Rom == 0);
 245
 246     /* Capture data for later */
 247     OldColor = Pfn1->u3.e1.PageColor;
 248     OldCache = Pfn1->u3.e1.CacheAttribute;
 249
 250     /* Could be either on free or zero list */
 251     ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
 252     ASSERT_LIST_INVARIANT(ListHead);
 253     ListName = ListHead->ListName;
 254     ASSERT(ListName <= FreePageList);
 255
 256     /* Remove a page */
 257     ListHead->Total--;
 258
 259     /* Get the forward and back pointers */
 260     OldFlink = Pfn1->u1.Flink;
 261     OldBlink = Pfn1->u2.Blink;
 262
 263     /* Check if the next entry is the list head */
 264     if (OldFlink != LIST_HEAD)
 265     {
 266         /* It is not, so set the backlink of the actual entry, to our backlink */
 267         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 268     }
 269     else
 270     {
 271         /* Set the list head's backlink instead */
 272         ListHead->Blink = OldBlink;
 273     }
 274
 275     /* Check if the back entry is the list head */
 276     if (OldBlink != LIST_HEAD)
 277     {
 278         /* It is not, so set the backlink of the actual entry, to our backlink */
 279         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 280     }
 281     else
 282     {
 283         /* Set the list head's backlink instead */
 284         ListHead->Flink = OldFlink;
 285     }
 286
 287     /* We are not on a list anymore */
 288     ASSERT_LIST_INVARIANT(ListHead);
 289     Pfn1->u1.Flink = Pfn1->u2.Blink = 0;
 290
 291     /* Zero flags but restore color and cache */
 292     Pfn1->u3.e2.ShortFlags = 0;
 293     Pfn1->u3.e1.PageColor = OldColor;
 294     Pfn1->u3.e1.CacheAttribute = OldCache;
 295
 296     /* Get the first page on the color list */
 297     ASSERT(Color < MmSecondaryColors);
 298     ColorTable = &MmFreePagesByColor[ListName][Color];
 299     ASSERT(ColorTable->Count >= 1);
 300
 301     /* Set the forward link to whoever we were pointing to */
 302     ColorTable->Flink = Pfn1->OriginalPte.u.Long;
 303
 304     /* Get the first page on the color list */
 305     if (ColorTable->Flink == LIST_HEAD)
 306     {
 307         /* This is the beginning of the list, so set the sentinel value */
 308         ColorTable->Blink = (PVOID)LIST_HEAD;
 309     }
 310     else
 311     {
 312         /* The list is empty, so we are the first page */
 313         MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 314     }
 315
 316     /* One less page */
 317     ColorTable->Count--;
 318
 319     /* ReactOS Hack */
 320     Pfn1->OriginalPte.u.Long = 0;
 321
 322     /* See if we hit any thresholds */
 323     if (MmAvailablePages == MmHighMemoryThreshold)
 324     {
 325         /* Clear the high memory event */
 326         KeClearEvent(MiHighMemoryEvent);
 327     }
 328     else if (MmAvailablePages == MmLowMemoryThreshold)
 329     {
 330         /* Signal the low memory event */
 331         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 332     }
 333
 334     /* One less page */
 335     if (--MmAvailablePages < MmMinimumFreePages)
 336     {
 337         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 338
 339         DPRINT1("Running low on pages: %d remaining\n", MmAvailablePages);
 340
 341         /* Call RosMm and see if it can release any pages for us */
 342         MmRebalanceMemoryConsumers();
 343     }
 344
 345 #if MI_TRACE_PFNS
 346     //ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 347     Pfn1->PfnUsage = MI_PFN_CURRENT_USAGE;
 348     memcpy(Pfn1->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 349     //MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 350     //memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 351 #endif
 352
 353     /* Return the page */
 354     return PageIndex;
 355 }
 356
 357 PFN_NUMBER
 358 NTAPI
 359 MiRemoveAnyPage(IN ULONG Color)
 360 {
 361     PFN_NUMBER PageIndex;
 362     PMMPFN Pfn1;
 363
 364     /* Make sure PFN lock is held and we have pages */
 365     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 366     ASSERT(MmAvailablePages != 0);
 367     ASSERT(Color < MmSecondaryColors);
 368
 369     /* Check the colored free list */
 370     PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 371     if (PageIndex == LIST_HEAD)
 372     {
 373         /* Check the colored zero list */
 374         PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 375         if (PageIndex == LIST_HEAD)
 376         {
 377             /* Check the free list */
 378             ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 379             PageIndex = MmFreePageListHead.Flink;
 380             Color = PageIndex & MmSecondaryColorMask;
 381             if (PageIndex == LIST_HEAD)
 382             {
 383                 /* Check the zero list */
 384                 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 385                 PageIndex = MmZeroedPageListHead.Flink;
 386                 Color = PageIndex & MmSecondaryColorMask;
 387                 ASSERT(PageIndex != LIST_HEAD);
 388                 if (PageIndex == LIST_HEAD)
 389                 {
 390                     /* FIXME: Should check the standby list */
 391                     ASSERT(MmZeroedPageListHead.Total == 0);
 392                 }
 393             }
 394         }
 395     }
 396
 397     /* Remove the page from its list */
 398     PageIndex = MiRemovePageByColor(PageIndex, Color);
 399
 400     /* Sanity checks */
 401     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 402     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 403            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 404     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 405     ASSERT(Pfn1->u2.ShareCount == 0);
 406     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 407     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 408
 409     /* Return the page */
 410     return PageIndex;
 411 }
 412
 413 PFN_NUMBER
 414 NTAPI
 415 MiRemoveZeroPage(IN ULONG Color)
 416 {
 417     PFN_NUMBER PageIndex;
 418     PMMPFN Pfn1;
 419     BOOLEAN Zero = FALSE;
 420
 421     /* Make sure PFN lock is held and we have pages */
 422     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 423     ASSERT(MmAvailablePages != 0);
 424     ASSERT(Color < MmSecondaryColors);
 425
 426     /* Check the colored zero list */
 427     PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 428     if (PageIndex == LIST_HEAD)
 429     {
 430         /* Check the zero list */
 431         ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 432         PageIndex = MmZeroedPageListHead.Flink;
 433         if (PageIndex == LIST_HEAD)
 434         {
 435             /* This means there's no zero pages, we have to look for free ones */
 436             ASSERT(MmZeroedPageListHead.Total == 0);
 437             Zero = TRUE;
 438
 439             /* Check the colored free list */
 440             PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 441             if (PageIndex == LIST_HEAD)
 442             {
 443                 /* Check the free list */
 444                 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 445                 PageIndex = MmFreePageListHead.Flink;
 446                 Color = PageIndex & MmSecondaryColorMask;
 447                 ASSERT(PageIndex != LIST_HEAD);
 448                 if (PageIndex == LIST_HEAD)
 449                 {
 450                     /* FIXME: Should check the standby list */
 451                     ASSERT(MmZeroedPageListHead.Total == 0);
 452                 }
 453             }
 454         }
 455         else
 456         {
 457             Color = PageIndex & MmSecondaryColorMask;
 458         }
 459     }
 460
 461     /* Sanity checks */
 462     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 463     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 464            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 465
 466     /* Remove the page from its list */
 467     PageIndex = MiRemovePageByColor(PageIndex, Color);
 468     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageIndex));
 469
 470     /* Zero it, if needed */
 471     if (Zero) MiZeroPhysicalPage(PageIndex);
 472
 473     /* Sanity checks */
 474     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 475     ASSERT(Pfn1->u2.ShareCount == 0);
 476     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 477     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 478
 479     /* Return the page */
 480     return PageIndex;
 481 }
 482
 483 VOID
 484 NTAPI
 485 MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
 486 {
 487     PMMPFNLIST ListHead;
 488     PFN_NUMBER LastPage;
 489     PMMPFN Pfn1;
 490     ULONG Color;
 491     PMMPFN Blink;
 492     PMMCOLOR_TABLES ColorTable;
 493
 494     /* Make sure the page index is valid */
 495     ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
 496     ASSERT((PageFrameIndex != 0) &&
 497            (PageFrameIndex <= MmHighestPhysicalPage) &&
 498            (PageFrameIndex >= MmLowestPhysicalPage));
 499
 500     /* Get the PFN entry */
 501     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 502
 503     /* Sanity checks that a right kind of page is being inserted here */
 504     ASSERT(Pfn1->u4.MustBeCached == 0);
 505     ASSERT(Pfn1->u3.e1.Rom != 1);
 506     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 507     ASSERT(Pfn1->u4.VerifierAllocation == 0);
 508     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 509
 510     /* Get the free page list and increment its count */
 511     ListHead = &MmFreePageListHead;
 512     ASSERT_LIST_INVARIANT(ListHead);
 513     ListHead->Total++;
 514
 515     /* Get the last page on the list */
 516     LastPage = ListHead->Blink;
 517     if (LastPage != LIST_HEAD)
 518     {
 519         /* Link us with the previous page, so we're at the end now */
 520         MI_PFN_ELEMENT(LastPage)->u1.Flink = PageFrameIndex;
 521     }
 522     else
 523     {
 524         /* The list is empty, so we are the first page */
 525         ListHead->Flink = PageFrameIndex;
 526     }
 527
 528     /* Now make the list head point back to us (since we go at the end) */
 529     ListHead->Blink = PageFrameIndex;
 530     ASSERT_LIST_INVARIANT(ListHead);
 531
 532     /* And initialize our own list pointers */
 533     Pfn1->u1.Flink = LIST_HEAD;
 534     Pfn1->u2.Blink = LastPage;
 535
 536     /* Set the list name and default priority */
 537     Pfn1->u3.e1.PageLocation = FreePageList;
 538     Pfn1->u4.Priority = 3;
 539
 540     /* Clear some status fields */
 541     Pfn1->u4.InPageError = 0;
 542     Pfn1->u4.AweAllocation = 0;
 543
 544     /* Increase available pages */
 545     MmAvailablePages++;
 546
 547     /* Check if we've reached the configured low memory threshold */
 548     if (MmAvailablePages == MmLowMemoryThreshold)
 549     {
 550         /* Clear the event, because now we're ABOVE the threshold */
 551         KeClearEvent(MiLowMemoryEvent);
 552     }
 553     else if (MmAvailablePages == MmHighMemoryThreshold)
 554     {
 555         /* Otherwise check if we reached the high threshold and signal the event */
 556         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 557     }
 558
 559     /* Get the page color */
 560     Color = PageFrameIndex & MmSecondaryColorMask;
 561
 562     /* Get the first page on the color list */
 563     ColorTable = &MmFreePagesByColor[FreePageList][Color];
 564     if (ColorTable->Flink == LIST_HEAD)
 565     {
 566         /* The list is empty, so we are the first page */
 567         Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 568         ColorTable->Flink = PageFrameIndex;
 569     }
 570     else
 571     {
 572         /* Get the previous page */
 573         Blink = (PMMPFN)ColorTable->Blink;
 574
 575         /* Make it link to us, and link back to it */
 576         Blink->OriginalPte.u.Long = PageFrameIndex;
 577         Pfn1->u4.PteFrame = MiGetPfnEntryIndex(Blink);
 578     }
 579
 580     /* Now initialize our own list pointers */
 581     ColorTable->Blink = Pfn1;
 582
 583     /* This page is now the last */
 584     Pfn1->OriginalPte.u.Long = LIST_HEAD;
 585
 586     /* And increase the count in the colored list */
 587     ColorTable->Count++;
 588
 589     /* Notify zero page thread if enough pages are on the free list now */
 590     if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive))
 591     {
 592         /* Set the event */
 593         MmZeroingPageThreadActive = TRUE;
 594         KeSetEvent(&MmZeroingPageEvent, IO_NO_INCREMENT, FALSE);
 595     }
 596
 597 #if MI_TRACE_PFNS
 598     Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 599     RtlZeroMemory(Pfn1->ProcessName, 16);
 600 #endif
 601 }
 602
 603 /* Note: This function is hardcoded only for the zeroed page list, for now */
 604 VOID
 605 NTAPI
 606 MiInsertPageInList(IN PMMPFNLIST ListHead,
 607                    IN PFN_NUMBER PageFrameIndex)
 608 {
 609     PFN_NUMBER Flink;
 610     PMMPFN Pfn1, Pfn2;
 611     MMLISTS ListName;
 612     PMMCOLOR_TABLES ColorHead;
 613     ULONG Color;
 614
 615     /* For free pages, use MiInsertPageInFreeList */
 616     ASSERT(ListHead != &MmFreePageListHead);
 617
 618     /* Make sure the lock is held */
 619     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 620
 621     /* Make sure the PFN is valid */
 622     ASSERT((PageFrameIndex) &&
 623            (PageFrameIndex <= MmHighestPhysicalPage) &&
 624            (PageFrameIndex >= MmLowestPhysicalPage));
 625
 626     /* Page should be unused */
 627     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 628     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 629     ASSERT(Pfn1->u3.e1.Rom != 1);
 630
 631     /* Only used for zero pages in ReactOS */
 632     ListName = ListHead->ListName;
 633     ASSERT(ListName == ZeroedPageList);
 634     ListHead->Total++;
 635
 636     /* Don't handle bad pages yet yet */
 637     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 638
 639     /* Make the head of the list point to this page now */
 640     Flink = ListHead->Flink;
 641     ListHead->Flink = PageFrameIndex;
 642
 643     /* Make the page point to the previous head, and back to the list */
 644     Pfn1->u1.Flink = Flink;
 645     Pfn1->u2.Blink = LIST_HEAD;
 646
 647     /* Was the list empty? */
 648     if (Flink != LIST_HEAD)
 649     {
 650         /* It wasn't, so update the backlink of the previous head page */
 651         Pfn2 = MI_PFN_ELEMENT(Flink);
 652         Pfn2->u2.Blink = PageFrameIndex;
 653     }
 654     else
 655     {
 656         /* It was empty, so have it loop back around to this new page */
 657         ListHead->Blink = PageFrameIndex;
 658     }
 659
 660     /* Move the page onto its new location */
 661     Pfn1->u3.e1.PageLocation = ListName;
 662
 663     /* One more page on the system */
 664     MmAvailablePages++;
 665
 666     /* Check if we've reached the configured low memory threshold */
 667     if (MmAvailablePages == MmLowMemoryThreshold)
 668     {
 669         /* Clear the event, because now we're ABOVE the threshold */
 670         KeClearEvent(MiLowMemoryEvent);
 671     }
 672     else if (MmAvailablePages == MmHighMemoryThreshold)
 673     {
 674         /* Otherwise check if we reached the high threshold and signal the event */
 675         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 676     }
 677
 678     /* Sanity checks */
 679     ASSERT(ListName == ZeroedPageList);
 680     ASSERT(Pfn1->u4.InPageError == 0);
 681
 682     /* Get the page color */
 683     Color = PageFrameIndex & MmSecondaryColorMask;
 684
 685     /* Get the list for this color */
 686     ColorHead = &MmFreePagesByColor[ZeroedPageList][Color];
 687
 688     /* Get the old head */
 689     Flink = ColorHead->Flink;
 690
 691     /* Make this page point back to the list, and point forwards to the old head */
 692     Pfn1->OriginalPte.u.Long = Flink;
 693     Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 694
 695     /* Set the new head */
 696     ColorHead->Flink = PageFrameIndex;
 697
 698     /* Was the head empty? */
 699     if (Flink != LIST_HEAD)
 700     {
 701         /* No, so make the old head point to this page */
 702         Pfn2 = MI_PFN_ELEMENT(Flink);
 703         Pfn2->u4.PteFrame = PageFrameIndex;
 704     }
 705     else
 706     {
 707         /* Yes, make it loop back to this page */
 708         ColorHead->Blink = (PVOID)Pfn1;
 709     }
 710
 711     /* One more paged on the colored list */
 712     ColorHead->Count++;
 713
 714 #if MI_TRACE_PFNS
 715     //ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET);
 716     Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 717     MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 718     RtlZeroMemory(Pfn1->ProcessName, 16);
 719 #endif
 720 }
 721
 722 VOID
 723 NTAPI
 724 MiInitializePfn(IN PFN_NUMBER PageFrameIndex,
 725                 IN PMMPTE PointerPte,
 726                 IN BOOLEAN Modified)
 727 {
 728     PMMPFN Pfn1;
 729     NTSTATUS Status;
 730     PMMPTE PointerPtePte;
 731     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 732
 733     /* Setup the PTE */
 734     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 735     Pfn1->PteAddress = PointerPte;
 736
 737     /* Check if this PFN is part of a valid address space */
 738     if (PointerPte->u.Hard.Valid == 1)
 739     {
 740         /* Only valid from MmCreateProcessAddressSpace path */
 741         ASSERT(PsGetCurrentProcess()->Vm.WorkingSetSize == 0);
 742
 743         /* Make this a demand zero PTE */
 744         MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
 745     }
 746     else
 747     {
 748         /* Copy the PTE data */
 749         Pfn1->OriginalPte = *PointerPte;
 750         ASSERT(!((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
 751                  (Pfn1->OriginalPte.u.Soft.Transition == 1)));
 752     }
 753
 754     /* Otherwise this is a fresh page -- set it up */
 755     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 756     Pfn1->u3.e2.ReferenceCount = 1;
 757     Pfn1->u2.ShareCount = 1;
 758     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 759     ASSERT(Pfn1->u3.e1.Rom == 0);
 760     Pfn1->u3.e1.Modified = Modified;
 761
 762     /* Get the page table for the PTE */
 763     PointerPtePte = MiAddressToPte(PointerPte);
 764     if (PointerPtePte->u.Hard.Valid == 0)
 765     {
 766         /* Make sure the PDE gets paged in properly */
 767         Status = MiCheckPdeForPagedPool(PointerPte);
 768         if (!NT_SUCCESS(Status))
 769         {
 770             /* Crash */
 771             KeBugCheckEx(MEMORY_MANAGEMENT,
 772                          0x61940,
 773                          (ULONG_PTR)PointerPte,
 774                          (ULONG_PTR)PointerPtePte->u.Long,
 775                          (ULONG_PTR)MiPteToAddress(PointerPte));
 776         }
 777     }
 778
 779     /* Get the PFN for the page table */
 780     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
 781     ASSERT(PageFrameIndex != 0);
 782     Pfn1->u4.PteFrame = PageFrameIndex;
 783
 784     /* Increase its share count so we don't get rid of it */
 785     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 786     Pfn1->u2.ShareCount++;
 787 }
 788
 789 VOID
 790 NTAPI
 791 MiInitializePfnAndMakePteValid(IN PFN_NUMBER PageFrameIndex,
 792                                IN PMMPTE PointerPte,
 793                                IN MMPTE TempPte)
 794 {
 795     PMMPFN Pfn1;
 796     NTSTATUS Status;
 797     PMMPTE PointerPtePte;
 798     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 799
 800     /* PTE must be invalid */
 801     ASSERT(PointerPte->u.Hard.Valid == 0);
 802
 803     /* Setup the PTE */
 804     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 805     Pfn1->PteAddress = PointerPte;
 806     Pfn1->OriginalPte = DemandZeroPte;
 807
 808     /* Otherwise this is a fresh page -- set it up */
 809     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 810     Pfn1->u3.e2.ReferenceCount++;
 811     Pfn1->u2.ShareCount++;
 812     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 813     ASSERT(Pfn1->u3.e1.Rom == 0);
 814     Pfn1->u3.e1.Modified = 1;
 815
 816     /* Get the page table for the PTE */
 817     PointerPtePte = MiAddressToPte(PointerPte);
 818     if (PointerPtePte->u.Hard.Valid == 0)
 819     {
 820         /* Make sure the PDE gets paged in properly */
 821         Status = MiCheckPdeForPagedPool(PointerPte);
 822         if (!NT_SUCCESS(Status))
 823         {
 824             /* Crash */
 825             KeBugCheckEx(MEMORY_MANAGEMENT,
 826                          0x61940,
 827                          (ULONG_PTR)PointerPte,
 828                          (ULONG_PTR)PointerPtePte->u.Long,
 829                          (ULONG_PTR)MiPteToAddress(PointerPte));
 830         }
 831     }
 832
 833     /* Get the PFN for the page table */
 834     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
 835     ASSERT(PageFrameIndex != 0);
 836     Pfn1->u4.PteFrame = PageFrameIndex;
 837
 838     /* Increase its share count so we don't get rid of it */
 839     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 840     Pfn1->u2.ShareCount++;
 841
 842     /* Write valid PTE */
 843     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 844 }
 845
 846
 847 PFN_NUMBER
 848 NTAPI
 849 MiAllocatePfn(IN PMMPTE PointerPte,
 850               IN ULONG Protection)
 851 {
 852     KIRQL OldIrql;
 853     PFN_NUMBER PageFrameIndex;
 854     MMPTE TempPte;
 855
 856     /* Sanity check that we aren't passed a valid PTE */
 857     ASSERT(PointerPte->u.Hard.Valid == 0);
 858
 859     /* Make an empty software PTE */
 860     MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
 861
 862     /* Lock the PFN database */
 863     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 864
 865     /* Check if we're running low on pages */
 866     if (MmAvailablePages < 128)
 867     {
 868         DPRINT1("Warning, running low on memory: %d pages left\n", MmAvailablePages);
 869
 870         //MiEnsureAvailablePageOrWait(NULL, OldIrql);
 871
 872         /* Call RosMm and see if it can release any pages for us */
 873         MmRebalanceMemoryConsumers();
 874     }
 875
 876     /* Grab a page */
 877     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 878     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 879     PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
 880
 881     /* Write the software PTE */
 882     MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 883     PointerPte->u.Soft.Protection |= Protection;
 884
 885     /* Initialize its PFN entry */
 886     MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
 887
 888     /* Release the PFN lock and return the page */
 889     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 890     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 891     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 892     return PageFrameIndex;
 893 }
 894
 895 VOID
 896 NTAPI
 897 MiDecrementShareCount(IN PMMPFN Pfn1,
 898                       IN PFN_NUMBER PageFrameIndex)
 899 {
 900     ASSERT(PageFrameIndex > 0);
 901     ASSERT(MI_PFN_ELEMENT(PageFrameIndex) != NULL);
 902     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
 903     ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
 904
 905     /* Page must be in-use */
 906     if ((Pfn1->u3.e1.PageLocation != ActiveAndValid) &&
 907         (Pfn1->u3.e1.PageLocation != StandbyPageList))
 908     {
 909         /* Otherwise we have PFN corruption */
 910         KeBugCheckEx(PFN_LIST_CORRUPT,
 911                      0x99,
 912                      PageFrameIndex,
 913                      Pfn1->u3.e1.PageLocation,
 914                      0);
 915     }
 916
 917     /* Check if the share count is now 0 */
 918     ASSERT(Pfn1->u2.ShareCount < 0xF000000);
 919     if (!--Pfn1->u2.ShareCount)
 920     {
 921         /* ReactOS does not handle these */
 922         ASSERT(Pfn1->u3.e1.PrototypePte == 0);
 923
 924         /* Put the page in transition */
 925         Pfn1->u3.e1.PageLocation = TransitionPage;
 926
 927         /* PFN lock must be held */
 928         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 929
 930         /* Page should at least have one reference */
 931         ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 932         if (Pfn1->u3.e2.ReferenceCount == 1)
 933         {
 934             if(Pfn1->u3.e1.PrototypePte == 0)
 935             {
 936                 /* In ReactOS, this path should always be hit with a deleted PFN */
 937                 ASSERT(MI_IS_PFN_DELETED(Pfn1) == TRUE);
 938             }
 939
 940             /* Clear the last reference */
 941             Pfn1->u3.e2.ReferenceCount = 0;
 942             ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 943
 944             /* Mark the page temporarily as valid, we're going to make it free soon */
 945             Pfn1->u3.e1.PageLocation = ActiveAndValid;
 946
 947             /* Bring it back into the free list */
 948             MiInsertPageInFreeList(PageFrameIndex);
 949         }
 950         else
 951         {
 952             /* Otherwise, just drop the reference count */
 953             InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
 954         }
 955     }
 956 }
 957
 958 VOID
 959 NTAPI
 960 MiDecrementReferenceCount(IN PMMPFN Pfn1,
 961                           IN PFN_NUMBER PageFrameIndex)
 962 {
 963     /* PFN lock must be held */
 964     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 965
 966     /* Sanity checks on the page */
 967     ASSERT(PageFrameIndex < MmHighestPhysicalPage);
 968     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
 969     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 970
 971     /* Dereference the page, bail out if it's still alive */
 972     InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
 973     if (Pfn1->u3.e2.ReferenceCount) return;
 974
 975     /* Nobody should still have reference to this page */
 976     if (Pfn1->u2.ShareCount != 0)
 977     {
 978         /* Otherwise something's really wrong */
 979         KeBugCheckEx(PFN_LIST_CORRUPT, 7, PageFrameIndex, Pfn1->u2.ShareCount, 0);
 980     }
 981
 982     /* And it should be lying on some page list */
 983     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
 984
 985     /* Did someone set the delete flag? */
 986     if (MI_IS_PFN_DELETED(Pfn1))
 987     {
 988         /* Insert it into the free list, there's nothing left to do */
 989         MiInsertPageInFreeList(PageFrameIndex);
 990         return;
 991     }
 992
 993     /* We don't have a modified list yet */
 994     ASSERT(Pfn1->u3.e1.Modified == 0);
 995     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 996
 997     /* FIXME: Normally it would go on the standby list, but we're pushing it on the free list */
 998     MiInsertPageInFreeList(PageFrameIndex);
 999 }
1000
1001 VOID
1002 NTAPI
1003 MiInitializePfnForOtherProcess(IN PFN_NUMBER PageFrameIndex,
1004                                IN PMMPTE PointerPte,
1005                                IN PFN_NUMBER PteFrame)
1006 {
1007     PMMPFN Pfn1;
1008
1009     /* Setup the PTE */
1010     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1011     Pfn1->PteAddress = PointerPte;
1012
1013     /* Make this a software PTE */
1014     MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
1015
1016     /* Setup the page */
1017     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
1018     Pfn1->u3.e2.ReferenceCount = 1;
1019     Pfn1->u2.ShareCount = 1;
1020     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1021     Pfn1->u3.e1.Modified = TRUE;
1022     Pfn1->u4.InPageError = FALSE;
1023
1024     /* Did we get a PFN for the page table */
1025     if (PteFrame)
1026     {
1027         /* Store it */
1028         Pfn1->u4.PteFrame = PteFrame;
1029
1030         /* Increase its share count so we don't get rid of it */
1031         Pfn1 = MI_PFN_ELEMENT(PteFrame);
1032         Pfn1->u2.ShareCount++;
1033     }
1034 }
1035
1036 /* EOF */