reactos/ntoskrnl/mm/ARM3/pfnlist.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pfnlist.c
   5  * PURPOSE:         ARM Memory Manager PFN List Manipulation
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include "../ARM3/miarm.h"
  17
  18 #if DBG
  19 #define ASSERT_LIST_INVARIANT(x) \
  20 do { \
  21         ASSERT(((x)->Total == 0 && \
  22             (x)->Flink == LIST_HEAD && \
  23                         (x)->Blink == LIST_HEAD) || \
  24                    ((x)->Total != 0 && \
  25                         (x)->Flink != LIST_HEAD && \
  26                         (x)->Blink != LIST_HEAD)); \
  27 } while (0)
  28 #else
  29 #define ASSERT_LIST_INVARIANT(x)
  30 #endif
  31
  32 /* GLOBALS ********************************************************************/
  33
  34 BOOLEAN MmDynamicPfn;
  35 BOOLEAN MmMirroring;
  36 ULONG MmSystemPageColor;
  37
  38 MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
  39 MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
  40 MMPFNLIST MmStandbyPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  41 MMPFNLIST MmModifiedPageListHead = {0, ModifiedPageList, LIST_HEAD, LIST_HEAD};
  42 MMPFNLIST MmModifiedNoWritePageListHead = {0, ModifiedNoWritePageList, LIST_HEAD, LIST_HEAD};
  43 MMPFNLIST MmBadPageListHead = {0, BadPageList, LIST_HEAD, LIST_HEAD};
  44 MMPFNLIST MmRomPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  45
  46 PMMPFNLIST MmPageLocationList[] =
  47 {
  48     &MmZeroedPageListHead,
  49     &MmFreePageListHead,
  50     &MmStandbyPageListHead,
  51     &MmModifiedPageListHead,
  52     &MmModifiedNoWritePageListHead,
  53     &MmBadPageListHead,
  54     NULL,
  55     NULL
  56 };
  57
  58 ULONG MI_PFN_CURRENT_USAGE;
  59 CHAR MI_PFN_CURRENT_PROCESS_NAME[16] = "None yet";
  60
  61 /* FUNCTIONS ******************************************************************/
  62
  63 VOID
  64 NTAPI
  65 MiZeroPhysicalPage(IN PFN_NUMBER PageFrameIndex)
  66 {
  67     KIRQL OldIrql;
  68     PVOID VirtualAddress;
  69     PEPROCESS Process = PsGetCurrentProcess();
  70
  71     /* Map in hyperspace, then wipe it using XMMI or MEMSET */
  72     VirtualAddress = MiMapPageInHyperSpace(Process, PageFrameIndex, &OldIrql);
  73     ASSERT(VirtualAddress);
  74     KeZeroPages(VirtualAddress, PAGE_SIZE);
  75     MiUnmapPageInHyperSpace(Process, VirtualAddress, OldIrql);
  76 }
  77
  78 VOID
  79 NTAPI
  80 MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
  81 {
  82     PFN_NUMBER OldFlink, OldBlink;
  83     PMMPFNLIST ListHead;
  84     MMLISTS ListName;
  85     ULONG Color;
  86     PMMCOLOR_TABLES ColorTable;
  87     PMMPFN Pfn1;
  88
  89     /* Make sure the PFN lock is held */
  90     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
  91
  92     /* Make sure the PFN entry isn't in-use */
  93     ASSERT(Entry->u3.e1.WriteInProgress == 0);
  94     ASSERT(Entry->u3.e1.ReadInProgress == 0);
  95
  96     /* Find the list for this entry, make sure it's the free or zero list */
  97     ListHead = MmPageLocationList[Entry->u3.e1.PageLocation];
  98     ListName = ListHead->ListName;
  99     ASSERT(ListHead != NULL);
 100     ASSERT(ListName <= FreePageList);
 101     ASSERT_LIST_INVARIANT(ListHead);
 102
 103     /* Remove one count */
 104     ASSERT(ListHead->Total != 0);
 105     ListHead->Total--;
 106
 107     /* Get the forward and back pointers */
 108     OldFlink = Entry->u1.Flink;
 109     OldBlink = Entry->u2.Blink;
 110
 111     /* Check if the next entry is the list head */
 112     if (OldFlink != LIST_HEAD)
 113     {
 114         /* It is not, so set the backlink of the actual entry, to our backlink */
 115         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 116     }
 117     else
 118     {
 119         /* Set the list head's backlink instead */
 120         ListHead->Blink = OldBlink;
 121     }
 122
 123     /* Check if the back entry is the list head */
 124     if (OldBlink != LIST_HEAD)
 125     {
 126         /* It is not, so set the backlink of the actual entry, to our backlink */
 127         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 128     }
 129     else
 130     {
 131         /* Set the list head's backlink instead */
 132         ListHead->Flink = OldFlink;
 133     }
 134
 135     /* Get the page color */
 136     OldBlink = MiGetPfnEntryIndex(Entry);
 137     Color = OldBlink & MmSecondaryColorMask;
 138
 139     /* Get the first page on the color list */
 140     ColorTable = &MmFreePagesByColor[ListName][Color];
 141
 142     /* Check if this was was actually the head */
 143     OldFlink = ColorTable->Flink;
 144     if (OldFlink == OldBlink)
 145     {
 146         /* Make the table point to the next page this page was linking to */
 147         ColorTable->Flink = Entry->OriginalPte.u.Long;
 148         if (ColorTable->Flink != LIST_HEAD)
 149         {
 150             /* And make the previous link point to the head now */
 151             MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 152         }
 153         else
 154         {
 155             /* And if that page was the head, loop the list back around */
 156             ColorTable->Blink = (PVOID)LIST_HEAD;
 157         }
 158     }
 159     else
 160     {
 161         /* This page shouldn't be pointing back to the head */
 162         ASSERT(Entry->u4.PteFrame != COLORED_LIST_HEAD);
 163
 164         /* Make the back link point to whoever the next page is */
 165         Pfn1 = MI_PFN_ELEMENT(Entry->u4.PteFrame);
 166         Pfn1->OriginalPte.u.Long = Entry->OriginalPte.u.Long;
 167
 168         /* Check if this page was pointing to the head */
 169         if (Entry->OriginalPte.u.Long != LIST_HEAD)
 170         {
 171             /* Make the back link point to the head */
 172             Pfn1 = MI_PFN_ELEMENT(Entry->OriginalPte.u.Long);
 173             Pfn1->u4.PteFrame = Entry->u4.PteFrame;
 174         }
 175         else
 176         {
 177             /* Then the table is directly back pointing to this page now */
 178             ColorTable->Blink = Pfn1;
 179         }
 180     }
 181
 182     /* One less colored page */
 183     ASSERT(ColorTable->Count >= 1);
 184     ColorTable->Count--;
 185
 186     /* ReactOS Hack */
 187     Entry->OriginalPte.u.Long = 0;
 188
 189     /* We are not on a list anymore */
 190     Entry->u1.Flink = Entry->u2.Blink = 0;
 191     ASSERT_LIST_INVARIANT(ListHead);
 192
 193     /* See if we hit any thresholds */
 194     if (MmAvailablePages == MmHighMemoryThreshold)
 195     {
 196         /* Clear the high memory event */
 197         KeClearEvent(MiHighMemoryEvent);
 198     }
 199     else if (MmAvailablePages == MmLowMemoryThreshold)
 200     {
 201         /* Signal the low memory event */
 202         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 203     }
 204
 205     /* One less page */
 206     if (--MmAvailablePages < MmMinimumFreePages)
 207     {
 208         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 209     }
 210
 211 #if MI_TRACE_PFNS
 212     ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 213     Entry->PfnUsage = MI_PFN_CURRENT_USAGE;
 214     memcpy(Entry->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 215 //    MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 216 //    memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 217 #endif
 218 }
 219
 220 PFN_NUMBER
 221 NTAPI
 222 MiRemovePageByColor(IN PFN_NUMBER PageIndex,
 223                     IN ULONG Color)
 224 {
 225     PMMPFN Pfn1;
 226     PMMPFNLIST ListHead;
 227     MMLISTS ListName;
 228     PFN_NUMBER OldFlink, OldBlink;
 229     USHORT OldColor, OldCache;
 230     PMMCOLOR_TABLES ColorTable;
 231
 232     /* Make sure PFN lock is held */
 233     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 234     ASSERT(Color < MmSecondaryColors);
 235
 236     /* Get the PFN entry */
 237     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 238     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 239     ASSERT(Pfn1->u3.e1.Rom == 0);
 240
 241     /* Capture data for later */
 242     OldColor = Pfn1->u3.e1.PageColor;
 243     OldCache = Pfn1->u3.e1.CacheAttribute;
 244
 245     /* Could be either on free or zero list */
 246     ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
 247     ASSERT_LIST_INVARIANT(ListHead);
 248     ListName = ListHead->ListName;
 249     ASSERT(ListName <= FreePageList);
 250
 251     /* Remove a page */
 252     ListHead->Total--;
 253
 254     /* Get the forward and back pointers */
 255     OldFlink = Pfn1->u1.Flink;
 256     OldBlink = Pfn1->u2.Blink;
 257
 258     /* Check if the next entry is the list head */
 259     if (OldFlink != LIST_HEAD)
 260     {
 261         /* It is not, so set the backlink of the actual entry, to our backlink */
 262         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 263     }
 264     else
 265     {
 266         /* Set the list head's backlink instead */
 267         ListHead->Blink = OldBlink;
 268     }
 269
 270     /* Check if the back entry is the list head */
 271     if (OldBlink != LIST_HEAD)
 272     {
 273         /* It is not, so set the backlink of the actual entry, to our backlink */
 274         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 275     }
 276     else
 277     {
 278         /* Set the list head's backlink instead */
 279         ListHead->Flink = OldFlink;
 280     }
 281
 282     /* We are not on a list anymore */
 283         ASSERT_LIST_INVARIANT(ListHead);
 284     Pfn1->u1.Flink = Pfn1->u2.Blink = 0;
 285
 286     /* Zero flags but restore color and cache */
 287     Pfn1->u3.e2.ShortFlags = 0;
 288     Pfn1->u3.e1.PageColor = OldColor;
 289     Pfn1->u3.e1.CacheAttribute = OldCache;
 290
 291     /* Get the first page on the color list */
 292     ASSERT(Color < MmSecondaryColors);
 293     ColorTable = &MmFreePagesByColor[ListName][Color];
 294     ASSERT(ColorTable->Count >= 1);
 295
 296     /* Set the forward link to whoever we were pointing to */
 297     ColorTable->Flink = Pfn1->OriginalPte.u.Long;
 298
 299     /* Get the first page on the color list */
 300     if (ColorTable->Flink == LIST_HEAD)
 301     {
 302         /* This is the beginning of the list, so set the sentinel value */
 303         ColorTable->Blink = (PVOID)LIST_HEAD;
 304     }
 305     else
 306     {
 307         /* The list is empty, so we are the first page */
 308         MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 309     }
 310
 311     /* One less page */
 312     ColorTable->Count--;
 313
 314     /* ReactOS Hack */
 315     Pfn1->OriginalPte.u.Long = 0;
 316
 317     /* See if we hit any thresholds */
 318     if (MmAvailablePages == MmHighMemoryThreshold)
 319     {
 320         /* Clear the high memory event */
 321         KeClearEvent(MiHighMemoryEvent);
 322     }
 323     else if (MmAvailablePages == MmLowMemoryThreshold)
 324     {
 325         /* Signal the low memory event */
 326         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 327     }
 328
 329     /* One less page */
 330     if (--MmAvailablePages < MmMinimumFreePages)
 331     {
 332         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 333     }
 334
 335 #if MI_TRACE_PFNS
 336     //ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 337     Pfn1->PfnUsage = MI_PFN_CURRENT_USAGE;
 338     memcpy(Pfn1->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 339     //MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 340     //memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 341 #endif
 342
 343     /* Return the page */
 344     return PageIndex;
 345 }
 346
 347 PFN_NUMBER
 348 NTAPI
 349 MiRemoveAnyPage(IN ULONG Color)
 350 {
 351     PFN_NUMBER PageIndex;
 352     PMMPFN Pfn1;
 353
 354     /* Make sure PFN lock is held and we have pages */
 355     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 356     ASSERT(MmAvailablePages != 0);
 357     ASSERT(Color < MmSecondaryColors);
 358
 359     /* Check the colored free list */
 360     PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 361     if (PageIndex == LIST_HEAD)
 362     {
 363         /* Check the colored zero list */
 364         PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 365         if (PageIndex == LIST_HEAD)
 366         {
 367             /* Check the free list */
 368             ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 369             PageIndex = MmFreePageListHead.Flink;
 370             Color = PageIndex & MmSecondaryColorMask;
 371             if (PageIndex == LIST_HEAD)
 372             {
 373                 /* Check the zero list */
 374                 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 375                 PageIndex = MmZeroedPageListHead.Flink;
 376                 Color = PageIndex & MmSecondaryColorMask;
 377                 ASSERT(PageIndex != LIST_HEAD);
 378                 if (PageIndex == LIST_HEAD)
 379                 {
 380                     /* FIXME: Should check the standby list */
 381                     ASSERT(MmZeroedPageListHead.Total == 0);
 382                 }
 383             }
 384         }
 385     }
 386
 387     /* Remove the page from its list */
 388     PageIndex = MiRemovePageByColor(PageIndex, Color);
 389
 390     /* Sanity checks */
 391     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 392     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 393            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 394     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 395     ASSERT(Pfn1->u2.ShareCount == 0);
 396     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 397     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 398
 399     /* Return the page */
 400     return PageIndex;
 401 }
 402
 403 PFN_NUMBER
 404 NTAPI
 405 MiRemoveZeroPage(IN ULONG Color)
 406 {
 407     PFN_NUMBER PageIndex;
 408     PMMPFN Pfn1;
 409     BOOLEAN Zero = FALSE;
 410
 411     /* Make sure PFN lock is held and we have pages */
 412     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 413     ASSERT(MmAvailablePages != 0);
 414     ASSERT(Color < MmSecondaryColors);
 415
 416     /* Check the colored zero list */
 417     PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 418     if (PageIndex == LIST_HEAD)
 419     {
 420         /* Check the zero list */
 421         ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 422         PageIndex = MmZeroedPageListHead.Flink;
 423         if (PageIndex == LIST_HEAD)
 424         {
 425             /* This means there's no zero pages, we have to look for free ones */
 426             ASSERT(MmZeroedPageListHead.Total == 0);
 427             Zero = TRUE;
 428
 429             /* Check the colored free list */
 430             PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 431             if (PageIndex == LIST_HEAD)
 432             {
 433                 /* Check the free list */
 434                 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 435                 PageIndex = MmFreePageListHead.Flink;
 436                 Color = PageIndex & MmSecondaryColorMask;
 437                 ASSERT(PageIndex != LIST_HEAD);
 438                 if (PageIndex == LIST_HEAD)
 439                 {
 440                     /* FIXME: Should check the standby list */
 441                     ASSERT(MmZeroedPageListHead.Total == 0);
 442                 }
 443             }
 444         }
 445         else
 446         {
 447             Color = PageIndex & MmSecondaryColorMask;
 448         }
 449     }
 450
 451     /* Sanity checks */
 452     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 453     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 454            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 455
 456     /* Remove the page from its list */
 457     PageIndex = MiRemovePageByColor(PageIndex, Color);
 458     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageIndex));
 459
 460     /* Zero it, if needed */
 461     if (Zero) MiZeroPhysicalPage(PageIndex);
 462
 463     /* Sanity checks */
 464     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 465     ASSERT(Pfn1->u2.ShareCount == 0);
 466     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 467     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 468
 469     /* Return the page */
 470     return PageIndex;
 471 }
 472
 473 VOID
 474 NTAPI
 475 MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
 476 {
 477     PMMPFNLIST ListHead;
 478     PFN_NUMBER LastPage;
 479     PMMPFN Pfn1;
 480     ULONG Color;
 481     PMMPFN Blink;
 482     PMMCOLOR_TABLES ColorTable;
 483
 484     /* Make sure the page index is valid */
 485     ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
 486     ASSERT((PageFrameIndex != 0) &&
 487            (PageFrameIndex <= MmHighestPhysicalPage) &&
 488            (PageFrameIndex >= MmLowestPhysicalPage));
 489
 490     /* Get the PFN entry */
 491     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 492
 493     /* Sanity checks that a right kind of page is being inserted here */
 494     ASSERT(Pfn1->u4.MustBeCached == 0);
 495     ASSERT(Pfn1->u3.e1.Rom != 1);
 496     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 497     ASSERT(Pfn1->u4.VerifierAllocation == 0);
 498     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 499
 500     /* Get the free page list and increment its count */
 501     ListHead = &MmFreePageListHead;
 502     ASSERT_LIST_INVARIANT(ListHead);
 503     ListHead->Total++;
 504
 505     /* Get the last page on the list */
 506     LastPage = ListHead->Blink;
 507     if (LastPage != LIST_HEAD)
 508     {
 509         /* Link us with the previous page, so we're at the end now */
 510         MI_PFN_ELEMENT(LastPage)->u1.Flink = PageFrameIndex;
 511     }
 512     else
 513     {
 514         /* The list is empty, so we are the first page */
 515         ListHead->Flink = PageFrameIndex;
 516     }
 517
 518     /* Now make the list head point back to us (since we go at the end) */
 519     ListHead->Blink = PageFrameIndex;
 520     ASSERT_LIST_INVARIANT(ListHead);
 521
 522     /* And initialize our own list pointers */
 523     Pfn1->u1.Flink = LIST_HEAD;
 524     Pfn1->u2.Blink = LastPage;
 525
 526     /* Set the list name and default priority */
 527     Pfn1->u3.e1.PageLocation = FreePageList;
 528     Pfn1->u4.Priority = 3;
 529
 530     /* Clear some status fields */
 531     Pfn1->u4.InPageError = 0;
 532     Pfn1->u4.AweAllocation = 0;
 533
 534     /* Increase available pages */
 535     MmAvailablePages++;
 536
 537     /* Check if we've reached the configured low memory threshold */
 538     if (MmAvailablePages == MmLowMemoryThreshold)
 539     {
 540         /* Clear the event, because now we're ABOVE the threshold */
 541         KeClearEvent(MiLowMemoryEvent);
 542     }
 543     else if (MmAvailablePages == MmHighMemoryThreshold)
 544     {
 545         /* Otherwise check if we reached the high threshold and signal the event */
 546         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 547     }
 548
 549     /* Get the page color */
 550     Color = PageFrameIndex & MmSecondaryColorMask;
 551
 552     /* Get the first page on the color list */
 553     ColorTable = &MmFreePagesByColor[FreePageList][Color];
 554     if (ColorTable->Flink == LIST_HEAD)
 555     {
 556         /* The list is empty, so we are the first page */
 557         Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 558         ColorTable->Flink = PageFrameIndex;
 559     }
 560     else
 561     {
 562         /* Get the previous page */
 563         Blink = (PMMPFN)ColorTable->Blink;
 564
 565         /* Make it link to us, and link back to it */
 566         Blink->OriginalPte.u.Long = PageFrameIndex;
 567         Pfn1->u4.PteFrame = MiGetPfnEntryIndex(Blink);
 568     }
 569
 570     /* Now initialize our own list pointers */
 571     ColorTable->Blink = Pfn1;
 572
 573     /* This page is now the last */
 574     Pfn1->OriginalPte.u.Long = LIST_HEAD;
 575
 576     /* And increase the count in the colored list */
 577     ColorTable->Count++;
 578
 579     /* Notify zero page thread if enough pages are on the free list now */
 580     if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive))
 581     {
 582         /* Set the event */
 583         MmZeroingPageThreadActive = TRUE;
 584         KeSetEvent(&MmZeroingPageEvent, IO_NO_INCREMENT, FALSE);
 585     }
 586
 587 #if MI_TRACE_PFNS
 588     Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 589     RtlZeroMemory(Pfn1->ProcessName, 16);
 590 #endif
 591 }
 592
 593 /* Note: This function is hardcoded only for the zeroed page list, for now */
 594 VOID
 595 NTAPI
 596 MiInsertPageInList(IN PMMPFNLIST ListHead,
 597                    IN PFN_NUMBER PageFrameIndex)
 598 {
 599     PFN_NUMBER Flink;
 600     PMMPFN Pfn1, Pfn2;
 601     MMLISTS ListName;
 602     PMMCOLOR_TABLES ColorHead;
 603     ULONG Color;
 604
 605     /* For free pages, use MiInsertPageInFreeList */
 606     ASSERT(ListHead != &MmFreePageListHead);
 607
 608     /* Make sure the lock is held */
 609     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 610
 611     /* Make sure the PFN is valid */
 612     ASSERT((PageFrameIndex) &&
 613            (PageFrameIndex <= MmHighestPhysicalPage) &&
 614            (PageFrameIndex >= MmLowestPhysicalPage));
 615
 616     /* Page should be unused */
 617     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 618     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 619     ASSERT(Pfn1->u3.e1.Rom != 1);
 620
 621     /* Only used for zero pages in ReactOS */
 622     ListName = ListHead->ListName;
 623     ASSERT(ListName == ZeroedPageList);
 624     ListHead->Total++;
 625
 626     /* Don't handle bad pages yet yet */
 627     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 628
 629     /* Make the head of the list point to this page now */
 630     Flink = ListHead->Flink;
 631     ListHead->Flink = PageFrameIndex;
 632
 633     /* Make the page point to the previous head, and back to the list */
 634     Pfn1->u1.Flink = Flink;
 635     Pfn1->u2.Blink = LIST_HEAD;
 636
 637     /* Was the list empty? */
 638     if (Flink != LIST_HEAD)
 639     {
 640         /* It wasn't, so update the backlink of the previous head page */
 641         Pfn2 = MI_PFN_ELEMENT(Flink);
 642         Pfn2->u2.Blink = PageFrameIndex;
 643     }
 644     else
 645     {
 646         /* It was empty, so have it loop back around to this new page */
 647         ListHead->Blink = PageFrameIndex;
 648     }
 649
 650     /* Move the page onto its new location */
 651     Pfn1->u3.e1.PageLocation = ListName;
 652
 653     /* One more page on the system */
 654     MmAvailablePages++;
 655
 656     /* Check if we've reached the configured low memory threshold */
 657     if (MmAvailablePages == MmLowMemoryThreshold)
 658     {
 659         /* Clear the event, because now we're ABOVE the threshold */
 660         KeClearEvent(MiLowMemoryEvent);
 661     }
 662     else if (MmAvailablePages == MmHighMemoryThreshold)
 663     {
 664         /* Otherwise check if we reached the high threshold and signal the event */
 665         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 666     }
 667
 668     /* Sanity checks */
 669     ASSERT(ListName == ZeroedPageList);
 670     ASSERT(Pfn1->u4.InPageError == 0);
 671
 672     /* Get the page color */
 673     Color = PageFrameIndex & MmSecondaryColorMask;
 674
 675     /* Get the list for this color */
 676     ColorHead = &MmFreePagesByColor[ZeroedPageList][Color];
 677
 678     /* Get the old head */
 679     Flink = ColorHead->Flink;
 680
 681     /* Make this page point back to the list, and point forwards to the old head */
 682     Pfn1->OriginalPte.u.Long = Flink;
 683     Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 684
 685     /* Set the new head */
 686     ColorHead->Flink = PageFrameIndex;
 687
 688     /* Was the head empty? */
 689     if (Flink != LIST_HEAD)
 690     {
 691         /* No, so make the old head point to this page */
 692         Pfn2 = MI_PFN_ELEMENT(Flink);
 693         Pfn2->u4.PteFrame = PageFrameIndex;
 694     }
 695     else
 696     {
 697         /* Yes, make it loop back to this page */
 698         ColorHead->Blink = (PVOID)Pfn1;
 699     }
 700
 701     /* One more paged on the colored list */
 702     ColorHead->Count++;
 703
 704 #if MI_TRACE_PFNS
 705     //ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET);
 706     Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 707     MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 708     RtlZeroMemory(Pfn1->ProcessName, 16);
 709 #endif
 710 }
 711
 712 VOID
 713 NTAPI
 714 MiInitializePfn(IN PFN_NUMBER PageFrameIndex,
 715                 IN PMMPTE PointerPte,
 716                 IN BOOLEAN Modified)
 717 {
 718     PMMPFN Pfn1;
 719     NTSTATUS Status;
 720     PMMPTE PointerPtePte;
 721     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 722
 723     /* Setup the PTE */
 724     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 725     Pfn1->PteAddress = PointerPte;
 726
 727     /* Check if this PFN is part of a valid address space */
 728     if (PointerPte->u.Hard.Valid == 1)
 729     {
 730         /* Only valid from MmCreateProcessAddressSpace path */
 731         ASSERT(PsGetCurrentProcess()->Vm.WorkingSetSize == 0);
 732
 733         /* Make this a demand zero PTE */
 734         MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
 735     }
 736     else
 737     {
 738         /* Copy the PTE data */
 739         Pfn1->OriginalPte = *PointerPte;
 740         ASSERT(!((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
 741                  (Pfn1->OriginalPte.u.Soft.Transition == 1)));
 742     }
 743
 744     /* Otherwise this is a fresh page -- set it up */
 745     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 746     Pfn1->u3.e2.ReferenceCount = 1;
 747     Pfn1->u2.ShareCount = 1;
 748     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 749     ASSERT(Pfn1->u3.e1.Rom == 0);
 750     Pfn1->u3.e1.Modified = Modified;
 751
 752     /* Get the page table for the PTE */
 753     PointerPtePte = MiAddressToPte(PointerPte);
 754     if (PointerPtePte->u.Hard.Valid == 0)
 755     {
 756         /* Make sure the PDE gets paged in properly */
 757         Status = MiCheckPdeForPagedPool(PointerPte);
 758         if (!NT_SUCCESS(Status))
 759         {
 760             /* Crash */
 761             KeBugCheckEx(MEMORY_MANAGEMENT,
 762                          0x61940,
 763                          (ULONG_PTR)PointerPte,
 764                          (ULONG_PTR)PointerPtePte->u.Long,
 765                          (ULONG_PTR)MiPteToAddress(PointerPte));
 766         }
 767     }
 768
 769     /* Get the PFN for the page table */
 770     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
 771     ASSERT(PageFrameIndex != 0);
 772     Pfn1->u4.PteFrame = PageFrameIndex;
 773
 774     /* Increase its share count so we don't get rid of it */
 775     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 776     Pfn1->u2.ShareCount++;
 777 }
 778
 779 VOID
 780 NTAPI
 781 MiInitializePfnAndMakePteValid(IN PFN_NUMBER PageFrameIndex,
 782                                IN PMMPTE PointerPte,
 783                                IN MMPTE TempPte)
 784 {
 785     PMMPFN Pfn1;
 786     NTSTATUS Status;
 787     PMMPTE PointerPtePte;
 788     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 789
 790     /* PTE must be invalid */
 791     ASSERT(PointerPte->u.Hard.Valid == 0);
 792
 793     /* Setup the PTE */
 794     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 795     Pfn1->PteAddress = PointerPte;
 796     Pfn1->OriginalPte = DemandZeroPte;
 797
 798     /* Otherwise this is a fresh page -- set it up */
 799     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 800     Pfn1->u3.e2.ReferenceCount++;
 801     Pfn1->u2.ShareCount++;
 802     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 803     ASSERT(Pfn1->u3.e1.Rom == 0);
 804     Pfn1->u3.e1.Modified = 1;
 805
 806     /* Get the page table for the PTE */
 807     PointerPtePte = MiAddressToPte(PointerPte);
 808     if (PointerPtePte->u.Hard.Valid == 0)
 809     {
 810         /* Make sure the PDE gets paged in properly */
 811         Status = MiCheckPdeForPagedPool(PointerPte);
 812         if (!NT_SUCCESS(Status))
 813         {
 814             /* Crash */
 815             KeBugCheckEx(MEMORY_MANAGEMENT,
 816                          0x61940,
 817                          (ULONG_PTR)PointerPte,
 818                          (ULONG_PTR)PointerPtePte->u.Long,
 819                          (ULONG_PTR)MiPteToAddress(PointerPte));
 820         }
 821     }
 822
 823     /* Get the PFN for the page table */
 824     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
 825     ASSERT(PageFrameIndex != 0);
 826     Pfn1->u4.PteFrame = PageFrameIndex;
 827
 828     /* Increase its share count so we don't get rid of it */
 829     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 830     Pfn1->u2.ShareCount++;
 831
 832     /* Write valid PTE */
 833     MI_WRITE_VALID_PTE(PointerPte, TempPte);
 834 }
 835
 836
 837 PFN_NUMBER
 838 NTAPI
 839 MiAllocatePfn(IN PMMPTE PointerPte,
 840               IN ULONG Protection)
 841 {
 842     KIRQL OldIrql;
 843     PFN_NUMBER PageFrameIndex;
 844     MMPTE TempPte;
 845
 846     /* Sanity check that we aren't passed a valid PTE */
 847     ASSERT(PointerPte->u.Hard.Valid == 0);
 848
 849     /* Make an empty software PTE */
 850     MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
 851
 852     /* Check if we're running low on pages */
 853     if (MmAvailablePages < 128)
 854     {
 855         DPRINT1("Warning, running low on memory: %d pages left\n", MmAvailablePages);
 856
 857         //MiEnsureAvailablePageOrWait(NULL, OldIrql);
 858
 859         /* Call RosMm and see if it can release any pages for us */
 860         MmRebalanceMemoryConsumers();
 861
 862         DPRINT1("Rebalance complete: %d pages left\n", MmAvailablePages);
 863     }
 864
 865     /* Lock the PFN database */
 866     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 867
 868     /* Grab a page */
 869     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 870     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 871     PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
 872
 873     /* Write the software PTE */
 874     MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 875     PointerPte->u.Soft.Protection |= Protection;
 876
 877     /* Initialize its PFN entry */
 878     MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
 879
 880     /* Release the PFN lock and return the page */
 881     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 882     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 883     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 884     return PageFrameIndex;
 885 }
 886
 887 VOID
 888 NTAPI
 889 MiDecrementShareCount(IN PMMPFN Pfn1,
 890                       IN PFN_NUMBER PageFrameIndex)
 891 {
 892     ASSERT(PageFrameIndex > 0);
 893     ASSERT(MI_PFN_ELEMENT(PageFrameIndex) != NULL);
 894     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
 895     ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
 896
 897     /* Page must be in-use */
 898     if ((Pfn1->u3.e1.PageLocation != ActiveAndValid) &&
 899         (Pfn1->u3.e1.PageLocation != StandbyPageList))
 900     {
 901         /* Otherwise we have PFN corruption */
 902         KeBugCheckEx(PFN_LIST_CORRUPT,
 903                      0x99,
 904                      PageFrameIndex,
 905                      Pfn1->u3.e1.PageLocation,
 906                      0);
 907     }
 908
 909     /* Check if the share count is now 0 */
 910     ASSERT(Pfn1->u2.ShareCount < 0xF000000);
 911     if (!--Pfn1->u2.ShareCount)
 912     {
 913         /* ReactOS does not handle these */
 914         ASSERT(Pfn1->u3.e1.PrototypePte == 0);
 915
 916         /* Put the page in transition */
 917         Pfn1->u3.e1.PageLocation = TransitionPage;
 918
 919         /* PFN lock must be held */
 920         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 921
 922         /* Page should at least have one reference */
 923         ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 924         if (Pfn1->u3.e2.ReferenceCount == 1)
 925         {
 926             /* In ReactOS, this path should always be hit with a deleted PFN */
 927             ASSERT(MI_IS_PFN_DELETED(Pfn1) == TRUE);
 928
 929             /* Clear the last reference */
 930             Pfn1->u3.e2.ReferenceCount = 0;
 931             ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 932
 933             /* Mark the page temporarily as valid, we're going to make it free soon */
 934             Pfn1->u3.e1.PageLocation = ActiveAndValid;
 935
 936             /* Bring it back into the free list */
 937             MiInsertPageInFreeList(PageFrameIndex);
 938         }
 939         else
 940         {
 941             /* Otherwise, just drop the reference count */
 942             InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
 943         }
 944     }
 945 }
 946
 947 VOID
 948 NTAPI
 949 MiDecrementReferenceCount(IN PMMPFN Pfn1,
 950                           IN PFN_NUMBER PageFrameIndex)
 951 {
 952     /* PFN lock must be held */
 953     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 954
 955     /* Sanity checks on the page */
 956     ASSERT(PageFrameIndex < MmHighestPhysicalPage);
 957     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
 958     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 959
 960     /* Dereference the page, bail out if it's still alive */
 961     InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
 962     if (Pfn1->u3.e2.ReferenceCount) return;
 963
 964     /* Nobody should still have reference to this page */
 965     if (Pfn1->u2.ShareCount != 0)
 966     {
 967         /* Otherwise something's really wrong */
 968         KeBugCheckEx(PFN_LIST_CORRUPT, 7, PageFrameIndex, Pfn1->u2.ShareCount, 0);
 969     }
 970
 971     /* And it should be lying on some page list */
 972     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
 973
 974     /* Did someone set the delete flag? */
 975     if (MI_IS_PFN_DELETED(Pfn1))
 976     {
 977         /* Insert it into the free list, there's nothing left to do */
 978         MiInsertPageInFreeList(PageFrameIndex);
 979         return;
 980     }
 981
 982     /* We don't have a modified list yet */
 983     ASSERT(Pfn1->u3.e1.Modified == 0);
 984     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 985
 986     /* FIXME: Normally it would go on the standby list, but we're pushing it on the free list */
 987     MiInsertPageInFreeList(PageFrameIndex);
 988 }
 989
 990 VOID
 991 NTAPI
 992 MiInitializePfnForOtherProcess(IN PFN_NUMBER PageFrameIndex,
 993                                IN PMMPTE PointerPte,
 994                                IN PFN_NUMBER PteFrame)
 995 {
 996     PMMPFN Pfn1;
 997
 998     /* Setup the PTE */
 999     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1000     Pfn1->PteAddress = PointerPte;
1001
1002     /* Make this a software PTE */
1003     MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
1004
1005     /* Setup the page */
1006     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
1007     Pfn1->u3.e2.ReferenceCount = 1;
1008     Pfn1->u2.ShareCount = 1;
1009     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1010     Pfn1->u3.e1.Modified = TRUE;
1011     Pfn1->u4.InPageError = FALSE;
1012
1013     /* Did we get a PFN for the page table */
1014     if (PteFrame)
1015     {
1016         /* Store it */
1017         Pfn1->u4.PteFrame = PteFrame;
1018
1019         /* Increase its share count so we don't get rid of it */
1020         Pfn1 = MI_PFN_ELEMENT(PteFrame);
1021         Pfn1->u2.ShareCount++;
1022     }
1023 }
1024
1025 /* EOF */