reactos/ntoskrnl/mm/ARM3/pfnlist.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pfnlist.c
   5  * PURPOSE:         ARM Memory Manager PFN List Manipulation
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #line 15 "ARM³::PFNLIST"
  16 #define MODULE_INVOLVED_IN_ARM3
  17 #include "../ARM3/miarm.h"
  18
  19 #if DBG
  20 #define ASSERT_LIST_INVARIANT(x) \
  21 do { \
  22         ASSERT(((x)->Total == 0 && \
  23             (x)->Flink == LIST_HEAD && \
  24                         (x)->Blink == LIST_HEAD) || \
  25                    ((x)->Total != 0 && \
  26                         (x)->Flink != LIST_HEAD && \
  27                         (x)->Blink != LIST_HEAD)); \
  28 } while (0)
  29 #else
  30 #define ASSERT_LIST_INVARIANT(x)
  31 #endif
  32
  33 /* GLOBALS ********************************************************************/
  34
  35 BOOLEAN MmDynamicPfn;
  36 BOOLEAN MmMirroring;
  37 ULONG MmSystemPageColor;
  38
  39 MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
  40 MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
  41 MMPFNLIST MmStandbyPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  42 MMPFNLIST MmModifiedPageListHead = {0, ModifiedPageList, LIST_HEAD, LIST_HEAD};
  43 MMPFNLIST MmModifiedNoWritePageListHead = {0, ModifiedNoWritePageList, LIST_HEAD, LIST_HEAD};
  44 MMPFNLIST MmBadPageListHead = {0, BadPageList, LIST_HEAD, LIST_HEAD};
  45 MMPFNLIST MmRomPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  46
  47 PMMPFNLIST MmPageLocationList[] =
  48 {
  49     &MmZeroedPageListHead,
  50     &MmFreePageListHead,
  51     &MmStandbyPageListHead,
  52     &MmModifiedPageListHead,
  53     &MmModifiedNoWritePageListHead,
  54     &MmBadPageListHead,
  55     NULL,
  56     NULL
  57 };
  58
  59 ULONG MI_PFN_CURRENT_USAGE;
  60 CHAR MI_PFN_CURRENT_PROCESS_NAME[16] = "None yet";
  61
  62 /* FUNCTIONS ******************************************************************/
  63
  64 VOID
  65 NTAPI
  66 MiZeroPhysicalPage(IN PFN_NUMBER PageFrameIndex)
  67 {
  68     KIRQL OldIrql;
  69     PVOID VirtualAddress;
  70     PEPROCESS Process = PsGetCurrentProcess();
  71
  72     /* Map in hyperspace, then wipe it using XMMI or MEMSET */
  73     VirtualAddress = MiMapPageInHyperSpace(Process, PageFrameIndex, &OldIrql);
  74     ASSERT(VirtualAddress);
  75     KeZeroPages(VirtualAddress, PAGE_SIZE);
  76     MiUnmapPageInHyperSpace(Process, VirtualAddress, OldIrql);
  77 }
  78
  79 VOID
  80 NTAPI
  81 MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
  82 {
  83     PFN_NUMBER OldFlink, OldBlink;
  84     PMMPFNLIST ListHead;
  85     MMLISTS ListName;
  86     ULONG Color;
  87     PMMCOLOR_TABLES ColorTable;
  88     PMMPFN Pfn1;
  89
  90     /* Make sure the PFN lock is held */
  91     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
  92
  93     /* Make sure the PFN entry isn't in-use */
  94     ASSERT(Entry->u3.e1.WriteInProgress == 0);
  95     ASSERT(Entry->u3.e1.ReadInProgress == 0);
  96
  97     /* Find the list for this entry, make sure it's the free or zero list */
  98     ListHead = MmPageLocationList[Entry->u3.e1.PageLocation];
  99     ListName = ListHead->ListName;
 100     ASSERT(ListHead != NULL);
 101     ASSERT(ListName <= FreePageList);
 102     ASSERT_LIST_INVARIANT(ListHead);
 103
 104     /* Remove one count */
 105     ASSERT(ListHead->Total != 0);
 106     ListHead->Total--;
 107
 108     /* Get the forward and back pointers */
 109     OldFlink = Entry->u1.Flink;
 110     OldBlink = Entry->u2.Blink;
 111
 112     /* Check if the next entry is the list head */
 113     if (OldFlink != LIST_HEAD)
 114     {
 115         /* It is not, so set the backlink of the actual entry, to our backlink */
 116         MiGetPfnEntry(OldFlink)->u2.Blink = OldBlink;
 117     }
 118     else
 119     {
 120         /* Set the list head's backlink instead */
 121         ListHead->Blink = OldBlink;
 122     }
 123
 124     /* Check if the back entry is the list head */
 125     if (OldBlink != LIST_HEAD)
 126     {
 127         /* It is not, so set the backlink of the actual entry, to our backlink */
 128         MiGetPfnEntry(OldBlink)->u1.Flink = OldFlink;
 129     }
 130     else
 131     {
 132         /* Set the list head's backlink instead */
 133         ListHead->Flink = OldFlink;
 134     }
 135
 136     /* Get the page color */
 137     OldBlink = MiGetPfnEntryIndex(Entry);
 138     Color = OldBlink & MmSecondaryColorMask;
 139
 140     /* Get the first page on the color list */
 141     ColorTable = &MmFreePagesByColor[ListName][Color];
 142
 143     /* Check if this was was actually the head */
 144     OldFlink = ColorTable->Flink;
 145     if (OldFlink == OldBlink)
 146     {
 147         /* Make the table point to the next page this page was linking to */
 148         ColorTable->Flink = Entry->OriginalPte.u.Long;
 149         if (ColorTable->Flink != LIST_HEAD)
 150         {
 151             /* And make the previous link point to the head now */
 152             MiGetPfnEntry(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 153         }
 154         else
 155         {
 156             /* And if that page was the head, loop the list back around */
 157             ColorTable->Blink = (PVOID)LIST_HEAD;
 158         }
 159     }
 160     else
 161     {
 162         /* This page shouldn't be pointing back to the head */
 163         ASSERT(Entry->u4.PteFrame != COLORED_LIST_HEAD);
 164
 165         /* Make the back link point to whoever the next page is */
 166         Pfn1 = MiGetPfnEntry(Entry->u4.PteFrame);
 167         Pfn1->OriginalPte.u.Long = Entry->OriginalPte.u.Long;
 168
 169         /* Check if this page was pointing to the head */
 170         if (Entry->OriginalPte.u.Long != LIST_HEAD)
 171         {
 172             /* Make the back link point to the head */
 173             Pfn1 = MiGetPfnEntry(Entry->OriginalPte.u.Long);
 174             Pfn1->u4.PteFrame = Entry->u4.PteFrame;
 175         }
 176         else
 177         {
 178             /* Then the table is directly back pointing to this page now */
 179             ColorTable->Blink = Pfn1;
 180         }
 181     }
 182
 183     /* One less colored page */
 184     ASSERT(ColorTable->Count >= 1);
 185     ColorTable->Count--;
 186
 187     /* ReactOS Hack */
 188     Entry->OriginalPte.u.Long = 0;
 189
 190     /* We are not on a list anymore */
 191     Entry->u1.Flink = Entry->u2.Blink = 0;
 192     ASSERT_LIST_INVARIANT(ListHead);
 193
 194     /* See if we hit any thresholds */
 195     if (MmAvailablePages == MmHighMemoryThreshold)
 196     {
 197         /* Clear the high memory event */
 198         KeClearEvent(MiHighMemoryEvent);
 199     }
 200     else if (MmAvailablePages == MmLowMemoryThreshold)
 201     {
 202         /* Signal the low memory event */
 203         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 204     }
 205
 206     /* One less page */
 207     if (--MmAvailablePages < MmMinimumFreePages)
 208     {
 209         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 210     }
 211
 212 #if MI_TRACE_PFNS
 213     ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 214     Entry->PfnUsage = MI_PFN_CURRENT_USAGE;
 215     memcpy(Entry->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 216 //    MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 217 //    memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 218 #endif
 219 }
 220
 221 PFN_NUMBER
 222 NTAPI
 223 MiRemovePageByColor(IN PFN_NUMBER PageIndex,
 224                     IN ULONG Color)
 225 {
 226     PMMPFN Pfn1;
 227     PMMPFNLIST ListHead;
 228     MMLISTS ListName;
 229     PFN_NUMBER OldFlink, OldBlink;
 230     ULONG OldColor, OldCache;
 231     PMMCOLOR_TABLES ColorTable;
 232
 233     /* Make sure PFN lock is held */
 234     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 235     ASSERT(Color < MmSecondaryColors);
 236
 237     /* Get the PFN entry */
 238     Pfn1 = MiGetPfnEntry(PageIndex);
 239     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 240     ASSERT(Pfn1->u3.e1.Rom == 0);
 241
 242     /* Capture data for later */
 243     OldColor = Pfn1->u3.e1.PageColor;
 244     OldCache = Pfn1->u3.e1.CacheAttribute;
 245
 246     /* Could be either on free or zero list */
 247     ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
 248     ASSERT_LIST_INVARIANT(ListHead);
 249     ListName = ListHead->ListName;
 250     ASSERT(ListName <= FreePageList);
 251
 252     /* Remove a page */
 253     ListHead->Total--;
 254
 255     /* Get the forward and back pointers */
 256     OldFlink = Pfn1->u1.Flink;
 257     OldBlink = Pfn1->u2.Blink;
 258
 259     /* Check if the next entry is the list head */
 260     if (OldFlink != LIST_HEAD)
 261     {
 262         /* It is not, so set the backlink of the actual entry, to our backlink */
 263         MiGetPfnEntry(OldFlink)->u2.Blink = OldBlink;
 264     }
 265     else
 266     {
 267         /* Set the list head's backlink instead */
 268         ListHead->Blink = OldBlink;
 269     }
 270
 271     /* Check if the back entry is the list head */
 272     if (OldBlink != LIST_HEAD)
 273     {
 274         /* It is not, so set the backlink of the actual entry, to our backlink */
 275         MiGetPfnEntry(OldBlink)->u1.Flink = OldFlink;
 276     }
 277     else
 278     {
 279         /* Set the list head's backlink instead */
 280         ListHead->Flink = OldFlink;
 281     }
 282
 283     /* We are not on a list anymore */
 284         ASSERT_LIST_INVARIANT(ListHead);
 285     Pfn1->u1.Flink = Pfn1->u2.Blink = 0;
 286
 287     /* Zero flags but restore color and cache */
 288     Pfn1->u3.e2.ShortFlags = 0;
 289     Pfn1->u3.e1.PageColor = OldColor;
 290     Pfn1->u3.e1.CacheAttribute = OldCache;
 291
 292     /* Get the first page on the color list */
 293     ASSERT(Color < MmSecondaryColors);
 294     ColorTable = &MmFreePagesByColor[ListName][Color];
 295     ASSERT(ColorTable->Count >= 1);
 296
 297     /* Set the forward link to whoever we were pointing to */
 298     ColorTable->Flink = Pfn1->OriginalPte.u.Long;
 299
 300     /* Get the first page on the color list */
 301     if (ColorTable->Flink == LIST_HEAD)
 302     {
 303         /* This is the beginning of the list, so set the sentinel value */
 304         ColorTable->Blink = (PVOID)LIST_HEAD;
 305     }
 306     else
 307     {
 308         /* The list is empty, so we are the first page */
 309         MiGetPfnEntry(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 310     }
 311
 312     /* One less page */
 313     ColorTable->Count--;
 314
 315     /* ReactOS Hack */
 316     Pfn1->OriginalPte.u.Long = 0;
 317
 318     /* See if we hit any thresholds */
 319     if (MmAvailablePages == MmHighMemoryThreshold)
 320     {
 321         /* Clear the high memory event */
 322         KeClearEvent(MiHighMemoryEvent);
 323     }
 324     else if (MmAvailablePages == MmLowMemoryThreshold)
 325     {
 326         /* Signal the low memory event */
 327         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 328     }
 329
 330     /* One less page */
 331     if (--MmAvailablePages < MmMinimumFreePages)
 332     {
 333         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 334     }
 335
 336 #if MI_TRACE_PFNS
 337     //ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 338     Pfn1->PfnUsage = MI_PFN_CURRENT_USAGE;
 339     memcpy(Pfn1->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 340     //MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 341     //memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 342 #endif
 343
 344     /* Return the page */
 345     return PageIndex;
 346 }
 347
 348 PFN_NUMBER
 349 NTAPI
 350 MiRemoveAnyPage(IN ULONG Color)
 351 {
 352     PFN_NUMBER PageIndex;
 353     PMMPFN Pfn1;
 354
 355     /* Make sure PFN lock is held and we have pages */
 356     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 357     ASSERT(MmAvailablePages != 0);
 358     ASSERT(Color < MmSecondaryColors);
 359 #if 0
 360     /* Check the colored free list */
 361     PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 362     DPRINT1("Found free page: %lx\n", PageIndex);
 363     if (PageIndex == LIST_HEAD)
 364     {
 365         /* Check the colored zero list */
 366         PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 367         DPRINT1("Found zero page: %lx\n", PageIndex);
 368         if (PageIndex == LIST_HEAD)
 369         {
 370 #endif
 371             /* Check the free list */
 372             ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 373             PageIndex = MmFreePageListHead.Flink;
 374             Color = PageIndex & MmSecondaryColorMask;
 375             if (PageIndex == LIST_HEAD)
 376             {
 377                 /* Check the zero list */
 378                 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 379                 PageIndex = MmZeroedPageListHead.Flink;
 380                 Color = PageIndex & MmSecondaryColorMask;
 381                 ASSERT(PageIndex != LIST_HEAD);
 382                 if (PageIndex == LIST_HEAD)
 383                 {
 384                     /* FIXME: Should check the standby list */
 385                     ASSERT(MmZeroedPageListHead.Total == 0);
 386                 }
 387             }
 388 #if 0
 389         }
 390     }
 391 #endif
 392     /* Remove the page from its list */
 393     PageIndex = MiRemovePageByColor(PageIndex, Color);
 394
 395     /* Sanity checks */
 396     Pfn1 = MiGetPfnEntry(PageIndex);
 397     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 398            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 399     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 400     ASSERT(Pfn1->u2.ShareCount == 0);
 401     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 402     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 403
 404     /* Return the page */
 405     return PageIndex;
 406 }
 407
 408 PFN_NUMBER
 409 NTAPI
 410 MiRemoveZeroPage(IN ULONG Color)
 411 {
 412     PFN_NUMBER PageIndex;
 413     PMMPFN Pfn1;
 414     BOOLEAN Zero;
 415
 416     /* Make sure PFN lock is held and we have pages */
 417     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 418     ASSERT(MmAvailablePages != 0);
 419     ASSERT(Color < MmSecondaryColors);
 420
 421     /* Check the colored zero list */
 422 #if 0
 423     PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 424     if (PageIndex == LIST_HEAD)
 425     {
 426 #endif
 427         /* Check the zero list */
 428         ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 429         PageIndex = MmZeroedPageListHead.Flink;
 430         Color = PageIndex & MmSecondaryColorMask;
 431         if (PageIndex == LIST_HEAD)
 432         {
 433             /* This means there's no zero pages, we have to look for free ones */
 434             ASSERT(MmZeroedPageListHead.Total == 0);
 435             Zero = TRUE;
 436 #if 0
 437             /* Check the colored free list */
 438             PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 439             if (PageIndex == LIST_HEAD)
 440             {
 441 #endif
 442                 /* Check the free list */
 443                 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 444                 PageIndex = MmFreePageListHead.Flink;
 445                 Color = PageIndex & MmSecondaryColorMask;
 446                 ASSERT(PageIndex != LIST_HEAD);
 447                 if (PageIndex == LIST_HEAD)
 448                 {
 449                     /* FIXME: Should check the standby list */
 450                     ASSERT(MmZeroedPageListHead.Total == 0);
 451                 }
 452 #if 0
 453             }
 454 #endif
 455         }
 456 #if 0
 457     }
 458 #endif
 459
 460     /* Sanity checks */
 461     Pfn1 = MiGetPfnEntry(PageIndex);
 462     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 463            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 464
 465     /* Remove the page from its list */
 466     PageIndex = MiRemovePageByColor(PageIndex, Color);
 467     ASSERT(Pfn1 == MiGetPfnEntry(PageIndex));
 468
 469     /* Zero it, if needed */
 470     if (Zero) MiZeroPhysicalPage(PageIndex);
 471
 472     /* Sanity checks */
 473     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 474     ASSERT(Pfn1->u2.ShareCount == 0);
 475     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 476     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 477
 478     /* Return the page */
 479     return PageIndex;
 480 }
 481
 482 VOID
 483 NTAPI
 484 MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
 485 {
 486     PMMPFNLIST ListHead;
 487     PFN_NUMBER LastPage;
 488     PMMPFN Pfn1;
 489     ULONG Color;
 490     PMMPFN Blink;
 491     PMMCOLOR_TABLES ColorTable;
 492
 493     /* Make sure the page index is valid */
 494     ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
 495     ASSERT((PageFrameIndex != 0) &&
 496            (PageFrameIndex <= MmHighestPhysicalPage) &&
 497            (PageFrameIndex >= MmLowestPhysicalPage));
 498
 499     /* Get the PFN entry */
 500     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 501
 502     /* Sanity checks that a right kind of page is being inserted here */
 503     ASSERT(Pfn1->u4.MustBeCached == 0);
 504     ASSERT(Pfn1->u3.e1.Rom != 1);
 505     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 506     ASSERT(Pfn1->u4.VerifierAllocation == 0);
 507     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 508
 509     /* Get the free page list and increment its count */
 510     ListHead = &MmFreePageListHead;
 511     ASSERT_LIST_INVARIANT(ListHead);
 512     ListHead->Total++;
 513
 514     /* Get the last page on the list */
 515     LastPage = ListHead->Blink;
 516     if (LastPage != LIST_HEAD)
 517     {
 518         /* Link us with the previous page, so we're at the end now */
 519         MiGetPfnEntry(LastPage)->u1.Flink = PageFrameIndex;
 520     }
 521     else
 522     {
 523         /* The list is empty, so we are the first page */
 524         ListHead->Flink = PageFrameIndex;
 525     }
 526
 527     /* Now make the list head point back to us (since we go at the end) */
 528     ListHead->Blink = PageFrameIndex;
 529     ASSERT_LIST_INVARIANT(ListHead);
 530
 531     /* And initialize our own list pointers */
 532     Pfn1->u1.Flink = LIST_HEAD;
 533     Pfn1->u2.Blink = LastPage;
 534
 535     /* Set the list name and default priority */
 536     Pfn1->u3.e1.PageLocation = FreePageList;
 537     Pfn1->u4.Priority = 3;
 538
 539     /* Clear some status fields */
 540     Pfn1->u4.InPageError = 0;
 541     Pfn1->u4.AweAllocation = 0;
 542
 543     /* Increase available pages */
 544     MmAvailablePages++;
 545
 546     /* Check if we've reached the configured low memory threshold */
 547     if (MmAvailablePages == MmLowMemoryThreshold)
 548     {
 549         /* Clear the event, because now we're ABOVE the threshold */
 550         KeClearEvent(MiLowMemoryEvent);
 551     }
 552     else if (MmAvailablePages == MmHighMemoryThreshold)
 553     {
 554         /* Otherwise check if we reached the high threshold and signal the event */
 555         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 556     }
 557
 558     /* Get the page color */
 559     Color = PageFrameIndex & MmSecondaryColorMask;
 560
 561     /* Get the first page on the color list */
 562     ColorTable = &MmFreePagesByColor[FreePageList][Color];
 563     if (ColorTable->Flink == LIST_HEAD)
 564     {
 565         /* The list is empty, so we are the first page */
 566         Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 567         ColorTable->Flink = PageFrameIndex;
 568     }
 569     else
 570     {
 571         /* Get the previous page */
 572         Blink = (PMMPFN)ColorTable->Blink;
 573
 574         /* Make it link to us */
 575         Pfn1->u4.PteFrame = MiGetPfnEntryIndex(Blink);
 576
 577         /* If there is an original pte, it should be an old link, NOT a ReactOS RMAP */
 578         ASSERT(Blink->u4.AweAllocation == FALSE);
 579         Blink->OriginalPte.u.Long = PageFrameIndex;
 580     }
 581
 582     /* Now initialize our own list pointers */
 583     ColorTable->Blink = Pfn1;
 584
 585     /* If there is an original pte, it should be an old link, NOT a ReactOS RMAP */
 586     ASSERT(Pfn1->u4.AweAllocation == FALSE);
 587     Pfn1->OriginalPte.u.Long = LIST_HEAD;
 588
 589     /* And increase the count in the colored list */
 590     ColorTable->Count++;
 591
 592     /* Notify zero page thread if enough pages are on the free list now */
 593     if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive))
 594     {
 595         /* Set the event */
 596         MmZeroingPageThreadActive = TRUE;
 597         KeSetEvent(&MmZeroingPageEvent, IO_NO_INCREMENT, FALSE);
 598     }
 599
 600 #if MI_TRACE_PFNS
 601     Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 602     RtlZeroMemory(Pfn1->ProcessName, 16);
 603 #endif
 604 }
 605
 606 /* Note: This function is hardcoded only for the zeroed page list, for now */
 607 VOID
 608 NTAPI
 609 MiInsertPageInList(IN PMMPFNLIST ListHead,
 610                    IN PFN_NUMBER PageFrameIndex)
 611 {
 612     PFN_NUMBER Flink;
 613     PMMPFN Pfn1, Pfn2;
 614     MMLISTS ListName;
 615     PMMCOLOR_TABLES ColorHead;
 616     ULONG Color;
 617
 618     /* For free pages, use MiInsertPageInFreeList */
 619     ASSERT(ListHead != &MmFreePageListHead);
 620
 621     /* Make sure the lock is held */
 622     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 623
 624     /* Make sure the PFN is valid */
 625     ASSERT((PageFrameIndex) &&
 626            (PageFrameIndex <= MmHighestPhysicalPage) &&
 627            (PageFrameIndex >= MmLowestPhysicalPage));
 628
 629     /* Page should be unused */
 630     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 631     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 632     ASSERT(Pfn1->u3.e1.Rom != 1);
 633
 634     /* Only used for zero pages in ReactOS */
 635     ListName = ListHead->ListName;
 636     ASSERT(ListName == ZeroedPageList);
 637     ListHead->Total++;
 638
 639     /* Don't handle bad pages yet yet */
 640     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 641
 642     /* Make the head of the list point to this page now */
 643     Flink = ListHead->Flink;
 644     ListHead->Flink = PageFrameIndex;
 645
 646     /* Make the page point to the previous head, and back to the list */
 647     Pfn1->u1.Flink = Flink;
 648     Pfn1->u2.Blink = LIST_HEAD;
 649
 650     /* Was the list empty? */
 651     if (Flink != LIST_HEAD)
 652     {
 653         /* It wasn't, so update the backlink of the previous head page */
 654         Pfn2 = MiGetPfnEntry(Flink);
 655         Pfn2->u2.Blink = PageFrameIndex;
 656     }
 657     else
 658     {
 659         /* It was empty, so have it loop back around to this new page */
 660         ListHead->Blink = PageFrameIndex;
 661     }
 662
 663     /* Move the page onto its new location */
 664     Pfn1->u3.e1.PageLocation = ListName;
 665
 666     /* One more page on the system */
 667     MmAvailablePages++;
 668
 669     /* Check if we've reached the configured low memory threshold */
 670     if (MmAvailablePages == MmLowMemoryThreshold)
 671     {
 672         /* Clear the event, because now we're ABOVE the threshold */
 673         KeClearEvent(MiLowMemoryEvent);
 674     }
 675     else if (MmAvailablePages == MmHighMemoryThreshold)
 676     {
 677         /* Otherwise check if we reached the high threshold and signal the event */
 678         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 679     }
 680
 681     /* Sanity checks */
 682     ASSERT(ListName == ZeroedPageList);
 683     ASSERT(Pfn1->u4.InPageError == 0);
 684
 685     /* Get the page color */
 686     Color = PageFrameIndex & MmSecondaryColorMask;
 687
 688     /* Get the list for this color */
 689     ColorHead = &MmFreePagesByColor[ZeroedPageList][Color];
 690
 691     /* Get the old head */
 692     Flink = ColorHead->Flink;
 693
 694     /* If there is an original pte, it should be an old link, NOT a ReactOS RMAP */
 695     ASSERT(Pfn1->u4.AweAllocation == FALSE);
 696
 697     /* Make this page point back to the list, and point forwards to the old head */
 698     Pfn1->OriginalPte.u.Long = Flink;
 699     Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 700
 701     /* Set the new head */
 702     ColorHead->Flink = PageFrameIndex;
 703
 704     /* Was the head empty? */
 705     if (Flink != LIST_HEAD)
 706     {
 707         /* No, so make the old head point to this page */
 708         Pfn2 = MiGetPfnEntry(Flink);
 709         Pfn2->u4.PteFrame = PageFrameIndex;
 710     }
 711     else
 712     {
 713         /* Yes, make it loop back to this page */
 714         ColorHead->Blink = (PVOID)Pfn1;
 715     }
 716
 717     /* One more paged on the colored list */
 718     ColorHead->Count++;
 719
 720 #if MI_TRACE_PFNS
 721     //ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET);
 722     Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 723     MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 724     RtlZeroMemory(Pfn1->ProcessName, 16);
 725 #endif
 726 }
 727
 728 VOID
 729 NTAPI
 730 MiInitializePfn(IN PFN_NUMBER PageFrameIndex,
 731                 IN PMMPTE PointerPte,
 732                 IN BOOLEAN Modified)
 733 {
 734     PMMPFN Pfn1;
 735     NTSTATUS Status;
 736     PMMPTE PointerPtePte;
 737     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 738
 739     /* Setup the PTE */
 740     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 741     Pfn1->PteAddress = PointerPte;
 742
 743     /* Check if this PFN is part of a valid address space */
 744     if (PointerPte->u.Hard.Valid == 1)
 745     {
 746         /* Only valid from MmCreateProcessAddressSpace path */
 747         ASSERT(PsGetCurrentProcess()->Vm.WorkingSetSize == 0);
 748
 749         /* Make this a demand zero PTE */
 750         MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
 751     }
 752     else
 753     {
 754         /* Copy the PTE data */
 755         Pfn1->OriginalPte = *PointerPte;
 756         ASSERT(!((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
 757                  (Pfn1->OriginalPte.u.Soft.Transition == 1)));
 758     }
 759
 760     /* Otherwise this is a fresh page -- set it up */
 761     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 762     Pfn1->u3.e2.ReferenceCount = 1;
 763     Pfn1->u2.ShareCount = 1;
 764     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 765     ASSERT(Pfn1->u3.e1.Rom == 0);
 766     Pfn1->u3.e1.Modified = Modified;
 767
 768     /* Get the page table for the PTE */
 769     PointerPtePte = MiAddressToPte(PointerPte);
 770     if (PointerPtePte->u.Hard.Valid == 0)
 771     {
 772         /* Make sure the PDE gets paged in properly */
 773         Status = MiCheckPdeForPagedPool(PointerPte);
 774         if (!NT_SUCCESS(Status))
 775         {
 776             /* Crash */
 777             KeBugCheckEx(MEMORY_MANAGEMENT,
 778                          0x61940,
 779                          (ULONG_PTR)PointerPte,
 780                          (ULONG_PTR)PointerPtePte->u.Long,
 781                          (ULONG_PTR)MiPteToAddress(PointerPte));
 782         }
 783     }
 784
 785     /* Get the PFN for the page table */
 786     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
 787     ASSERT(PageFrameIndex != 0);
 788     Pfn1->u4.PteFrame = PageFrameIndex;
 789
 790     /* Increase its share count so we don't get rid of it */
 791     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 792     Pfn1->u2.ShareCount++;
 793 }
 794
 795 PFN_NUMBER
 796 NTAPI
 797 MiAllocatePfn(IN PMMPTE PointerPte,
 798               IN ULONG Protection)
 799 {
 800     KIRQL OldIrql;
 801     PFN_NUMBER PageFrameIndex;
 802     MMPTE TempPte;
 803
 804     /* Sanity check that we aren't passed a valid PTE */
 805     ASSERT(PointerPte->u.Hard.Valid == 0);
 806
 807     /* Make an empty software PTE */
 808     MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
 809
 810     /* Lock the PFN database */
 811     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 812
 813     /* Check if we're running low on pages */
 814     if (MmAvailablePages < 128)
 815     {
 816         DPRINT1("Warning, running low on memory: %d pages left\n", MmAvailablePages);
 817         //MiEnsureAvailablePageOrWait(NULL, OldIrql);
 818     }
 819
 820     /* Grab a page */
 821     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 822     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 823     PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
 824
 825     /* Write the software PTE */
 826     MI_WRITE_INVALID_PTE(PointerPte, TempPte);
 827     PointerPte->u.Soft.Protection |= Protection;
 828
 829     /* Initialize its PFN entry */
 830     MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
 831
 832     /* Release the PFN lock and return the page */
 833     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 834     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 835     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 836     return PageFrameIndex;
 837 }
 838
 839 VOID
 840 NTAPI
 841 MiDecrementShareCount(IN PMMPFN Pfn1,
 842                       IN PFN_NUMBER PageFrameIndex)
 843 {
 844     ASSERT(PageFrameIndex > 0);
 845     ASSERT(MiGetPfnEntry(PageFrameIndex) != NULL);
 846     ASSERT(Pfn1 == MiGetPfnEntry(PageFrameIndex));
 847     ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
 848
 849     /* Page must be in-use */
 850     if ((Pfn1->u3.e1.PageLocation != ActiveAndValid) &&
 851         (Pfn1->u3.e1.PageLocation != StandbyPageList))
 852     {
 853         /* Otherwise we have PFN corruption */
 854         KeBugCheckEx(PFN_LIST_CORRUPT,
 855                      0x99,
 856                      PageFrameIndex,
 857                      Pfn1->u3.e1.PageLocation,
 858                      0);
 859     }
 860
 861     /* Check if the share count is now 0 */
 862     ASSERT(Pfn1->u2.ShareCount < 0xF000000);
 863     if (!--Pfn1->u2.ShareCount)
 864     {
 865         /* ReactOS does not handle these */
 866         ASSERT(Pfn1->u3.e1.PrototypePte == 0);
 867
 868         /* Put the page in transition */
 869         Pfn1->u3.e1.PageLocation = TransitionPage;
 870
 871         /* PFN lock must be held */
 872         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 873
 874         /* Page should at least have one reference */
 875         ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 876         if (Pfn1->u3.e2.ReferenceCount == 1)
 877         {
 878             /* In ReactOS, this path should always be hit with a deleted PFN */
 879             ASSERT(MI_IS_PFN_DELETED(Pfn1) == TRUE);
 880
 881             /* Clear the last reference */
 882             Pfn1->u3.e2.ReferenceCount = 0;
 883
 884             /*
 885              * OriginalPte is used by AweReferenceCount in ReactOS, but either
 886              * ways we shouldn't be seeing RMAP entries at this point
 887              */
 888             ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 889             ASSERT(Pfn1->u4.AweAllocation == FALSE);
 890
 891             /* Mark the page temporarily as valid, we're going to make it free soon */
 892             Pfn1->u3.e1.PageLocation = ActiveAndValid;
 893
 894             /* Bring it back into the free list */
 895             MiInsertPageInFreeList(PageFrameIndex);
 896         }
 897         else
 898         {
 899             /* Otherwise, just drop the reference count */
 900             InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
 901         }
 902     }
 903 }
 904
 905 VOID
 906 NTAPI
 907 MiDecrementReferenceCount(IN PMMPFN Pfn1,
 908                           IN PFN_NUMBER PageFrameIndex)
 909 {
 910     /* PFN lock must be held */
 911     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 912
 913     /* Sanity checks on the page */
 914     ASSERT(PageFrameIndex < MmHighestPhysicalPage);
 915     ASSERT(Pfn1 == MiGetPfnEntry(PageFrameIndex));
 916     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
 917
 918     /* Dereference the page, bail out if it's still alive */
 919     InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
 920     if (Pfn1->u3.e2.ReferenceCount) return;
 921
 922     /* Nobody should still have reference to this page */
 923     if (Pfn1->u2.ShareCount != 0)
 924     {
 925         /* Otherwise something's really wrong */
 926         KeBugCheckEx(PFN_LIST_CORRUPT, 7, PageFrameIndex, Pfn1->u2.ShareCount, 0);
 927     }
 928
 929     /* And it should be lying on some page list */
 930     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
 931
 932     /* Did someone set the delete flag? */
 933     if (MI_IS_PFN_DELETED(Pfn1))
 934     {
 935         /* Insert it into the free list, there's nothing left to do */
 936         MiInsertPageInFreeList(PageFrameIndex);
 937         return;
 938     }
 939
 940     /* We don't have a modified list yet */
 941     ASSERT(Pfn1->u3.e1.Modified == 0);
 942     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 943
 944     /* FIXME: Normally it would go on the standby list, but we're pushing it on the free list */
 945     MiInsertPageInFreeList(PageFrameIndex);
 946 }
 947
 948 VOID
 949 NTAPI
 950 MiInitializePfnForOtherProcess(IN PFN_NUMBER PageFrameIndex,
 951                                IN PMMPTE PointerPte,
 952                                IN PFN_NUMBER PteFrame)
 953 {
 954     PMMPFN Pfn1;
 955
 956     /* Setup the PTE */
 957     Pfn1 = MiGetPfnEntry(PageFrameIndex);
 958     Pfn1->PteAddress = PointerPte;
 959
 960     /* Make this a software PTE */
 961     MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
 962
 963     /* Setup the page */
 964     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 965     Pfn1->u3.e2.ReferenceCount = 1;
 966     Pfn1->u2.ShareCount = 1;
 967     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 968     Pfn1->u3.e1.Modified = TRUE;
 969     Pfn1->u4.InPageError = FALSE;
 970
 971     /* Did we get a PFN for the page table */
 972     if (PteFrame)
 973     {
 974         /* Store it */
 975         Pfn1->u4.PteFrame = PteFrame;
 976
 977         /* Increase its share count so we don't get rid of it */
 978         Pfn1 = MiGetPfnEntry(PteFrame);
 979         Pfn1->u2.ShareCount++;
 980     }
 981 }
 982
 983 /* EOF */