ntoskrnl/mm/ARM3/pfnlist.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pfnlist.c
   5  * PURPOSE:         ARM Memory Manager PFN List Manipulation
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include "../ARM3/miarm.h"
  17
  18 #if DBG
  19 #define ASSERT_LIST_INVARIANT(x) \
  20 do { \
  21     ASSERT(((x)->Total == 0 && \
  22             (x)->Flink == LIST_HEAD && \
  23             (x)->Blink == LIST_HEAD) || \
  24            ((x)->Total != 0 && \
  25             (x)->Flink != LIST_HEAD && \
  26             (x)->Blink != LIST_HEAD)); \
  27 } while (0)
  28 #else
  29 #define ASSERT_LIST_INVARIANT(x)
  30 #endif
  31
  32 /* GLOBALS ********************************************************************/
  33
  34 BOOLEAN MmDynamicPfn;
  35 BOOLEAN MmMirroring;
  36 ULONG MmSystemPageColor;
  37
  38 ULONG MmTransitionSharedPages;
  39 ULONG MmTotalPagesForPagingFile;
  40
  41 MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
  42 MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
  43 MMPFNLIST MmStandbyPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  44 MMPFNLIST MmStandbyPageListByPriority[8];
  45 MMPFNLIST MmModifiedPageListHead = {0, ModifiedPageList, LIST_HEAD, LIST_HEAD};
  46 MMPFNLIST MmModifiedPageListByColor[1] = {{0, ModifiedPageList, LIST_HEAD, LIST_HEAD}};
  47 MMPFNLIST MmModifiedNoWritePageListHead = {0, ModifiedNoWritePageList, LIST_HEAD, LIST_HEAD};
  48 MMPFNLIST MmBadPageListHead = {0, BadPageList, LIST_HEAD, LIST_HEAD};
  49 MMPFNLIST MmRomPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  50
  51 PMMPFNLIST MmPageLocationList[] =
  52 {
  53     &MmZeroedPageListHead,
  54     &MmFreePageListHead,
  55     &MmStandbyPageListHead,
  56     &MmModifiedPageListHead,
  57     &MmModifiedNoWritePageListHead,
  58     &MmBadPageListHead,
  59     NULL,
  60     NULL
  61 };
  62
  63 ULONG MI_PFN_CURRENT_USAGE;
  64 CHAR MI_PFN_CURRENT_PROCESS_NAME[16] = "None yet";
  65
  66 /* FUNCTIONS ******************************************************************/
  67
  68 VOID
  69 NTAPI
  70 MiZeroPhysicalPage(IN PFN_NUMBER PageFrameIndex)
  71 {
  72     KIRQL OldIrql;
  73     PVOID VirtualAddress;
  74     PEPROCESS Process = PsGetCurrentProcess();
  75
  76     /* Map in hyperspace, then wipe it using XMMI or MEMSET */
  77     VirtualAddress = MiMapPageInHyperSpace(Process, PageFrameIndex, &OldIrql);
  78     ASSERT(VirtualAddress);
  79     KeZeroPages(VirtualAddress, PAGE_SIZE);
  80     MiUnmapPageInHyperSpace(Process, VirtualAddress, OldIrql);
  81 }
  82
  83 VOID
  84 NTAPI
  85 MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
  86 {
  87     PFN_NUMBER OldFlink, OldBlink;
  88     PMMPFNLIST ListHead;
  89     MMLISTS ListName;
  90     ULONG Color;
  91     PMMCOLOR_TABLES ColorTable;
  92     PMMPFN Pfn1;
  93
  94     /* Make sure the PFN lock is held */
  95     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
  96
  97     /* Make sure the PFN entry isn't in-use */
  98     ASSERT(Entry->u3.e1.WriteInProgress == 0);
  99     ASSERT(Entry->u3.e1.ReadInProgress == 0);
 100
 101     /* Find the list for this entry, make sure it's the free or zero list */
 102     ListHead = MmPageLocationList[Entry->u3.e1.PageLocation];
 103     ListName = ListHead->ListName;
 104     ASSERT(ListHead != NULL);
 105     ASSERT(ListName <= FreePageList);
 106     ASSERT_LIST_INVARIANT(ListHead);
 107
 108     /* Remove one count */
 109     ASSERT(ListHead->Total != 0);
 110     ListHead->Total--;
 111
 112     /* Get the forward and back pointers */
 113     OldFlink = Entry->u1.Flink;
 114     OldBlink = Entry->u2.Blink;
 115
 116     /* Check if the next entry is the list head */
 117     if (OldFlink != LIST_HEAD)
 118     {
 119         /* It is not, so set the backlink of the actual entry, to our backlink */
 120         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 121     }
 122     else
 123     {
 124         /* Set the list head's backlink instead */
 125         ListHead->Blink = OldBlink;
 126     }
 127
 128     /* Check if the back entry is the list head */
 129     if (OldBlink != LIST_HEAD)
 130     {
 131         /* It is not, so set the backlink of the actual entry, to our backlink */
 132         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 133     }
 134     else
 135     {
 136         /* Set the list head's backlink instead */
 137         ListHead->Flink = OldFlink;
 138     }
 139
 140     /* Get the page color */
 141     OldBlink = MiGetPfnEntryIndex(Entry);
 142     Color = OldBlink & MmSecondaryColorMask;
 143
 144     /* Get the first page on the color list */
 145     ColorTable = &MmFreePagesByColor[ListName][Color];
 146
 147     /* Check if this was was actually the head */
 148     OldFlink = ColorTable->Flink;
 149     if (OldFlink == OldBlink)
 150     {
 151         /* Make the table point to the next page this page was linking to */
 152         ColorTable->Flink = Entry->OriginalPte.u.Long;
 153         if (ColorTable->Flink != LIST_HEAD)
 154         {
 155             /* And make the previous link point to the head now */
 156             MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 157         }
 158         else
 159         {
 160             /* And if that page was the head, loop the list back around */
 161             ColorTable->Blink = (PVOID)LIST_HEAD;
 162         }
 163     }
 164     else
 165     {
 166         /* This page shouldn't be pointing back to the head */
 167         ASSERT(Entry->u4.PteFrame != COLORED_LIST_HEAD);
 168
 169         /* Make the back link point to whoever the next page is */
 170         Pfn1 = MI_PFN_ELEMENT(Entry->u4.PteFrame);
 171         Pfn1->OriginalPte.u.Long = Entry->OriginalPte.u.Long;
 172
 173         /* Check if this page was pointing to the head */
 174         if (Entry->OriginalPte.u.Long != LIST_HEAD)
 175         {
 176             /* Make the back link point to the head */
 177             Pfn1 = MI_PFN_ELEMENT(Entry->OriginalPte.u.Long);
 178             Pfn1->u4.PteFrame = Entry->u4.PteFrame;
 179         }
 180         else
 181         {
 182             /* Then the table is directly back pointing to this page now */
 183             ColorTable->Blink = Pfn1;
 184         }
 185     }
 186
 187     /* One less colored page */
 188     ASSERT(ColorTable->Count >= 1);
 189     ColorTable->Count--;
 190
 191     /* ReactOS Hack */
 192     Entry->OriginalPte.u.Long = 0;
 193
 194     /* We are not on a list anymore */
 195     Entry->u1.Flink = Entry->u2.Blink = 0;
 196     ASSERT_LIST_INVARIANT(ListHead);
 197
 198     /* See if we hit any thresholds */
 199     if (MmAvailablePages == MmHighMemoryThreshold)
 200     {
 201         /* Clear the high memory event */
 202         KeClearEvent(MiHighMemoryEvent);
 203     }
 204     else if (MmAvailablePages == MmLowMemoryThreshold)
 205     {
 206         /* Signal the low memory event */
 207         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 208     }
 209
 210     /* One less page */
 211     if (--MmAvailablePages < MmMinimumFreePages)
 212     {
 213         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 214
 215         DPRINT1("Running low on pages: %lu remaining\n", MmAvailablePages);
 216
 217         /* Call RosMm and see if it can release any pages for us */
 218         MmRebalanceMemoryConsumers();
 219     }
 220
 221 #if MI_TRACE_PFNS
 222     ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 223     Entry->PfnUsage = MI_PFN_CURRENT_USAGE;
 224     memcpy(Entry->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 225 //    MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 226 //    memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 227 #endif
 228 }
 229
 230 VOID
 231 NTAPI
 232 MiUnlinkPageFromList(IN PMMPFN Pfn)
 233 {
 234     PMMPFNLIST ListHead;
 235     PFN_NUMBER OldFlink, OldBlink;
 236
 237     /* Make sure the PFN lock is held */
 238     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 239
 240     /* ARM3 should only call this for dead pages */
 241     ASSERT(Pfn->u3.e2.ReferenceCount == 0);
 242
 243     /* Transition pages are supposed to be standby/modified/nowrite */
 244     ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
 245     ASSERT(ListHead->ListName >= StandbyPageList);
 246
 247     /* Check if this was standby, or modified */
 248     if (ListHead == &MmStandbyPageListHead)
 249     {
 250         /* Should not be a ROM page */
 251         ASSERT(Pfn->u3.e1.Rom == 0);
 252
 253         /* Get the exact list */
 254         ListHead = &MmStandbyPageListByPriority[Pfn->u4.Priority];
 255
 256         /* See if we hit any thresholds */
 257         if (MmAvailablePages == MmHighMemoryThreshold)
 258         {
 259             /* Clear the high memory event */
 260             KeClearEvent(MiHighMemoryEvent);
 261         }
 262         else if (MmAvailablePages == MmLowMemoryThreshold)
 263         {
 264             /* Signal the low memory event */
 265             KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 266         }
 267
 268         /* Decrease transition page counter */
 269         ASSERT(Pfn->u3.e1.PrototypePte == 1); /* Only supported ARM3 case */
 270         MmTransitionSharedPages--;
 271
 272         /* One less page */
 273         if (--MmAvailablePages < MmMinimumFreePages)
 274         {
 275             /* FIXME: Should wake up the MPW and working set manager, if we had one */
 276             DPRINT1("Running low on pages: %lu remaining\n", MmAvailablePages);
 277
 278             /* Call RosMm and see if it can release any pages for us */
 279             MmRebalanceMemoryConsumers();
 280         }
 281     }
 282     else if (ListHead == &MmModifiedPageListHead)
 283     {
 284         /* Only shared memory (page-file backed) modified pages are supported */
 285         ASSERT(Pfn->OriginalPte.u.Soft.Prototype == 0);
 286
 287         /* Decrement the counters */
 288         ListHead->Total--;
 289         MmTotalPagesForPagingFile--;
 290
 291         /* Pick the correct colored list */
 292         ListHead = &MmModifiedPageListByColor[0];
 293
 294         /* Decrease transition page counter */
 295         MmTransitionSharedPages--;
 296     }
 297     else if (ListHead == &MmModifiedNoWritePageListHead)
 298     {
 299         /* List not yet supported */
 300         ASSERT(FALSE);
 301     }
 302
 303     /* Nothing should be in progress and the list should not be empty */
 304     ASSERT(Pfn->u3.e1.WriteInProgress == 0);
 305     ASSERT(Pfn->u3.e1.ReadInProgress == 0);
 306     ASSERT(ListHead->Total != 0);
 307
 308     /* Get the forward and back pointers */
 309     OldFlink = Pfn->u1.Flink;
 310     OldBlink = Pfn->u2.Blink;
 311
 312     /* Check if the next entry is the list head */
 313     if (OldFlink != LIST_HEAD)
 314     {
 315         /* It is not, so set the backlink of the actual entry, to our backlink */
 316         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 317     }
 318     else
 319     {
 320         /* Set the list head's backlink instead */
 321         ListHead->Blink = OldBlink;
 322     }
 323
 324     /* Check if the back entry is the list head */
 325     if (OldBlink != LIST_HEAD)
 326     {
 327         /* It is not, so set the backlink of the actual entry, to our backlink */
 328         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 329     }
 330     else
 331     {
 332         /* Set the list head's backlink instead */
 333         ListHead->Flink = OldFlink;
 334     }
 335
 336     /* ReactOS Hack */
 337     Pfn->OriginalPte.u.Long = 0;
 338
 339     /* We are not on a list anymore */
 340     Pfn->u1.Flink = Pfn->u2.Blink = 0;
 341     ASSERT_LIST_INVARIANT(ListHead);
 342
 343     /* Remove one entry from the list */
 344     ListHead->Total--;
 345 }
 346
 347 PFN_NUMBER
 348 NTAPI
 349 MiRemovePageByColor(IN PFN_NUMBER PageIndex,
 350                     IN ULONG Color)
 351 {
 352     PMMPFN Pfn1;
 353     PMMPFNLIST ListHead;
 354     MMLISTS ListName;
 355     PFN_NUMBER OldFlink, OldBlink;
 356     USHORT OldColor, OldCache;
 357     PMMCOLOR_TABLES ColorTable;
 358
 359     /* Make sure PFN lock is held */
 360     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 361     ASSERT(Color < MmSecondaryColors);
 362
 363     /* Get the PFN entry */
 364     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 365     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 366     ASSERT(Pfn1->u3.e1.Rom == 0);
 367
 368     /* Capture data for later */
 369     OldColor = Pfn1->u3.e1.PageColor;
 370     OldCache = Pfn1->u3.e1.CacheAttribute;
 371
 372     /* Could be either on free or zero list */
 373     ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
 374     ASSERT_LIST_INVARIANT(ListHead);
 375     ListName = ListHead->ListName;
 376     ASSERT(ListName <= FreePageList);
 377
 378     /* Remove a page */
 379     ListHead->Total--;
 380
 381     /* Get the forward and back pointers */
 382     OldFlink = Pfn1->u1.Flink;
 383     OldBlink = Pfn1->u2.Blink;
 384
 385     /* Check if the next entry is the list head */
 386     if (OldFlink != LIST_HEAD)
 387     {
 388         /* It is not, so set the backlink of the actual entry, to our backlink */
 389         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 390     }
 391     else
 392     {
 393         /* Set the list head's backlink instead */
 394         ListHead->Blink = OldBlink;
 395     }
 396
 397     /* Check if the back entry is the list head */
 398     if (OldBlink != LIST_HEAD)
 399     {
 400         /* It is not, so set the backlink of the actual entry, to our backlink */
 401         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 402     }
 403     else
 404     {
 405         /* Set the list head's backlink instead */
 406         ListHead->Flink = OldFlink;
 407     }
 408
 409     /* We are not on a list anymore */
 410     ASSERT_LIST_INVARIANT(ListHead);
 411     Pfn1->u1.Flink = Pfn1->u2.Blink = 0;
 412
 413     /* Zero flags but restore color and cache */
 414     Pfn1->u3.e2.ShortFlags = 0;
 415     Pfn1->u3.e1.PageColor = OldColor;
 416     Pfn1->u3.e1.CacheAttribute = OldCache;
 417
 418     /* Get the first page on the color list */
 419     ASSERT(Color < MmSecondaryColors);
 420     ColorTable = &MmFreePagesByColor[ListName][Color];
 421     ASSERT(ColorTable->Count >= 1);
 422
 423     /* Set the forward link to whoever we were pointing to */
 424     ColorTable->Flink = Pfn1->OriginalPte.u.Long;
 425
 426     /* Get the first page on the color list */
 427     if (ColorTable->Flink == LIST_HEAD)
 428     {
 429         /* This is the beginning of the list, so set the sentinel value */
 430         ColorTable->Blink = (PVOID)LIST_HEAD;
 431     }
 432     else
 433     {
 434         /* The list is empty, so we are the first page */
 435         MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 436     }
 437
 438     /* One less page */
 439     ColorTable->Count--;
 440
 441     /* ReactOS Hack */
 442     Pfn1->OriginalPte.u.Long = 0;
 443
 444     /* See if we hit any thresholds */
 445     if (MmAvailablePages == MmHighMemoryThreshold)
 446     {
 447         /* Clear the high memory event */
 448         KeClearEvent(MiHighMemoryEvent);
 449     }
 450     else if (MmAvailablePages == MmLowMemoryThreshold)
 451     {
 452         /* Signal the low memory event */
 453         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 454     }
 455
 456     /* One less page */
 457     if (--MmAvailablePages < MmMinimumFreePages)
 458     {
 459         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 460
 461         DPRINT1("Running low on pages: %lu remaining\n", MmAvailablePages);
 462
 463         /* Call RosMm and see if it can release any pages for us */
 464         MmRebalanceMemoryConsumers();
 465     }
 466
 467 #if MI_TRACE_PFNS
 468     //ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 469     Pfn1->PfnUsage = MI_PFN_CURRENT_USAGE;
 470     memcpy(Pfn1->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 471     //MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 472     //memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 473 #endif
 474
 475     /* Return the page */
 476     return PageIndex;
 477 }
 478
 479 PFN_NUMBER
 480 NTAPI
 481 MiRemoveAnyPage(IN ULONG Color)
 482 {
 483     PFN_NUMBER PageIndex;
 484     PMMPFN Pfn1;
 485
 486     /* Make sure PFN lock is held and we have pages */
 487     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 488     ASSERT(MmAvailablePages != 0);
 489     ASSERT(Color < MmSecondaryColors);
 490
 491     /* Check the colored free list */
 492     PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 493     if (PageIndex == LIST_HEAD)
 494     {
 495         /* Check the colored zero list */
 496         PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 497         if (PageIndex == LIST_HEAD)
 498         {
 499             /* Check the free list */
 500             ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 501             PageIndex = MmFreePageListHead.Flink;
 502             Color = PageIndex & MmSecondaryColorMask;
 503             if (PageIndex == LIST_HEAD)
 504             {
 505                 /* Check the zero list */
 506                 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 507                 PageIndex = MmZeroedPageListHead.Flink;
 508                 Color = PageIndex & MmSecondaryColorMask;
 509                 ASSERT(PageIndex != LIST_HEAD);
 510                 if (PageIndex == LIST_HEAD)
 511                 {
 512                     /* FIXME: Should check the standby list */
 513                     ASSERT(MmZeroedPageListHead.Total == 0);
 514                 }
 515             }
 516         }
 517     }
 518
 519     /* Remove the page from its list */
 520     PageIndex = MiRemovePageByColor(PageIndex, Color);
 521
 522     /* Sanity checks */
 523     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 524     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 525            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 526     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 527     ASSERT(Pfn1->u2.ShareCount == 0);
 528     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 529     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 530
 531     /* Return the page */
 532     return PageIndex;
 533 }
 534
 535 PFN_NUMBER
 536 NTAPI
 537 MiRemoveZeroPage(IN ULONG Color)
 538 {
 539     PFN_NUMBER PageIndex;
 540     PMMPFN Pfn1;
 541     BOOLEAN Zero = FALSE;
 542
 543     /* Make sure PFN lock is held and we have pages */
 544     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 545     ASSERT(MmAvailablePages != 0);
 546     ASSERT(Color < MmSecondaryColors);
 547
 548     /* Check the colored zero list */
 549     PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 550     if (PageIndex == LIST_HEAD)
 551     {
 552         /* Check the zero list */
 553         ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 554         PageIndex = MmZeroedPageListHead.Flink;
 555         if (PageIndex == LIST_HEAD)
 556         {
 557             /* This means there's no zero pages, we have to look for free ones */
 558             ASSERT(MmZeroedPageListHead.Total == 0);
 559             Zero = TRUE;
 560
 561             /* Check the colored free list */
 562             PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 563             if (PageIndex == LIST_HEAD)
 564             {
 565                 /* Check the free list */
 566                 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 567                 PageIndex = MmFreePageListHead.Flink;
 568                 Color = PageIndex & MmSecondaryColorMask;
 569                 ASSERT(PageIndex != LIST_HEAD);
 570                 if (PageIndex == LIST_HEAD)
 571                 {
 572                     /* FIXME: Should check the standby list */
 573                     ASSERT(MmZeroedPageListHead.Total == 0);
 574                 }
 575             }
 576         }
 577         else
 578         {
 579             Color = PageIndex & MmSecondaryColorMask;
 580         }
 581     }
 582
 583     /* Sanity checks */
 584     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 585     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 586            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 587
 588     /* Remove the page from its list */
 589     PageIndex = MiRemovePageByColor(PageIndex, Color);
 590     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageIndex));
 591
 592     /* Zero it, if needed */
 593     if (Zero) MiZeroPhysicalPage(PageIndex);
 594
 595     /* Sanity checks */
 596     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 597     ASSERT(Pfn1->u2.ShareCount == 0);
 598     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 599     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 600
 601     /* Return the page */
 602     return PageIndex;
 603 }
 604
 605 VOID
 606 NTAPI
 607 MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
 608 {
 609     PMMPFNLIST ListHead;
 610     PFN_NUMBER LastPage;
 611     PMMPFN Pfn1;
 612     ULONG Color;
 613     PMMPFN Blink;
 614     PMMCOLOR_TABLES ColorTable;
 615
 616     /* Make sure the page index is valid */
 617     ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
 618     ASSERT((PageFrameIndex != 0) &&
 619            (PageFrameIndex <= MmHighestPhysicalPage) &&
 620            (PageFrameIndex >= MmLowestPhysicalPage));
 621
 622     /* Get the PFN entry */
 623     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 624
 625     /* Sanity checks that a right kind of page is being inserted here */
 626     ASSERT(Pfn1->u4.MustBeCached == 0);
 627     ASSERT(Pfn1->u3.e1.Rom != 1);
 628     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 629     ASSERT(Pfn1->u4.VerifierAllocation == 0);
 630     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 631
 632     /* Get the free page list and increment its count */
 633     ListHead = &MmFreePageListHead;
 634     ASSERT_LIST_INVARIANT(ListHead);
 635     ListHead->Total++;
 636
 637     /* Get the last page on the list */
 638     LastPage = ListHead->Blink;
 639     if (LastPage != LIST_HEAD)
 640     {
 641         /* Link us with the previous page, so we're at the end now */
 642         MI_PFN_ELEMENT(LastPage)->u1.Flink = PageFrameIndex;
 643     }
 644     else
 645     {
 646         /* The list is empty, so we are the first page */
 647         ListHead->Flink = PageFrameIndex;
 648     }
 649
 650     /* Now make the list head point back to us (since we go at the end) */
 651     ListHead->Blink = PageFrameIndex;
 652     ASSERT_LIST_INVARIANT(ListHead);
 653
 654     /* And initialize our own list pointers */
 655     Pfn1->u1.Flink = LIST_HEAD;
 656     Pfn1->u2.Blink = LastPage;
 657
 658     /* Set the list name and default priority */
 659     Pfn1->u3.e1.PageLocation = FreePageList;
 660     Pfn1->u4.Priority = 3;
 661
 662     /* Clear some status fields */
 663     Pfn1->u4.InPageError = 0;
 664     Pfn1->u4.AweAllocation = 0;
 665
 666     /* Increase available pages */
 667     MmAvailablePages++;
 668
 669     /* Check if we've reached the configured low memory threshold */
 670     if (MmAvailablePages == MmLowMemoryThreshold)
 671     {
 672         /* Clear the event, because now we're ABOVE the threshold */
 673         KeClearEvent(MiLowMemoryEvent);
 674     }
 675     else if (MmAvailablePages == MmHighMemoryThreshold)
 676     {
 677         /* Otherwise check if we reached the high threshold and signal the event */
 678         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 679     }
 680
 681     /* Get the page color */
 682     Color = PageFrameIndex & MmSecondaryColorMask;
 683
 684     /* Get the first page on the color list */
 685     ColorTable = &MmFreePagesByColor[FreePageList][Color];
 686     if (ColorTable->Flink == LIST_HEAD)
 687     {
 688         /* The list is empty, so we are the first page */
 689         Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 690         ColorTable->Flink = PageFrameIndex;
 691     }
 692     else
 693     {
 694         /* Get the previous page */
 695         Blink = (PMMPFN)ColorTable->Blink;
 696
 697         /* Make it link to us, and link back to it */
 698         Blink->OriginalPte.u.Long = PageFrameIndex;
 699         Pfn1->u4.PteFrame = MiGetPfnEntryIndex(Blink);
 700     }
 701
 702     /* Now initialize our own list pointers */
 703     ColorTable->Blink = Pfn1;
 704
 705     /* This page is now the last */
 706     Pfn1->OriginalPte.u.Long = LIST_HEAD;
 707
 708     /* And increase the count in the colored list */
 709     ColorTable->Count++;
 710
 711     /* Notify zero page thread if enough pages are on the free list now */
 712     if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive))
 713     {
 714         /* Set the event */
 715         MmZeroingPageThreadActive = TRUE;
 716         KeSetEvent(&MmZeroingPageEvent, IO_NO_INCREMENT, FALSE);
 717     }
 718
 719 #if MI_TRACE_PFNS
 720     Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 721     RtlZeroMemory(Pfn1->ProcessName, 16);
 722 #endif
 723 }
 724
 725 VOID
 726 FASTCALL
 727 MiInsertStandbyListAtFront(IN PFN_NUMBER PageFrameIndex)
 728 {
 729     PMMPFNLIST ListHead;
 730     PFN_NUMBER Flink;
 731     PMMPFN Pfn1, Pfn2;
 732
 733     /* Make sure the lock is held */
 734     DPRINT1("Inserting page: %lx into standby list !\n", PageFrameIndex);
 735     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 736
 737     /* Make sure the PFN is valid */
 738     ASSERT((PageFrameIndex != 0) &&
 739            (PageFrameIndex <= MmHighestPhysicalPage) &&
 740            (PageFrameIndex >= MmLowestPhysicalPage));
 741
 742     /* Grab the PFN and validate it is the right kind of PFN being inserted */
 743     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 744     ASSERT(Pfn1->u4.MustBeCached == 0);
 745     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 746     ASSERT(Pfn1->u3.e1.PrototypePte == 1);
 747     ASSERT(Pfn1->u3.e1.Rom != 1);
 748
 749     /* One more transition page on a list */
 750     MmTransitionSharedPages++;
 751
 752     /* Get the standby page list and increment its count */
 753     ListHead = &MmStandbyPageListByPriority [Pfn1->u4.Priority];
 754     ASSERT_LIST_INVARIANT(ListHead);
 755     ListHead->Total++;
 756
 757     /* Make the head of the list point to this page now */
 758     Flink = ListHead->Flink;
 759     ListHead->Flink = PageFrameIndex;
 760
 761     /* Make the page point to the previous head, and back to the list */
 762     Pfn1->u1.Flink = Flink;
 763     Pfn1->u2.Blink = LIST_HEAD;
 764
 765     /* Was the list empty? */
 766     if (Flink != LIST_HEAD)
 767     {
 768         /* It wasn't, so update the backlink of the previous head page */
 769         Pfn2 = MI_PFN_ELEMENT(Flink);
 770         Pfn2->u2.Blink = PageFrameIndex;
 771     }
 772     else
 773     {
 774         /* It was empty, so have it loop back around to this new page */
 775         ListHead->Blink = PageFrameIndex;
 776     }
 777
 778     /* Move the page onto its new location */
 779     Pfn1->u3.e1.PageLocation = StandbyPageList;
 780
 781     /* One more page on the system */
 782     MmAvailablePages++;
 783
 784     /* Check if we've reached the configured low memory threshold */
 785     if (MmAvailablePages == MmLowMemoryThreshold)
 786     {
 787         /* Clear the event, because now we're ABOVE the threshold */
 788         KeClearEvent(MiLowMemoryEvent);
 789     }
 790     else if (MmAvailablePages == MmHighMemoryThreshold)
 791     {
 792         /* Otherwise check if we reached the high threshold and signal the event */
 793         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 794     }
 795 }
 796
 797 VOID
 798 NTAPI
 799 MiInsertPageInList(IN PMMPFNLIST ListHead,
 800                    IN PFN_NUMBER PageFrameIndex)
 801 {
 802     PFN_NUMBER Flink, LastPage;
 803     PMMPFN Pfn1, Pfn2;
 804     MMLISTS ListName;
 805     PMMCOLOR_TABLES ColorHead;
 806     ULONG Color;
 807
 808     /* For free pages, use MiInsertPageInFreeList */
 809     ASSERT(ListHead != &MmFreePageListHead);
 810
 811     /* Make sure the lock is held */
 812     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 813
 814     /* Make sure the PFN is valid */
 815     ASSERT((PageFrameIndex) &&
 816            (PageFrameIndex <= MmHighestPhysicalPage) &&
 817            (PageFrameIndex >= MmLowestPhysicalPage));
 818
 819     /* Page should be unused */
 820     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 821     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 822     ASSERT(Pfn1->u3.e1.Rom != 1);
 823
 824     /* Is a standby or modified page being inserted? */
 825     ListName = ListHead->ListName;
 826     if ((ListName == StandbyPageList) || (ListName == ModifiedPageList))
 827     {
 828         /* If the page is in transition, it must also be a prototype page */
 829         if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
 830             (Pfn1->OriginalPte.u.Soft.Transition == 1))
 831         {
 832             /* Crash the system on inconsistency */
 833             KeBugCheckEx(MEMORY_MANAGEMENT, 0x8888, 0, 0, 0);
 834         }
 835     }
 836
 837     /* Standby pages are prioritized, so we need to get the real head */
 838     if (ListHead == &MmStandbyPageListHead)
 839     {
 840         /* Obviously the prioritized list should still have the same name */
 841         ListHead = &MmStandbyPageListByPriority[Pfn1->u4.Priority];
 842         ASSERT(ListHead->ListName == ListName);
 843     }
 844
 845     /* Increment the list count */
 846     ListHead->Total++;
 847
 848     /* Is a modified page being inserted? */
 849     if (ListHead == &MmModifiedPageListHead)
 850     {
 851         /* For now, only single-prototype pages should end up in this path */
 852         DPRINT("Modified page being added: %lx\n", PageFrameIndex);
 853         ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 854
 855         /* Modified pages are colored when they are selected for page file */
 856         ListHead = &MmModifiedPageListByColor[0];
 857         ASSERT (ListHead->ListName == ListName);
 858         ListHead->Total++;
 859
 860         /* Increment the number of paging file modified pages */
 861         MmTotalPagesForPagingFile++;
 862     }
 863
 864     /* Don't handle bad pages yet yet */
 865     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 866
 867     /* Zero pages go to the head, all other pages go to the end */
 868     if (ListName == ZeroedPageList)
 869     {
 870         /* Make the head of the list point to this page now */
 871         Flink = ListHead->Flink;
 872         ListHead->Flink = PageFrameIndex;
 873
 874         /* Make the page point to the previous head, and back to the list */
 875         Pfn1->u1.Flink = Flink;
 876         Pfn1->u2.Blink = LIST_HEAD;
 877
 878         /* Was the list empty? */
 879         if (Flink != LIST_HEAD)
 880         {
 881             /* It wasn't, so update the backlink of the previous head page */
 882             Pfn2 = MI_PFN_ELEMENT(Flink);
 883             Pfn2->u2.Blink = PageFrameIndex;
 884         }
 885         else
 886         {
 887             /* It was empty, so have it loop back around to this new page */
 888             ListHead->Blink = PageFrameIndex;
 889         }
 890     }
 891     else
 892     {
 893         /* Get the last page on the list */
 894         LastPage = ListHead->Blink;
 895         if (LastPage != LIST_HEAD)
 896         {
 897             /* Link us with the previous page, so we're at the end now */
 898             MI_PFN_ELEMENT(LastPage)->u1.Flink = PageFrameIndex;
 899         }
 900         else
 901         {
 902             /* The list is empty, so we are the first page */
 903             ListHead->Flink = PageFrameIndex;
 904         }
 905
 906         /* Now make the list head point back to us (since we go at the end) */
 907         ListHead->Blink = PageFrameIndex;
 908         ASSERT_LIST_INVARIANT(ListHead);
 909
 910         /* And initialize our own list pointers */
 911         Pfn1->u1.Flink = LIST_HEAD;
 912         Pfn1->u2.Blink = LastPage;
 913     }
 914
 915     /* Move the page onto its new location */
 916     Pfn1->u3.e1.PageLocation = ListName;
 917
 918     /* For zero/free pages, we also have to handle the colored lists */
 919     if (ListName <= StandbyPageList)
 920     {
 921         /* One more page on the system */
 922         MmAvailablePages++;
 923
 924         /* Check if we've reached the configured low memory threshold */
 925         if (MmAvailablePages == MmLowMemoryThreshold)
 926         {
 927             /* Clear the event, because now we're ABOVE the threshold */
 928             KeClearEvent(MiLowMemoryEvent);
 929         }
 930         else if (MmAvailablePages == MmHighMemoryThreshold)
 931         {
 932             /* Otherwise check if we reached the high threshold and signal the event */
 933             KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 934         }
 935
 936         /* Sanity checks */
 937         ASSERT(ListName == ZeroedPageList);
 938         ASSERT(Pfn1->u4.InPageError == 0);
 939
 940         /* Get the page color */
 941         Color = PageFrameIndex & MmSecondaryColorMask;
 942
 943         /* Get the list for this color */
 944         ColorHead = &MmFreePagesByColor[ZeroedPageList][Color];
 945
 946         /* Get the old head */
 947         Flink = ColorHead->Flink;
 948
 949         /* Make this page point back to the list, and point forwards to the old head */
 950         Pfn1->OriginalPte.u.Long = Flink;
 951         Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 952
 953         /* Set the new head */
 954         ColorHead->Flink = PageFrameIndex;
 955
 956         /* Was the head empty? */
 957         if (Flink != LIST_HEAD)
 958         {
 959             /* No, so make the old head point to this page */
 960             Pfn2 = MI_PFN_ELEMENT(Flink);
 961             Pfn2->u4.PteFrame = PageFrameIndex;
 962         }
 963         else
 964         {
 965             /* Yes, make it loop back to this page */
 966             ColorHead->Blink = (PVOID)Pfn1;
 967         }
 968
 969         /* One more paged on the colored list */
 970         ColorHead->Count++;
 971
 972 #if MI_TRACE_PFNS
 973             //ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET);
 974             Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 975             MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 976             RtlZeroMemory(Pfn1->ProcessName, 16);
 977 #endif
 978     }
 979     else if (ListName == ModifiedPageList)
 980     {
 981         /* In ARM3, page must be destined for page file, and not yet written out */
 982         ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 983         ASSERT(Pfn1->OriginalPte.u.Soft.PageFileHigh == 0);
 984
 985         /* One more transition page */
 986         MmTransitionSharedPages++;
 987
 988         /* Increment the number of per-process modified pages */
 989         PsGetCurrentProcess()->ModifiedPageCount++;
 990
 991         /* FIXME: Wake up modified page writer if there are not enough free pages */
 992     }
 993     else if (ListName == ModifiedNoWritePageList)
 994     {
 995         /* This list is not yet implemented */
 996         ASSERT(FALSE);
 997     }
 998 }
 999
1000 VOID
1001 NTAPI
1002 MiInitializePfn(IN PFN_NUMBER PageFrameIndex,
1003                 IN PMMPTE PointerPte,
1004                 IN BOOLEAN Modified)
1005 {
1006     PMMPFN Pfn1;
1007     NTSTATUS Status;
1008     PMMPTE PointerPtePte;
1009     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1010
1011     /* Setup the PTE */
1012     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1013     Pfn1->PteAddress = PointerPte;
1014
1015     /* Check if this PFN is part of a valid address space */
1016     if (PointerPte->u.Hard.Valid == 1)
1017     {
1018         /* Only valid from MmCreateProcessAddressSpace path */
1019         ASSERT(PsGetCurrentProcess()->Vm.WorkingSetSize == 0);
1020
1021         /* Make this a demand zero PTE */
1022         MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
1023     }
1024     else
1025     {
1026         /* Copy the PTE data */
1027         Pfn1->OriginalPte = *PointerPte;
1028         ASSERT(!((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
1029                  (Pfn1->OriginalPte.u.Soft.Transition == 1)));
1030     }
1031
1032     /* Otherwise this is a fresh page -- set it up */
1033     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
1034     Pfn1->u3.e2.ReferenceCount = 1;
1035     Pfn1->u2.ShareCount = 1;
1036     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1037     ASSERT(Pfn1->u3.e1.Rom == 0);
1038     Pfn1->u3.e1.Modified = Modified;
1039
1040     /* Get the page table for the PTE */
1041     PointerPtePte = MiAddressToPte(PointerPte);
1042     if (PointerPtePte->u.Hard.Valid == 0)
1043     {
1044         /* Make sure the PDE gets paged in properly */
1045         Status = MiCheckPdeForPagedPool(PointerPte);
1046         if (!NT_SUCCESS(Status))
1047         {
1048             /* Crash */
1049             KeBugCheckEx(MEMORY_MANAGEMENT,
1050                          0x61940,
1051                          (ULONG_PTR)PointerPte,
1052                          (ULONG_PTR)PointerPtePte->u.Long,
1053                          (ULONG_PTR)MiPteToAddress(PointerPte));
1054         }
1055     }
1056
1057     /* Get the PFN for the page table */
1058     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
1059     ASSERT(PageFrameIndex != 0);
1060     Pfn1->u4.PteFrame = PageFrameIndex;
1061
1062     /* Increase its share count so we don't get rid of it */
1063     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1064     Pfn1->u2.ShareCount++;
1065 }
1066
1067 VOID
1068 NTAPI
1069 MiInitializePfnAndMakePteValid(IN PFN_NUMBER PageFrameIndex,
1070                                IN PMMPTE PointerPte,
1071                                IN MMPTE TempPte)
1072 {
1073     PMMPFN Pfn1;
1074     NTSTATUS Status;
1075     PMMPTE PointerPtePte;
1076     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1077
1078     /* PTE must be invalid */
1079     ASSERT(PointerPte->u.Hard.Valid == 0);
1080
1081     /* Setup the PTE */
1082     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1083     Pfn1->PteAddress = PointerPte;
1084     Pfn1->OriginalPte = DemandZeroPte;
1085
1086     /* Otherwise this is a fresh page -- set it up */
1087     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
1088     Pfn1->u3.e2.ReferenceCount++;
1089     Pfn1->u2.ShareCount++;
1090     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1091     ASSERT(Pfn1->u3.e1.Rom == 0);
1092     Pfn1->u3.e1.Modified = 1;
1093
1094     /* Get the page table for the PTE */
1095     PointerPtePte = MiAddressToPte(PointerPte);
1096     if (PointerPtePte->u.Hard.Valid == 0)
1097     {
1098         /* Make sure the PDE gets paged in properly */
1099         Status = MiCheckPdeForPagedPool(PointerPte);
1100         if (!NT_SUCCESS(Status))
1101         {
1102             /* Crash */
1103             KeBugCheckEx(MEMORY_MANAGEMENT,
1104                          0x61940,
1105                          (ULONG_PTR)PointerPte,
1106                          (ULONG_PTR)PointerPtePte->u.Long,
1107                          (ULONG_PTR)MiPteToAddress(PointerPte));
1108         }
1109     }
1110
1111     /* Get the PFN for the page table */
1112     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
1113     ASSERT(PageFrameIndex != 0);
1114     Pfn1->u4.PteFrame = PageFrameIndex;
1115
1116     /* Increase its share count so we don't get rid of it */
1117     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1118     Pfn1->u2.ShareCount++;
1119
1120     /* Write valid PTE */
1121     MI_WRITE_VALID_PTE(PointerPte, TempPte);
1122 }
1123
1124 NTSTATUS
1125 NTAPI
1126 MiInitializeAndChargePfn(OUT PPFN_NUMBER PageFrameIndex,
1127                          IN PMMPTE PointerPde,
1128                          IN PFN_NUMBER ContainingPageFrame,
1129                          IN BOOLEAN SessionAllocation)
1130 {
1131     MMPTE TempPte;
1132     KIRQL OldIrql;
1133
1134     /* Use either a global or local PDE */
1135     TempPte = SessionAllocation ? ValidKernelPdeLocal : ValidKernelPde;
1136
1137     /* Lock the PFN database */
1138     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1139
1140     /* Make sure nobody is racing us */
1141     if (PointerPde->u.Hard.Valid == 1)
1142     {
1143         /* Return special error if that was the case */
1144         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1145         return STATUS_RETRY;
1146     }
1147
1148     /* Grab a zero page and set the PFN, then make it valid */
1149     *PageFrameIndex = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
1150     TempPte.u.Hard.PageFrameNumber = *PageFrameIndex;
1151     MI_WRITE_VALID_PTE(PointerPde, TempPte);
1152
1153     /* Initialize the PFN */
1154     MiInitializePfnForOtherProcess(*PageFrameIndex,
1155                                    PointerPde,
1156                                    ContainingPageFrame);
1157     ASSERT(MI_PFN_ELEMENT(*PageFrameIndex)->u1.WsIndex == 0);
1158
1159     /* Release the lock and return success */
1160     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1161     return STATUS_SUCCESS;
1162 }
1163
1164 PFN_NUMBER
1165 NTAPI
1166 MiAllocatePfn(IN PMMPTE PointerPte,
1167               IN ULONG Protection)
1168 {
1169     KIRQL OldIrql;
1170     PFN_NUMBER PageFrameIndex;
1171     MMPTE TempPte;
1172
1173     /* Sanity check that we aren't passed a valid PTE */
1174     ASSERT(PointerPte->u.Hard.Valid == 0);
1175
1176     /* Make an empty software PTE */
1177     MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
1178
1179     /* Lock the PFN database */
1180     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1181
1182     /* Check if we're running low on pages */
1183     if (MmAvailablePages < 128)
1184     {
1185         DPRINT1("Warning, running low on memory: %lu pages left\n", MmAvailablePages);
1186
1187         //MiEnsureAvailablePageOrWait(NULL, OldIrql);
1188
1189         /* Call RosMm and see if it can release any pages for us */
1190         MmRebalanceMemoryConsumers();
1191     }
1192
1193     /* Grab a page */
1194     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
1195     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
1196     PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
1197
1198     /* Write the software PTE */
1199     MI_WRITE_INVALID_PTE(PointerPte, TempPte);
1200     PointerPte->u.Soft.Protection |= Protection;
1201
1202     /* Initialize its PFN entry */
1203     MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
1204
1205     /* Release the PFN lock and return the page */
1206     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
1207     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
1208     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1209     return PageFrameIndex;
1210 }
1211
1212 VOID
1213 NTAPI
1214 MiDecrementShareCount(IN PMMPFN Pfn1,
1215                       IN PFN_NUMBER PageFrameIndex)
1216 {
1217     PMMPTE PointerPte;
1218     MMPTE TempPte;
1219
1220     ASSERT(PageFrameIndex > 0);
1221     ASSERT(MI_PFN_ELEMENT(PageFrameIndex) != NULL);
1222     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
1223     ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
1224
1225     /* Page must be in-use */
1226     if ((Pfn1->u3.e1.PageLocation != ActiveAndValid) &&
1227         (Pfn1->u3.e1.PageLocation != StandbyPageList))
1228     {
1229         /* Otherwise we have PFN corruption */
1230         KeBugCheckEx(PFN_LIST_CORRUPT,
1231                      0x99,
1232                      PageFrameIndex,
1233                      Pfn1->u3.e1.PageLocation,
1234                      0);
1235     }
1236
1237     /* Check if the share count is now 0 */
1238     ASSERT(Pfn1->u2.ShareCount < 0xF000000);
1239     if (!--Pfn1->u2.ShareCount)
1240     {
1241         /* Was this a prototype PTE? */
1242         if (Pfn1->u3.e1.PrototypePte)
1243         {
1244             /* Grab the PTE address and make sure it's in prototype pool */
1245             PointerPte = Pfn1->PteAddress;
1246             ASSERT((PointerPte >= (PMMPTE)MmPagedPoolStart) && (PointerPte <= (PMMPTE)MmPagedPoolEnd));
1247
1248             /* The PTE that backs it should also be valdi */
1249             PointerPte = MiAddressToPte(PointerPte);
1250             ASSERT(PointerPte->u.Hard.Valid == 1);
1251
1252             /* Get the original prototype PTE and turn it into a transition PTE */
1253             PointerPte = Pfn1->PteAddress;
1254             TempPte = *PointerPte;
1255             TempPte.u.Soft.Transition = 1;
1256             TempPte.u.Soft.Valid = 0;
1257             TempPte.u.Soft.Prototype = 0;
1258             TempPte.u.Soft.Protection = Pfn1->OriginalPte.u.Soft.Protection;
1259             MI_WRITE_INVALID_PTE(PointerPte, TempPte);
1260             DPRINT("Marking PTE: %p as transition (%p - %lx)\n", PointerPte, Pfn1, MiGetPfnEntryIndex(Pfn1));
1261         }
1262
1263         /* Put the page in transition */
1264         Pfn1->u3.e1.PageLocation = TransitionPage;
1265
1266         /* PFN lock must be held */
1267         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1268
1269         /* Page should at least have one reference */
1270         ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1271         if (Pfn1->u3.e2.ReferenceCount == 1)
1272         {
1273             /* Is there still a PFN for this page? */
1274             if (MI_IS_PFN_DELETED(Pfn1) == TRUE)
1275             {
1276                 /* Clear the last reference */
1277                 Pfn1->u3.e2.ReferenceCount = 0;
1278                 ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
1279
1280                 /* Mark the page temporarily as valid, we're going to make it free soon */
1281                 Pfn1->u3.e1.PageLocation = ActiveAndValid;
1282
1283                 /* Bring it back into the free list */
1284                 MiInsertPageInFreeList(PageFrameIndex);
1285             }
1286             else
1287             {
1288                 /* PFN not yet deleted, drop a ref count */
1289                 MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1290             }
1291         }
1292         else
1293         {
1294             /* Otherwise, just drop the reference count */
1295             InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1296         }
1297     }
1298 }
1299
1300 VOID
1301 NTAPI
1302 MiDecrementReferenceCount(IN PMMPFN Pfn1,
1303                           IN PFN_NUMBER PageFrameIndex)
1304 {
1305     /* PFN lock must be held */
1306     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1307
1308     /* Sanity checks on the page */
1309     if (PageFrameIndex > MmHighestPhysicalPage ||
1310         Pfn1 != MI_PFN_ELEMENT(PageFrameIndex) ||
1311         Pfn1->u3.e2.ReferenceCount == 0 ||
1312         Pfn1->u3.e2.ReferenceCount >= 2500)
1313     {
1314         DPRINT1("PageFrameIndex=0x%lx, MmHighestPhysicalPage=0x%lx\n", PageFrameIndex, MmHighestPhysicalPage);
1315         DPRINT1("Pfn1=%p, Element=%p, RefCount=%u\n", Pfn1, MI_PFN_ELEMENT(PageFrameIndex), Pfn1->u3.e2.ReferenceCount);
1316         ASSERT(PageFrameIndex <= MmHighestPhysicalPage);
1317         ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
1318         ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1319         ASSERT(Pfn1->u3.e2.ReferenceCount < 2500);
1320     }
1321
1322     /* Dereference the page, bail out if it's still alive */
1323     InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1324     if (Pfn1->u3.e2.ReferenceCount) return;
1325
1326     /* Nobody should still have reference to this page */
1327     if (Pfn1->u2.ShareCount != 0)
1328     {
1329         /* Otherwise something's really wrong */
1330         KeBugCheckEx(PFN_LIST_CORRUPT, 7, PageFrameIndex, Pfn1->u2.ShareCount, 0);
1331     }
1332
1333     /* And it should be lying on some page list */
1334     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1335
1336     /* Did someone set the delete flag? */
1337     if (MI_IS_PFN_DELETED(Pfn1))
1338     {
1339         /* Insert it into the free list, there's nothing left to do */
1340         MiInsertPageInFreeList(PageFrameIndex);
1341         return;
1342     }
1343
1344     /* Check to see which list this page should go into */
1345     if (Pfn1->u3.e1.Modified == 1)
1346     {
1347         /* Push it into the modified page list */
1348         MiInsertPageInList(&MmModifiedPageListHead, PageFrameIndex);
1349     }
1350     else
1351     {
1352         /* Otherwise, insert this page into the standby list */
1353         ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
1354         MiInsertStandbyListAtFront(PageFrameIndex);
1355     }
1356 }
1357
1358 VOID
1359 NTAPI
1360 MiInitializePfnForOtherProcess(IN PFN_NUMBER PageFrameIndex,
1361                                IN PMMPTE PointerPte,
1362                                IN PFN_NUMBER PteFrame)
1363 {
1364     PMMPFN Pfn1;
1365
1366     /* Setup the PTE */
1367     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1368     Pfn1->PteAddress = PointerPte;
1369
1370     /* Make this a software PTE */
1371     MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
1372
1373     /* Setup the page */
1374     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
1375     Pfn1->u3.e2.ReferenceCount = 1;
1376     Pfn1->u2.ShareCount = 1;
1377     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1378     Pfn1->u3.e1.Modified = TRUE;
1379     Pfn1->u4.InPageError = FALSE;
1380
1381     /* Did we get a PFN for the page table */
1382     if (PteFrame)
1383     {
1384         /* Store it */
1385         Pfn1->u4.PteFrame = PteFrame;
1386
1387         /* Increase its share count so we don't get rid of it */
1388         Pfn1 = MI_PFN_ELEMENT(PteFrame);
1389         Pfn1->u2.ShareCount++;
1390     }
1391 }
1392
1393 /* EOF */