reactos/ntoskrnl/mm/ARM3/pfnlist.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         BSD - See COPYING.ARM in the top level directory
   4  * FILE:            ntoskrnl/mm/ARM3/pfnlist.c
   5  * PURPOSE:         ARM Memory Manager PFN List Manipulation
   6  * PROGRAMMERS:     ReactOS Portable Systems Group
   7  */
   8
   9 /* INCLUDES *******************************************************************/
  10
  11 #include <ntoskrnl.h>
  12 #define NDEBUG
  13 #include <debug.h>
  14
  15 #define MODULE_INVOLVED_IN_ARM3
  16 #include "../ARM3/miarm.h"
  17
  18 #if DBG
  19 #define ASSERT_LIST_INVARIANT(x) \
  20 do { \
  21     ASSERT(((x)->Total == 0 && \
  22             (x)->Flink == LIST_HEAD && \
  23             (x)->Blink == LIST_HEAD) || \
  24            ((x)->Total != 0 && \
  25             (x)->Flink != LIST_HEAD && \
  26             (x)->Blink != LIST_HEAD)); \
  27 } while (0)
  28 #else
  29 #define ASSERT_LIST_INVARIANT(x)
  30 #endif
  31
  32 /* GLOBALS ********************************************************************/
  33
  34 BOOLEAN MmDynamicPfn;
  35 BOOLEAN MmMirroring;
  36 ULONG MmSystemPageColor;
  37
  38 ULONG MmTransitionSharedPages;
  39 ULONG MmTotalPagesForPagingFile;
  40
  41 MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD};
  42 MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD};
  43 MMPFNLIST MmStandbyPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  44 MMPFNLIST MmStandbyPageListByPriority[8];
  45 MMPFNLIST MmModifiedPageListHead = {0, ModifiedPageList, LIST_HEAD, LIST_HEAD};
  46 MMPFNLIST MmModifiedPageListByColor[1] = {{0, ModifiedPageList, LIST_HEAD, LIST_HEAD}};
  47 MMPFNLIST MmModifiedNoWritePageListHead = {0, ModifiedNoWritePageList, LIST_HEAD, LIST_HEAD};
  48 MMPFNLIST MmBadPageListHead = {0, BadPageList, LIST_HEAD, LIST_HEAD};
  49 MMPFNLIST MmRomPageListHead = {0, StandbyPageList, LIST_HEAD, LIST_HEAD};
  50
  51 PMMPFNLIST MmPageLocationList[] =
  52 {
  53     &MmZeroedPageListHead,
  54     &MmFreePageListHead,
  55     &MmStandbyPageListHead,
  56     &MmModifiedPageListHead,
  57     &MmModifiedNoWritePageListHead,
  58     &MmBadPageListHead,
  59     NULL,
  60     NULL
  61 };
  62
  63 ULONG MI_PFN_CURRENT_USAGE;
  64 CHAR MI_PFN_CURRENT_PROCESS_NAME[16] = "None yet";
  65
  66 /* FUNCTIONS ******************************************************************/
  67
  68 VOID
  69 NTAPI
  70 MiZeroPhysicalPage(IN PFN_NUMBER PageFrameIndex)
  71 {
  72     KIRQL OldIrql;
  73     PVOID VirtualAddress;
  74     PEPROCESS Process = PsGetCurrentProcess();
  75
  76     /* Map in hyperspace, then wipe it using XMMI or MEMSET */
  77     VirtualAddress = MiMapPageInHyperSpace(Process, PageFrameIndex, &OldIrql);
  78     ASSERT(VirtualAddress);
  79     KeZeroPages(VirtualAddress, PAGE_SIZE);
  80     MiUnmapPageInHyperSpace(Process, VirtualAddress, OldIrql);
  81 }
  82
  83 VOID
  84 NTAPI
  85 MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry)
  86 {
  87     PFN_NUMBER OldFlink, OldBlink;
  88     PMMPFNLIST ListHead;
  89     MMLISTS ListName;
  90     ULONG Color;
  91     PMMCOLOR_TABLES ColorTable;
  92     PMMPFN Pfn1;
  93
  94     /* Make sure the PFN lock is held */
  95     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
  96
  97     /* Make sure the PFN entry isn't in-use */
  98     ASSERT(Entry->u3.e1.WriteInProgress == 0);
  99     ASSERT(Entry->u3.e1.ReadInProgress == 0);
 100
 101     /* Find the list for this entry, make sure it's the free or zero list */
 102     ListHead = MmPageLocationList[Entry->u3.e1.PageLocation];
 103     ListName = ListHead->ListName;
 104     ASSERT(ListHead != NULL);
 105     ASSERT(ListName <= FreePageList);
 106     ASSERT_LIST_INVARIANT(ListHead);
 107
 108     /* Remove one count */
 109     ASSERT(ListHead->Total != 0);
 110     ListHead->Total--;
 111
 112     /* Get the forward and back pointers */
 113     OldFlink = Entry->u1.Flink;
 114     OldBlink = Entry->u2.Blink;
 115
 116     /* Check if the next entry is the list head */
 117     if (OldFlink != LIST_HEAD)
 118     {
 119         /* It is not, so set the backlink of the actual entry, to our backlink */
 120         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 121     }
 122     else
 123     {
 124         /* Set the list head's backlink instead */
 125         ListHead->Blink = OldBlink;
 126     }
 127
 128     /* Check if the back entry is the list head */
 129     if (OldBlink != LIST_HEAD)
 130     {
 131         /* It is not, so set the backlink of the actual entry, to our backlink */
 132         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 133     }
 134     else
 135     {
 136         /* Set the list head's backlink instead */
 137         ListHead->Flink = OldFlink;
 138     }
 139
 140     /* Get the page color */
 141     OldBlink = MiGetPfnEntryIndex(Entry);
 142     Color = OldBlink & MmSecondaryColorMask;
 143
 144     /* Get the first page on the color list */
 145     ColorTable = &MmFreePagesByColor[ListName][Color];
 146
 147     /* Check if this was was actually the head */
 148     OldFlink = ColorTable->Flink;
 149     if (OldFlink == OldBlink)
 150     {
 151         /* Make the table point to the next page this page was linking to */
 152         ColorTable->Flink = Entry->OriginalPte.u.Long;
 153         if (ColorTable->Flink != LIST_HEAD)
 154         {
 155             /* And make the previous link point to the head now */
 156             MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 157         }
 158         else
 159         {
 160             /* And if that page was the head, loop the list back around */
 161             ColorTable->Blink = (PVOID)LIST_HEAD;
 162         }
 163     }
 164     else
 165     {
 166         /* This page shouldn't be pointing back to the head */
 167         ASSERT(Entry->u4.PteFrame != COLORED_LIST_HEAD);
 168
 169         /* Make the back link point to whoever the next page is */
 170         Pfn1 = MI_PFN_ELEMENT(Entry->u4.PteFrame);
 171         Pfn1->OriginalPte.u.Long = Entry->OriginalPte.u.Long;
 172
 173         /* Check if this page was pointing to the head */
 174         if (Entry->OriginalPte.u.Long != LIST_HEAD)
 175         {
 176             /* Make the back link point to the head */
 177             Pfn1 = MI_PFN_ELEMENT(Entry->OriginalPte.u.Long);
 178             Pfn1->u4.PteFrame = Entry->u4.PteFrame;
 179         }
 180         else
 181         {
 182             /* Then the table is directly back pointing to this page now */
 183             ColorTable->Blink = Pfn1;
 184         }
 185     }
 186
 187     /* One less colored page */
 188     ASSERT(ColorTable->Count >= 1);
 189     ColorTable->Count--;
 190
 191     /* ReactOS Hack */
 192     Entry->OriginalPte.u.Long = 0;
 193
 194     /* We are not on a list anymore */
 195     Entry->u1.Flink = Entry->u2.Blink = 0;
 196     ASSERT_LIST_INVARIANT(ListHead);
 197
 198     /* See if we hit any thresholds */
 199     if (MmAvailablePages == MmHighMemoryThreshold)
 200     {
 201         /* Clear the high memory event */
 202         KeClearEvent(MiHighMemoryEvent);
 203     }
 204     else if (MmAvailablePages == MmLowMemoryThreshold)
 205     {
 206         /* Signal the low memory event */
 207         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 208     }
 209
 210     /* One less page */
 211     if (--MmAvailablePages < MmMinimumFreePages)
 212     {
 213         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 214
 215         DPRINT1("Running low on pages: %lu remaining\n", MmAvailablePages);
 216
 217         /* Call RosMm and see if it can release any pages for us */
 218         MmRebalanceMemoryConsumers();
 219     }
 220
 221 #if MI_TRACE_PFNS
 222     ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 223     Entry->PfnUsage = MI_PFN_CURRENT_USAGE;
 224     memcpy(Entry->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 225 //    MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 226 //    memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 227 #endif
 228 }
 229
 230 VOID
 231 NTAPI
 232 MiUnlinkPageFromList(IN PMMPFN Pfn)
 233 {
 234     PMMPFNLIST ListHead;
 235     PFN_NUMBER OldFlink, OldBlink;
 236
 237     /* Make sure the PFN lock is held */
 238     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 239
 240     /* ARM3 should only call this for dead pages */
 241     ASSERT(Pfn->u3.e2.ReferenceCount == 0);
 242
 243     /* Transition pages are supposed to be standby/modified/nowrite */
 244     ListHead = MmPageLocationList[Pfn->u3.e1.PageLocation];
 245     ASSERT(ListHead->ListName >= StandbyPageList);
 246
 247     /* Check if this was standby, or modified */
 248     if (ListHead == &MmStandbyPageListHead)
 249     {
 250         /* Should not be a ROM page */
 251         ASSERT(Pfn->u3.e1.Rom == 0);
 252
 253         /* Get the exact list */
 254         ListHead = &MmStandbyPageListByPriority[Pfn->u4.Priority];
 255
 256         /* See if we hit any thresholds */
 257         if (MmAvailablePages == MmHighMemoryThreshold)
 258         {
 259             /* Clear the high memory event */
 260             KeClearEvent(MiHighMemoryEvent);
 261         }
 262         else if (MmAvailablePages == MmLowMemoryThreshold)
 263         {
 264             /* Signal the low memory event */
 265             KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 266         }
 267
 268         /* Decrease transition page counter */
 269         ASSERT(Pfn->u3.e1.PrototypePte == 1); /* Only supported ARM3 case */
 270         MmTransitionSharedPages--;
 271
 272         /* One less page */
 273         if (--MmAvailablePages < MmMinimumFreePages)
 274         {
 275             /* FIXME: Should wake up the MPW and working set manager, if we had one */
 276             DPRINT1("Running low on pages: %lu remaining\n", MmAvailablePages);
 277
 278             /* Call RosMm and see if it can release any pages for us */
 279             MmRebalanceMemoryConsumers();
 280         }
 281     }
 282     else if (ListHead == &MmModifiedPageListHead)
 283     {
 284         /* Only shared memory (page-file backed) modified pages are supported */
 285         ASSERT(Pfn->OriginalPte.u.Soft.Prototype == 0);
 286
 287         /* Decrement the counters */
 288         ListHead->Total--;
 289         MmTotalPagesForPagingFile--;
 290
 291         /* Pick the correct colored list */
 292         ListHead = &MmModifiedPageListByColor[0];
 293
 294         /* Decrease transition page counter */
 295         ASSERT(Pfn->u3.e1.PrototypePte == 1); /* Only supported ARM3 case */
 296         MmTransitionSharedPages--;
 297     }
 298     else if (ListHead == &MmModifiedNoWritePageListHead)
 299     {
 300         /* List not yet supported */
 301         ASSERT(FALSE);
 302     }
 303
 304     /* Nothing should be in progress and the list should not be empty */
 305     ASSERT(Pfn->u3.e1.WriteInProgress == 0);
 306     ASSERT(Pfn->u3.e1.ReadInProgress == 0);
 307     ASSERT(ListHead->Total != 0);
 308
 309     /* Get the forward and back pointers */
 310     OldFlink = Pfn->u1.Flink;
 311     OldBlink = Pfn->u2.Blink;
 312
 313     /* Check if the next entry is the list head */
 314     if (OldFlink != LIST_HEAD)
 315     {
 316         /* It is not, so set the backlink of the actual entry, to our backlink */
 317         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 318     }
 319     else
 320     {
 321         /* Set the list head's backlink instead */
 322         ListHead->Blink = OldBlink;
 323     }
 324
 325     /* Check if the back entry is the list head */
 326     if (OldBlink != LIST_HEAD)
 327     {
 328         /* It is not, so set the backlink of the actual entry, to our backlink */
 329         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 330     }
 331     else
 332     {
 333         /* Set the list head's backlink instead */
 334         ListHead->Flink = OldFlink;
 335     }
 336
 337     /* ReactOS Hack */
 338     Pfn->OriginalPte.u.Long = 0;
 339
 340     /* We are not on a list anymore */
 341     Pfn->u1.Flink = Pfn->u2.Blink = 0;
 342     ASSERT_LIST_INVARIANT(ListHead);
 343
 344     /* Remove one entry from the list */
 345     ListHead->Total--;
 346 }
 347
 348 PFN_NUMBER
 349 NTAPI
 350 MiRemovePageByColor(IN PFN_NUMBER PageIndex,
 351                     IN ULONG Color)
 352 {
 353     PMMPFN Pfn1;
 354     PMMPFNLIST ListHead;
 355     MMLISTS ListName;
 356     PFN_NUMBER OldFlink, OldBlink;
 357     USHORT OldColor, OldCache;
 358     PMMCOLOR_TABLES ColorTable;
 359
 360     /* Make sure PFN lock is held */
 361     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 362     ASSERT(Color < MmSecondaryColors);
 363
 364     /* Get the PFN entry */
 365     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 366     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 367     ASSERT(Pfn1->u3.e1.Rom == 0);
 368
 369     /* Capture data for later */
 370     OldColor = Pfn1->u3.e1.PageColor;
 371     OldCache = Pfn1->u3.e1.CacheAttribute;
 372
 373     /* Could be either on free or zero list */
 374     ListHead = MmPageLocationList[Pfn1->u3.e1.PageLocation];
 375     ASSERT_LIST_INVARIANT(ListHead);
 376     ListName = ListHead->ListName;
 377     ASSERT(ListName <= FreePageList);
 378
 379     /* Remove a page */
 380     ListHead->Total--;
 381
 382     /* Get the forward and back pointers */
 383     OldFlink = Pfn1->u1.Flink;
 384     OldBlink = Pfn1->u2.Blink;
 385
 386     /* Check if the next entry is the list head */
 387     if (OldFlink != LIST_HEAD)
 388     {
 389         /* It is not, so set the backlink of the actual entry, to our backlink */
 390         MI_PFN_ELEMENT(OldFlink)->u2.Blink = OldBlink;
 391     }
 392     else
 393     {
 394         /* Set the list head's backlink instead */
 395         ListHead->Blink = OldBlink;
 396     }
 397
 398     /* Check if the back entry is the list head */
 399     if (OldBlink != LIST_HEAD)
 400     {
 401         /* It is not, so set the backlink of the actual entry, to our backlink */
 402         MI_PFN_ELEMENT(OldBlink)->u1.Flink = OldFlink;
 403     }
 404     else
 405     {
 406         /* Set the list head's backlink instead */
 407         ListHead->Flink = OldFlink;
 408     }
 409
 410     /* We are not on a list anymore */
 411     ASSERT_LIST_INVARIANT(ListHead);
 412     Pfn1->u1.Flink = Pfn1->u2.Blink = 0;
 413
 414     /* Zero flags but restore color and cache */
 415     Pfn1->u3.e2.ShortFlags = 0;
 416     Pfn1->u3.e1.PageColor = OldColor;
 417     Pfn1->u3.e1.CacheAttribute = OldCache;
 418
 419     /* Get the first page on the color list */
 420     ASSERT(Color < MmSecondaryColors);
 421     ColorTable = &MmFreePagesByColor[ListName][Color];
 422     ASSERT(ColorTable->Count >= 1);
 423
 424     /* Set the forward link to whoever we were pointing to */
 425     ColorTable->Flink = Pfn1->OriginalPte.u.Long;
 426
 427     /* Get the first page on the color list */
 428     if (ColorTable->Flink == LIST_HEAD)
 429     {
 430         /* This is the beginning of the list, so set the sentinel value */
 431         ColorTable->Blink = (PVOID)LIST_HEAD;
 432     }
 433     else
 434     {
 435         /* The list is empty, so we are the first page */
 436         MI_PFN_ELEMENT(ColorTable->Flink)->u4.PteFrame = COLORED_LIST_HEAD;
 437     }
 438
 439     /* One less page */
 440     ColorTable->Count--;
 441
 442     /* ReactOS Hack */
 443     Pfn1->OriginalPte.u.Long = 0;
 444
 445     /* See if we hit any thresholds */
 446     if (MmAvailablePages == MmHighMemoryThreshold)
 447     {
 448         /* Clear the high memory event */
 449         KeClearEvent(MiHighMemoryEvent);
 450     }
 451     else if (MmAvailablePages == MmLowMemoryThreshold)
 452     {
 453         /* Signal the low memory event */
 454         KeSetEvent(MiLowMemoryEvent, 0, FALSE);
 455     }
 456
 457     /* One less page */
 458     if (--MmAvailablePages < MmMinimumFreePages)
 459     {
 460         /* FIXME: Should wake up the MPW and working set manager, if we had one */
 461
 462         DPRINT1("Running low on pages: %lu remaining\n", MmAvailablePages);
 463
 464         /* Call RosMm and see if it can release any pages for us */
 465         MmRebalanceMemoryConsumers();
 466     }
 467
 468 #if MI_TRACE_PFNS
 469     //ASSERT(MI_PFN_CURRENT_USAGE != MI_USAGE_NOT_SET);
 470     Pfn1->PfnUsage = MI_PFN_CURRENT_USAGE;
 471     memcpy(Pfn1->ProcessName, MI_PFN_CURRENT_PROCESS_NAME, 16);
 472     //MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 473     //memcpy(MI_PFN_CURRENT_PROCESS_NAME, "Not Set", 16);
 474 #endif
 475
 476     /* Return the page */
 477     return PageIndex;
 478 }
 479
 480 PFN_NUMBER
 481 NTAPI
 482 MiRemoveAnyPage(IN ULONG Color)
 483 {
 484     PFN_NUMBER PageIndex;
 485     PMMPFN Pfn1;
 486
 487     /* Make sure PFN lock is held and we have pages */
 488     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 489     ASSERT(MmAvailablePages != 0);
 490     ASSERT(Color < MmSecondaryColors);
 491
 492     /* Check the colored free list */
 493     PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 494     if (PageIndex == LIST_HEAD)
 495     {
 496         /* Check the colored zero list */
 497         PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 498         if (PageIndex == LIST_HEAD)
 499         {
 500             /* Check the free list */
 501             ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 502             PageIndex = MmFreePageListHead.Flink;
 503             Color = PageIndex & MmSecondaryColorMask;
 504             if (PageIndex == LIST_HEAD)
 505             {
 506                 /* Check the zero list */
 507                 ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 508                 PageIndex = MmZeroedPageListHead.Flink;
 509                 Color = PageIndex & MmSecondaryColorMask;
 510                 ASSERT(PageIndex != LIST_HEAD);
 511                 if (PageIndex == LIST_HEAD)
 512                 {
 513                     /* FIXME: Should check the standby list */
 514                     ASSERT(MmZeroedPageListHead.Total == 0);
 515                 }
 516             }
 517         }
 518     }
 519
 520     /* Remove the page from its list */
 521     PageIndex = MiRemovePageByColor(PageIndex, Color);
 522
 523     /* Sanity checks */
 524     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 525     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 526            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 527     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 528     ASSERT(Pfn1->u2.ShareCount == 0);
 529     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 530     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 531
 532     /* Return the page */
 533     return PageIndex;
 534 }
 535
 536 PFN_NUMBER
 537 NTAPI
 538 MiRemoveZeroPage(IN ULONG Color)
 539 {
 540     PFN_NUMBER PageIndex;
 541     PMMPFN Pfn1;
 542     BOOLEAN Zero = FALSE;
 543
 544     /* Make sure PFN lock is held and we have pages */
 545     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 546     ASSERT(MmAvailablePages != 0);
 547     ASSERT(Color < MmSecondaryColors);
 548
 549     /* Check the colored zero list */
 550     PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink;
 551     if (PageIndex == LIST_HEAD)
 552     {
 553         /* Check the zero list */
 554         ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 555         PageIndex = MmZeroedPageListHead.Flink;
 556         if (PageIndex == LIST_HEAD)
 557         {
 558             /* This means there's no zero pages, we have to look for free ones */
 559             ASSERT(MmZeroedPageListHead.Total == 0);
 560             Zero = TRUE;
 561
 562             /* Check the colored free list */
 563             PageIndex = MmFreePagesByColor[FreePageList][Color].Flink;
 564             if (PageIndex == LIST_HEAD)
 565             {
 566                 /* Check the free list */
 567                 ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 568                 PageIndex = MmFreePageListHead.Flink;
 569                 Color = PageIndex & MmSecondaryColorMask;
 570                 ASSERT(PageIndex != LIST_HEAD);
 571                 if (PageIndex == LIST_HEAD)
 572                 {
 573                     /* FIXME: Should check the standby list */
 574                     ASSERT(MmZeroedPageListHead.Total == 0);
 575                 }
 576             }
 577         }
 578         else
 579         {
 580             Color = PageIndex & MmSecondaryColorMask;
 581         }
 582     }
 583
 584     /* Sanity checks */
 585     Pfn1 = MI_PFN_ELEMENT(PageIndex);
 586     ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) ||
 587            (Pfn1->u3.e1.PageLocation == ZeroedPageList));
 588
 589     /* Remove the page from its list */
 590     PageIndex = MiRemovePageByColor(PageIndex, Color);
 591     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageIndex));
 592
 593     /* Zero it, if needed */
 594     if (Zero) MiZeroPhysicalPage(PageIndex);
 595
 596     /* Sanity checks */
 597     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 598     ASSERT(Pfn1->u2.ShareCount == 0);
 599     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
 600     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
 601
 602     /* Return the page */
 603     return PageIndex;
 604 }
 605
 606 VOID
 607 NTAPI
 608 MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex)
 609 {
 610     PMMPFNLIST ListHead;
 611     PFN_NUMBER LastPage;
 612     PMMPFN Pfn1;
 613     ULONG Color;
 614     PMMPFN Blink;
 615     PMMCOLOR_TABLES ColorTable;
 616
 617     /* Make sure the page index is valid */
 618     ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
 619     ASSERT((PageFrameIndex != 0) &&
 620            (PageFrameIndex <= MmHighestPhysicalPage) &&
 621            (PageFrameIndex >= MmLowestPhysicalPage));
 622
 623     /* Get the PFN entry */
 624     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 625
 626     /* Sanity checks that a right kind of page is being inserted here */
 627     ASSERT(Pfn1->u4.MustBeCached == 0);
 628     ASSERT(Pfn1->u3.e1.Rom != 1);
 629     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 630     ASSERT(Pfn1->u4.VerifierAllocation == 0);
 631     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 632
 633     /* Get the free page list and increment its count */
 634     ListHead = &MmFreePageListHead;
 635     ASSERT_LIST_INVARIANT(ListHead);
 636     ListHead->Total++;
 637
 638     /* Get the last page on the list */
 639     LastPage = ListHead->Blink;
 640     if (LastPage != LIST_HEAD)
 641     {
 642         /* Link us with the previous page, so we're at the end now */
 643         MI_PFN_ELEMENT(LastPage)->u1.Flink = PageFrameIndex;
 644     }
 645     else
 646     {
 647         /* The list is empty, so we are the first page */
 648         ListHead->Flink = PageFrameIndex;
 649     }
 650
 651     /* Now make the list head point back to us (since we go at the end) */
 652     ListHead->Blink = PageFrameIndex;
 653     ASSERT_LIST_INVARIANT(ListHead);
 654
 655     /* And initialize our own list pointers */
 656     Pfn1->u1.Flink = LIST_HEAD;
 657     Pfn1->u2.Blink = LastPage;
 658
 659     /* Set the list name and default priority */
 660     Pfn1->u3.e1.PageLocation = FreePageList;
 661     Pfn1->u4.Priority = 3;
 662
 663     /* Clear some status fields */
 664     Pfn1->u4.InPageError = 0;
 665     Pfn1->u4.AweAllocation = 0;
 666
 667     /* Increase available pages */
 668     MmAvailablePages++;
 669
 670     /* Check if we've reached the configured low memory threshold */
 671     if (MmAvailablePages == MmLowMemoryThreshold)
 672     {
 673         /* Clear the event, because now we're ABOVE the threshold */
 674         KeClearEvent(MiLowMemoryEvent);
 675     }
 676     else if (MmAvailablePages == MmHighMemoryThreshold)
 677     {
 678         /* Otherwise check if we reached the high threshold and signal the event */
 679         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 680     }
 681
 682     /* Get the page color */
 683     Color = PageFrameIndex & MmSecondaryColorMask;
 684
 685     /* Get the first page on the color list */
 686     ColorTable = &MmFreePagesByColor[FreePageList][Color];
 687     if (ColorTable->Flink == LIST_HEAD)
 688     {
 689         /* The list is empty, so we are the first page */
 690         Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 691         ColorTable->Flink = PageFrameIndex;
 692     }
 693     else
 694     {
 695         /* Get the previous page */
 696         Blink = (PMMPFN)ColorTable->Blink;
 697
 698         /* Make it link to us, and link back to it */
 699         Blink->OriginalPte.u.Long = PageFrameIndex;
 700         Pfn1->u4.PteFrame = MiGetPfnEntryIndex(Blink);
 701     }
 702
 703     /* Now initialize our own list pointers */
 704     ColorTable->Blink = Pfn1;
 705
 706     /* This page is now the last */
 707     Pfn1->OriginalPte.u.Long = LIST_HEAD;
 708
 709     /* And increase the count in the colored list */
 710     ColorTable->Count++;
 711
 712     /* Notify zero page thread if enough pages are on the free list now */
 713     if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive))
 714     {
 715         /* Set the event */
 716         MmZeroingPageThreadActive = TRUE;
 717         KeSetEvent(&MmZeroingPageEvent, IO_NO_INCREMENT, FALSE);
 718     }
 719
 720 #if MI_TRACE_PFNS
 721     Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 722     RtlZeroMemory(Pfn1->ProcessName, 16);
 723 #endif
 724 }
 725
 726 VOID
 727 FASTCALL
 728 MiInsertStandbyListAtFront(IN PFN_NUMBER PageFrameIndex)
 729 {
 730     PMMPFNLIST ListHead;
 731     PFN_NUMBER Flink;
 732     PMMPFN Pfn1, Pfn2;
 733
 734     /* Make sure the lock is held */
 735     DPRINT1("Inserting page: %lx into standby list !\n", PageFrameIndex);
 736     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 737
 738     /* Make sure the PFN is valid */
 739     ASSERT((PageFrameIndex != 0) &&
 740            (PageFrameIndex <= MmHighestPhysicalPage) &&
 741            (PageFrameIndex >= MmLowestPhysicalPage));
 742
 743     /* Grab the PFN and validate it is the right kind of PFN being inserted */
 744     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 745     ASSERT(Pfn1->u4.MustBeCached == 0);
 746     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 747     ASSERT(Pfn1->u3.e1.PrototypePte == 1);
 748     ASSERT(Pfn1->u3.e1.Rom != 1);
 749
 750     /* One more transition page on a list */
 751     MmTransitionSharedPages++;
 752
 753     /* Get the standby page list and increment its count */
 754     ListHead = &MmStandbyPageListByPriority [Pfn1->u4.Priority];
 755     ASSERT_LIST_INVARIANT(ListHead);
 756     ListHead->Total++;
 757
 758     /* Make the head of the list point to this page now */
 759     Flink = ListHead->Flink;
 760     ListHead->Flink = PageFrameIndex;
 761
 762     /* Make the page point to the previous head, and back to the list */
 763     Pfn1->u1.Flink = Flink;
 764     Pfn1->u2.Blink = LIST_HEAD;
 765
 766     /* Was the list empty? */
 767     if (Flink != LIST_HEAD)
 768     {
 769         /* It wasn't, so update the backlink of the previous head page */
 770         Pfn2 = MI_PFN_ELEMENT(Flink);
 771         Pfn2->u2.Blink = PageFrameIndex;
 772     }
 773     else
 774     {
 775         /* It was empty, so have it loop back around to this new page */
 776         ListHead->Blink = PageFrameIndex;
 777     }
 778
 779     /* Move the page onto its new location */
 780     Pfn1->u3.e1.PageLocation = StandbyPageList;
 781
 782     /* One more page on the system */
 783     MmAvailablePages++;
 784
 785     /* Check if we've reached the configured low memory threshold */
 786     if (MmAvailablePages == MmLowMemoryThreshold)
 787     {
 788         /* Clear the event, because now we're ABOVE the threshold */
 789         KeClearEvent(MiLowMemoryEvent);
 790     }
 791     else if (MmAvailablePages == MmHighMemoryThreshold)
 792     {
 793         /* Otherwise check if we reached the high threshold and signal the event */
 794         KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 795     }
 796 }
 797
 798 VOID
 799 NTAPI
 800 MiInsertPageInList(IN PMMPFNLIST ListHead,
 801                    IN PFN_NUMBER PageFrameIndex)
 802 {
 803     PFN_NUMBER Flink, LastPage;
 804     PMMPFN Pfn1, Pfn2;
 805     MMLISTS ListName;
 806     PMMCOLOR_TABLES ColorHead;
 807     ULONG Color;
 808
 809     /* For free pages, use MiInsertPageInFreeList */
 810     ASSERT(ListHead != &MmFreePageListHead);
 811
 812     /* Make sure the lock is held */
 813     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
 814
 815     /* Make sure the PFN is valid */
 816     ASSERT((PageFrameIndex) &&
 817            (PageFrameIndex <= MmHighestPhysicalPage) &&
 818            (PageFrameIndex >= MmLowestPhysicalPage));
 819
 820     /* Page should be unused */
 821     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
 822     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 823     ASSERT(Pfn1->u3.e1.Rom != 1);
 824
 825     /* Is a standby or modified page being inserted? */
 826     ListName = ListHead->ListName;
 827     if ((ListName == StandbyPageList) || (ListName == ModifiedPageList))
 828     {
 829         /* If the page is in transition, it must also be a prototype page */
 830         if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
 831             (Pfn1->OriginalPte.u.Soft.Transition == 1))
 832         {
 833             /* Crash the system on inconsistency */
 834             KeBugCheckEx(MEMORY_MANAGEMENT, 0x8888, 0, 0, 0);
 835         }
 836     }
 837
 838     /* Standby pages are prioritized, so we need to get the real head */
 839     if (ListHead == &MmStandbyPageListHead)
 840     {
 841         /* Obviously the prioritized list should still have the same name */
 842         ListHead = &MmStandbyPageListByPriority[Pfn1->u4.Priority];
 843         ASSERT(ListHead->ListName == ListName);
 844     }
 845
 846     /* Increment the list count */
 847     ListHead->Total++;
 848
 849     /* Is a modified page being inserted? */
 850     if (ListHead == &MmModifiedPageListHead)
 851     {
 852         /* For now, only single-prototype pages should end up in this path */
 853         DPRINT("Modified page being added: %lx\n", PageFrameIndex);
 854         ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 855
 856         /* Modified pages are colored when they are selected for page file */
 857         ListHead = &MmModifiedPageListByColor[0];
 858         ASSERT (ListHead->ListName == ListName);
 859         ListHead->Total++;
 860
 861         /* Increment the number of paging file modified pages */
 862         MmTotalPagesForPagingFile++;
 863     }
 864
 865     /* Don't handle bad pages yet yet */
 866     ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
 867
 868     /* Zero pages go to the head, all other pages go to the end */
 869     if (ListName == ZeroedPageList)
 870     {
 871         /* Make the head of the list point to this page now */
 872         Flink = ListHead->Flink;
 873         ListHead->Flink = PageFrameIndex;
 874
 875         /* Make the page point to the previous head, and back to the list */
 876         Pfn1->u1.Flink = Flink;
 877         Pfn1->u2.Blink = LIST_HEAD;
 878
 879         /* Was the list empty? */
 880         if (Flink != LIST_HEAD)
 881         {
 882             /* It wasn't, so update the backlink of the previous head page */
 883             Pfn2 = MI_PFN_ELEMENT(Flink);
 884             Pfn2->u2.Blink = PageFrameIndex;
 885         }
 886         else
 887         {
 888             /* It was empty, so have it loop back around to this new page */
 889             ListHead->Blink = PageFrameIndex;
 890         }
 891     }
 892     else
 893     {
 894         /* Get the last page on the list */
 895         LastPage = ListHead->Blink;
 896         if (LastPage != LIST_HEAD)
 897         {
 898             /* Link us with the previous page, so we're at the end now */
 899             MI_PFN_ELEMENT(LastPage)->u1.Flink = PageFrameIndex;
 900         }
 901         else
 902         {
 903             /* The list is empty, so we are the first page */
 904             ListHead->Flink = PageFrameIndex;
 905         }
 906
 907         /* Now make the list head point back to us (since we go at the end) */
 908         ListHead->Blink = PageFrameIndex;
 909         ASSERT_LIST_INVARIANT(ListHead);
 910
 911         /* And initialize our own list pointers */
 912         Pfn1->u1.Flink = LIST_HEAD;
 913         Pfn1->u2.Blink = LastPage;
 914     }
 915
 916     /* Move the page onto its new location */
 917     Pfn1->u3.e1.PageLocation = ListName;
 918
 919     /* For zero/free pages, we also have to handle the colored lists */
 920     if (ListName <= StandbyPageList)
 921     {
 922         /* One more page on the system */
 923         MmAvailablePages++;
 924
 925         /* Check if we've reached the configured low memory threshold */
 926         if (MmAvailablePages == MmLowMemoryThreshold)
 927         {
 928             /* Clear the event, because now we're ABOVE the threshold */
 929             KeClearEvent(MiLowMemoryEvent);
 930         }
 931         else if (MmAvailablePages == MmHighMemoryThreshold)
 932         {
 933             /* Otherwise check if we reached the high threshold and signal the event */
 934             KeSetEvent(MiHighMemoryEvent, 0, FALSE);
 935         }
 936
 937         /* Sanity checks */
 938         ASSERT(ListName == ZeroedPageList);
 939         ASSERT(Pfn1->u4.InPageError == 0);
 940
 941         /* Get the page color */
 942         Color = PageFrameIndex & MmSecondaryColorMask;
 943
 944         /* Get the list for this color */
 945         ColorHead = &MmFreePagesByColor[ZeroedPageList][Color];
 946
 947         /* Get the old head */
 948         Flink = ColorHead->Flink;
 949
 950         /* Make this page point back to the list, and point forwards to the old head */
 951         Pfn1->OriginalPte.u.Long = Flink;
 952         Pfn1->u4.PteFrame = COLORED_LIST_HEAD;
 953
 954         /* Set the new head */
 955         ColorHead->Flink = PageFrameIndex;
 956
 957         /* Was the head empty? */
 958         if (Flink != LIST_HEAD)
 959         {
 960             /* No, so make the old head point to this page */
 961             Pfn2 = MI_PFN_ELEMENT(Flink);
 962             Pfn2->u4.PteFrame = PageFrameIndex;
 963         }
 964         else
 965         {
 966             /* Yes, make it loop back to this page */
 967             ColorHead->Blink = (PVOID)Pfn1;
 968         }
 969
 970         /* One more paged on the colored list */
 971         ColorHead->Count++;
 972
 973 #if MI_TRACE_PFNS
 974             //ASSERT(MI_PFN_CURRENT_USAGE == MI_USAGE_NOT_SET);
 975             Pfn1->PfnUsage = MI_USAGE_FREE_PAGE;
 976             MI_PFN_CURRENT_USAGE = MI_USAGE_NOT_SET;
 977             RtlZeroMemory(Pfn1->ProcessName, 16);
 978 #endif
 979     }
 980     else if (ListName == ModifiedPageList)
 981     {
 982         /* In ARM3, page must be destined for page file, and not yet written out */
 983         ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
 984         ASSERT(Pfn1->OriginalPte.u.Soft.PageFileHigh == 0);
 985
 986         /* One more transition page */
 987         ASSERT(Pfn1->u3.e1.PrototypePte == 1);
 988         MmTransitionSharedPages++;
 989
 990         /* Increment the number of per-process modified pages */
 991         PsGetCurrentProcess()->ModifiedPageCount++;
 992
 993         /* FIXME: Wake up modified page writer if there are not enough free pages */
 994     }
 995     else if (ListName == ModifiedNoWritePageList)
 996     {
 997         /* This list is not yet implemented */
 998         ASSERT(FALSE);
 999     }
1000 }
1001
1002 VOID
1003 NTAPI
1004 MiInitializePfn(IN PFN_NUMBER PageFrameIndex,
1005                 IN PMMPTE PointerPte,
1006                 IN BOOLEAN Modified)
1007 {
1008     PMMPFN Pfn1;
1009     NTSTATUS Status;
1010     PMMPTE PointerPtePte;
1011     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1012
1013     /* Setup the PTE */
1014     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1015     Pfn1->PteAddress = PointerPte;
1016
1017     /* Check if this PFN is part of a valid address space */
1018     if (PointerPte->u.Hard.Valid == 1)
1019     {
1020         /* Only valid from MmCreateProcessAddressSpace path */
1021         ASSERT(PsGetCurrentProcess()->Vm.WorkingSetSize == 0);
1022
1023         /* Make this a demand zero PTE */
1024         MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
1025     }
1026     else
1027     {
1028         /* Copy the PTE data */
1029         Pfn1->OriginalPte = *PointerPte;
1030         ASSERT(!((Pfn1->OriginalPte.u.Soft.Prototype == 0) &&
1031                  (Pfn1->OriginalPte.u.Soft.Transition == 1)));
1032     }
1033
1034     /* Otherwise this is a fresh page -- set it up */
1035     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
1036     Pfn1->u3.e2.ReferenceCount = 1;
1037     Pfn1->u2.ShareCount = 1;
1038     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1039     ASSERT(Pfn1->u3.e1.Rom == 0);
1040     Pfn1->u3.e1.Modified = Modified;
1041
1042     /* Get the page table for the PTE */
1043     PointerPtePte = MiAddressToPte(PointerPte);
1044     if (PointerPtePte->u.Hard.Valid == 0)
1045     {
1046         /* Make sure the PDE gets paged in properly */
1047         Status = MiCheckPdeForPagedPool(PointerPte);
1048         if (!NT_SUCCESS(Status))
1049         {
1050             /* Crash */
1051             KeBugCheckEx(MEMORY_MANAGEMENT,
1052                          0x61940,
1053                          (ULONG_PTR)PointerPte,
1054                          (ULONG_PTR)PointerPtePte->u.Long,
1055                          (ULONG_PTR)MiPteToAddress(PointerPte));
1056         }
1057     }
1058
1059     /* Get the PFN for the page table */
1060     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
1061     ASSERT(PageFrameIndex != 0);
1062     Pfn1->u4.PteFrame = PageFrameIndex;
1063
1064     /* Increase its share count so we don't get rid of it */
1065     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1066     Pfn1->u2.ShareCount++;
1067 }
1068
1069 VOID
1070 NTAPI
1071 MiInitializePfnAndMakePteValid(IN PFN_NUMBER PageFrameIndex,
1072                                IN PMMPTE PointerPte,
1073                                IN MMPTE TempPte)
1074 {
1075     PMMPFN Pfn1;
1076     NTSTATUS Status;
1077     PMMPTE PointerPtePte;
1078     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1079
1080     /* PTE must be invalid */
1081     ASSERT(PointerPte->u.Hard.Valid == 0);
1082
1083     /* Setup the PTE */
1084     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1085     Pfn1->PteAddress = PointerPte;
1086     Pfn1->OriginalPte = DemandZeroPte;
1087
1088     /* Otherwise this is a fresh page -- set it up */
1089     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
1090     Pfn1->u3.e2.ReferenceCount++;
1091     Pfn1->u2.ShareCount++;
1092     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1093     ASSERT(Pfn1->u3.e1.Rom == 0);
1094     Pfn1->u3.e1.Modified = 1;
1095
1096     /* Get the page table for the PTE */
1097     PointerPtePte = MiAddressToPte(PointerPte);
1098     if (PointerPtePte->u.Hard.Valid == 0)
1099     {
1100         /* Make sure the PDE gets paged in properly */
1101         Status = MiCheckPdeForPagedPool(PointerPte);
1102         if (!NT_SUCCESS(Status))
1103         {
1104             /* Crash */
1105             KeBugCheckEx(MEMORY_MANAGEMENT,
1106                          0x61940,
1107                          (ULONG_PTR)PointerPte,
1108                          (ULONG_PTR)PointerPtePte->u.Long,
1109                          (ULONG_PTR)MiPteToAddress(PointerPte));
1110         }
1111     }
1112
1113     /* Get the PFN for the page table */
1114     PageFrameIndex = PFN_FROM_PTE(PointerPtePte);
1115     ASSERT(PageFrameIndex != 0);
1116     Pfn1->u4.PteFrame = PageFrameIndex;
1117
1118     /* Increase its share count so we don't get rid of it */
1119     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1120     Pfn1->u2.ShareCount++;
1121
1122     /* Write valid PTE */
1123     MI_WRITE_VALID_PTE(PointerPte, TempPte);
1124 }
1125
1126 NTSTATUS
1127 NTAPI
1128 MiInitializeAndChargePfn(OUT PPFN_NUMBER PageFrameIndex,
1129                          IN PMMPTE PointerPde,
1130                          IN PFN_NUMBER ContainingPageFrame,
1131                          IN BOOLEAN SessionAllocation)
1132 {
1133     MMPTE TempPte;
1134     KIRQL OldIrql;
1135
1136     /* Use either a global or local PDE */
1137     TempPte = SessionAllocation ? ValidKernelPdeLocal : ValidKernelPde;
1138
1139     /* Lock the PFN database */
1140     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1141
1142     /* Make sure nobody is racing us */
1143     if (PointerPde->u.Hard.Valid == 1)
1144     {
1145         /* Return special error if that was the case */
1146         KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1147         return STATUS_RETRY;
1148     }
1149
1150     /* Grab a zero page and set the PFN, then make it valid */
1151     *PageFrameIndex = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
1152     TempPte.u.Hard.PageFrameNumber = *PageFrameIndex;
1153     MI_WRITE_VALID_PTE(PointerPde, TempPte);
1154
1155     /* Initialize the PFN */
1156     MiInitializePfnForOtherProcess(*PageFrameIndex,
1157                                    PointerPde,
1158                                    ContainingPageFrame);
1159     ASSERT(MI_PFN_ELEMENT(*PageFrameIndex)->u1.WsIndex == 0);
1160
1161     /* Release the lock and return success */
1162     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1163     return STATUS_SUCCESS;
1164 }
1165
1166 PFN_NUMBER
1167 NTAPI
1168 MiAllocatePfn(IN PMMPTE PointerPte,
1169               IN ULONG Protection)
1170 {
1171     KIRQL OldIrql;
1172     PFN_NUMBER PageFrameIndex;
1173     MMPTE TempPte;
1174
1175     /* Sanity check that we aren't passed a valid PTE */
1176     ASSERT(PointerPte->u.Hard.Valid == 0);
1177
1178     /* Make an empty software PTE */
1179     MI_MAKE_SOFTWARE_PTE(&TempPte, MM_READWRITE);
1180
1181     /* Lock the PFN database */
1182     OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
1183
1184     /* Check if we're running low on pages */
1185     if (MmAvailablePages < 128)
1186     {
1187         DPRINT1("Warning, running low on memory: %lu pages left\n", MmAvailablePages);
1188
1189         //MiEnsureAvailablePageOrWait(NULL, OldIrql);
1190
1191         /* Call RosMm and see if it can release any pages for us */
1192         MmRebalanceMemoryConsumers();
1193     }
1194
1195     /* Grab a page */
1196     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
1197     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
1198     PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR());
1199
1200     /* Write the software PTE */
1201     MI_WRITE_INVALID_PTE(PointerPte, TempPte);
1202     PointerPte->u.Soft.Protection |= Protection;
1203
1204     /* Initialize its PFN entry */
1205     MiInitializePfn(PageFrameIndex, PointerPte, TRUE);
1206
1207     /* Release the PFN lock and return the page */
1208     ASSERT_LIST_INVARIANT(&MmFreePageListHead);
1209     ASSERT_LIST_INVARIANT(&MmZeroedPageListHead);
1210     KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
1211     return PageFrameIndex;
1212 }
1213
1214 VOID
1215 NTAPI
1216 MiDecrementShareCount(IN PMMPFN Pfn1,
1217                       IN PFN_NUMBER PageFrameIndex)
1218 {
1219     PMMPTE PointerPte;
1220     MMPTE TempPte;
1221
1222     ASSERT(PageFrameIndex > 0);
1223     ASSERT(MI_PFN_ELEMENT(PageFrameIndex) != NULL);
1224     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
1225     ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
1226
1227     /* Page must be in-use */
1228     if ((Pfn1->u3.e1.PageLocation != ActiveAndValid) &&
1229         (Pfn1->u3.e1.PageLocation != StandbyPageList))
1230     {
1231         /* Otherwise we have PFN corruption */
1232         KeBugCheckEx(PFN_LIST_CORRUPT,
1233                      0x99,
1234                      PageFrameIndex,
1235                      Pfn1->u3.e1.PageLocation,
1236                      0);
1237     }
1238
1239     /* Check if the share count is now 0 */
1240     ASSERT(Pfn1->u2.ShareCount < 0xF000000);
1241     if (!--Pfn1->u2.ShareCount)
1242     {
1243         /* Was this a prototype PTE? */
1244         if (Pfn1->u3.e1.PrototypePte)
1245         {
1246             /* Grab the PTE address and make sure it's in prototype pool */
1247             PointerPte = Pfn1->PteAddress;
1248             ASSERT((PointerPte >= (PMMPTE)MmPagedPoolStart) && (PointerPte <= (PMMPTE)MmPagedPoolEnd));
1249
1250             /* The PTE that backs it should also be valdi */
1251             PointerPte = MiAddressToPte(PointerPte);
1252             ASSERT(PointerPte->u.Hard.Valid == 1);
1253
1254             /* Get the original prototype PTE and turn it into a transition PTE */
1255             PointerPte = Pfn1->PteAddress;
1256             TempPte = *PointerPte;
1257             TempPte.u.Soft.Transition = 1;
1258             TempPte.u.Soft.Valid = 0;
1259             TempPte.u.Soft.Prototype = 0;
1260             TempPte.u.Soft.Protection = Pfn1->OriginalPte.u.Soft.Protection;
1261             MI_WRITE_INVALID_PTE(PointerPte, TempPte);
1262             DPRINT("Marking PTE: %p as transition (%p - %lx)\n", PointerPte, Pfn1, MiGetPfnEntryIndex(Pfn1));
1263         }
1264
1265         /* Put the page in transition */
1266         Pfn1->u3.e1.PageLocation = TransitionPage;
1267
1268         /* PFN lock must be held */
1269         ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1270
1271         /* Page should at least have one reference */
1272         ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1273         if (Pfn1->u3.e2.ReferenceCount == 1)
1274         {
1275             /* Is there still a PFN for this page? */
1276             if (MI_IS_PFN_DELETED(Pfn1) == TRUE)
1277             {
1278                 /* Clear the last reference */
1279                 Pfn1->u3.e2.ReferenceCount = 0;
1280                 ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0);
1281
1282                 /* Mark the page temporarily as valid, we're going to make it free soon */
1283                 Pfn1->u3.e1.PageLocation = ActiveAndValid;
1284
1285                 /* Bring it back into the free list */
1286                 MiInsertPageInFreeList(PageFrameIndex);
1287             }
1288             else
1289             {
1290                 /* PFN not yet deleted, drop a ref count */
1291                 MiDecrementReferenceCount(Pfn1, PageFrameIndex);
1292             }
1293         }
1294         else
1295         {
1296             /* Otherwise, just drop the reference count */
1297             InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1298         }
1299     }
1300 }
1301
1302 VOID
1303 NTAPI
1304 MiDecrementReferenceCount(IN PMMPFN Pfn1,
1305                           IN PFN_NUMBER PageFrameIndex)
1306 {
1307     /* PFN lock must be held */
1308     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1309
1310     /* Sanity checks on the page */
1311     ASSERT(PageFrameIndex < MmHighestPhysicalPage);
1312     ASSERT(Pfn1 == MI_PFN_ELEMENT(PageFrameIndex));
1313     ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
1314
1315     /* Dereference the page, bail out if it's still alive */
1316     InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount);
1317     if (Pfn1->u3.e2.ReferenceCount) return;
1318
1319     /* Nobody should still have reference to this page */
1320     if (Pfn1->u2.ShareCount != 0)
1321     {
1322         /* Otherwise something's really wrong */
1323         KeBugCheckEx(PFN_LIST_CORRUPT, 7, PageFrameIndex, Pfn1->u2.ShareCount, 0);
1324     }
1325
1326     /* And it should be lying on some page list */
1327     ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid);
1328
1329     /* Did someone set the delete flag? */
1330     if (MI_IS_PFN_DELETED(Pfn1))
1331     {
1332         /* Insert it into the free list, there's nothing left to do */
1333         MiInsertPageInFreeList(PageFrameIndex);
1334         return;
1335     }
1336
1337     /* Check to see which list this page should go into */
1338     if (Pfn1->u3.e1.Modified == 1)
1339     {
1340         /* Push it into the modified page list */
1341         MiInsertPageInList(&MmModifiedPageListHead, PageFrameIndex);
1342     }
1343     else
1344     {
1345         /* Otherwise, insert this page into the standby list */
1346         ASSERT(Pfn1->u3.e1.RemovalRequested == 0);
1347         MiInsertStandbyListAtFront(PageFrameIndex);
1348     }
1349 }
1350
1351 VOID
1352 NTAPI
1353 MiInitializePfnForOtherProcess(IN PFN_NUMBER PageFrameIndex,
1354                                IN PMMPTE PointerPte,
1355                                IN PFN_NUMBER PteFrame)
1356 {
1357     PMMPFN Pfn1;
1358
1359     /* Setup the PTE */
1360     Pfn1 = MI_PFN_ELEMENT(PageFrameIndex);
1361     Pfn1->PteAddress = PointerPte;
1362
1363     /* Make this a software PTE */
1364     MI_MAKE_SOFTWARE_PTE(&Pfn1->OriginalPte, MM_READWRITE);
1365
1366     /* Setup the page */
1367     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
1368     Pfn1->u3.e2.ReferenceCount = 1;
1369     Pfn1->u2.ShareCount = 1;
1370     Pfn1->u3.e1.PageLocation = ActiveAndValid;
1371     Pfn1->u3.e1.Modified = TRUE;
1372     Pfn1->u4.InPageError = FALSE;
1373
1374     /* Did we get a PFN for the page table */
1375     if (PteFrame)
1376     {
1377         /* Store it */
1378         Pfn1->u4.PteFrame = PteFrame;
1379
1380         /* Increase its share count so we don't get rid of it */
1381         Pfn1 = MI_PFN_ELEMENT(PteFrame);
1382         Pfn1->u2.ShareCount++;
1383     }
1384 }
1385
1386 /* EOF */