ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #if defined (ALLOC_PRAGMA)
  17 #pragma alloc_text(INIT, MmInitializeBalancer)
  18 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  19 #pragma alloc_text(INIT, MiInitBalancerThread)
  20 #endif
  21
  22
  23 /* TYPES ********************************************************************/
  24 typedef struct _MM_ALLOCATION_REQUEST
  25 {
  26    PFN_NUMBER Page;
  27    LIST_ENTRY ListEntry;
  28    KEVENT Event;
  29 }
  30 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  31
  32 /* GLOBALS ******************************************************************/
  33
  34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  35 static ULONG MiMinimumAvailablePages;
  36 static ULONG MiNrTotalPages;
  37 static LIST_ENTRY AllocationListHead;
  38 static KSPIN_LOCK AllocationListLock;
  39 static ULONG MiPagesRequired = 0;
  40 static ULONG MiMinimumPagesPerRun = 10;
  41
  42 static CLIENT_ID MiBalancerThreadId;
  43 static HANDLE MiBalancerThreadHandle = NULL;
  44 static KEVENT MiBalancerEvent;
  45 static KTIMER MiBalancerTimer;
  46 static LONG MiBalancerWork = 0;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID MmPrintMemoryStatistic(VOID)
  51 {
  52    DbgPrint("MC_CACHE %d, MC_USER %d, MC_PPOOL %d, MC_NPPOOL %d, MmAvailablePages %d\n",
  53             MiMemoryConsumers[MC_CACHE].PagesUsed, MiMemoryConsumers[MC_USER].PagesUsed,
  54             MiMemoryConsumers[MC_PPOOL].PagesUsed, MiMemoryConsumers[MC_NPPOOL].PagesUsed,
  55             MmAvailablePages);
  56 }
  57
  58 VOID
  59 INIT_FUNCTION
  60 NTAPI
  61 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  62 {
  63    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  64    InitializeListHead(&AllocationListHead);
  65    KeInitializeSpinLock(&AllocationListLock);
  66
  67    MiNrTotalPages = NrAvailablePages;
  68
  69    /* Set up targets. */
  70    MiMinimumAvailablePages = 64;
  71     if ((NrAvailablePages + NrSystemPages) >= 8192)
  72     {
  73         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  74     }
  75     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  76     {
  77         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  78     }
  79     else
  80     {
  81         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  82     }
  83    MiMemoryConsumers[MC_USER].PagesTarget =
  84       NrAvailablePages - MiMinimumAvailablePages;
  85    MiMemoryConsumers[MC_PPOOL].PagesTarget = NrAvailablePages / 2;
  86    MiMemoryConsumers[MC_NPPOOL].PagesTarget = 0xFFFFFFFF;
  87    MiMemoryConsumers[MC_NPPOOL].PagesUsed = NrSystemPages;
  88    MiMemoryConsumers[MC_SYSTEM].PagesTarget = 0xFFFFFFFF;
  89    MiMemoryConsumers[MC_SYSTEM].PagesUsed = 0;
  90 }
  91
  92 VOID
  93 INIT_FUNCTION
  94 NTAPI
  95 MmInitializeMemoryConsumer(ULONG Consumer,
  96                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  97                                             PULONG NrFreed))
  98 {
  99    MiMemoryConsumers[Consumer].Trim = Trim;
 100 }
 101
 102 VOID
 103 NTAPI
 104 MiZeroPhysicalPage(
 105     IN PFN_NUMBER PageFrameIndex
 106 );
 107
 108 NTSTATUS
 109 NTAPI
 110 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
 111 {
 112    PMM_ALLOCATION_REQUEST Request;
 113    PLIST_ENTRY Entry;
 114    KIRQL OldIrql;
 115
 116    if (Page == 0)
 117    {
 118       DPRINT1("Tried to release page zero.\n");
 119       KeBugCheck(MEMORY_MANAGEMENT);
 120    }
 121
 122    KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 123    if (MmGetReferenceCountPage(Page) == 1)
 124    {
 125       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 126       if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
 127       {
 128          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 129          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 130          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 131          MmDereferencePage(Page);
 132          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 133       }
 134       else
 135       {
 136          Entry = RemoveHeadList(&AllocationListHead);
 137          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 138          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 139          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 140          MiZeroPhysicalPage(Page);
 141          Request->Page = Page;
 142          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 143       }
 144    }
 145    else
 146    {
 147       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 148       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 149       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 150       MmDereferencePage(Page);
 151       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 152    }
 153
 154    return(STATUS_SUCCESS);
 155 }
 156
 157 VOID
 158 NTAPI
 159 MiTrimMemoryConsumer(ULONG Consumer)
 160 {
 161    LONG Target;
 162    ULONG NrFreedPages;
 163
 164    Target = MiMemoryConsumers[Consumer].PagesUsed -
 165             MiMemoryConsumers[Consumer].PagesTarget;
 166    if (Target < 1)
 167    {
 168       Target = 1;
 169    }
 170
 171    if (MiMemoryConsumers[Consumer].Trim != NULL)
 172    {
 173       MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 174    }
 175 }
 176
 177 NTSTATUS
 178 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 179 {
 180     PFN_NUMBER CurrentPage;
 181     PFN_NUMBER NextPage;
 182     NTSTATUS Status;
 183
 184     (*NrFreedPages) = 0;
 185
 186     CurrentPage = MmGetLRUFirstUserPage();
 187     while (CurrentPage != 0 && Target > 0)
 188     {
 189         NextPage = MmGetLRUNextUserPage(CurrentPage);
 190
 191         Status = MmPageOutPhysicalAddress(CurrentPage);
 192         if (NT_SUCCESS(Status))
 193         {
 194             DPRINT("Succeeded\n");
 195             Target--;
 196             (*NrFreedPages)++;
 197         }
 198
 199         CurrentPage = NextPage;
 200     }
 201     return(STATUS_SUCCESS);
 202 }
 203
 204 VOID
 205 NTAPI
 206 MmRebalanceMemoryConsumers(VOID)
 207 {
 208    LONG Target;
 209    ULONG i;
 210    ULONG NrFreedPages;
 211    NTSTATUS Status;
 212
 213    Target = (MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
 214    Target = max(Target, (LONG) MiMinimumPagesPerRun);
 215
 216    for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
 217    {
 218       if (MiMemoryConsumers[i].Trim != NULL)
 219       {
 220          Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 221          if (!NT_SUCCESS(Status))
 222          {
 223             KeBugCheck(MEMORY_MANAGEMENT);
 224          }
 225          Target = Target - NrFreedPages;
 226       }
 227    }
 228 }
 229
 230 static BOOLEAN
 231 MiIsBalancerThread(VOID)
 232 {
 233    return MiBalancerThreadHandle != NULL &&
 234           PsGetCurrentThread() == MiBalancerThreadId.UniqueThread;
 235 }
 236
 237 NTSTATUS
 238 NTAPI
 239 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 240                             PPFN_NUMBER AllocatedPage)
 241 {
 242    ULONG OldUsed;
 243    PFN_NUMBER Page;
 244    KIRQL OldIrql;
 245
 246    /*
 247     * Make sure we don't exceed our individual target.
 248     */
 249    OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 250    if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
 251          !MiIsBalancerThread())
 252    {
 253       if (!CanWait)
 254       {
 255          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 256          return(STATUS_NO_MEMORY);
 257       }
 258       MiTrimMemoryConsumer(Consumer);
 259    }
 260
 261    /*
 262     * Allocate always memory for the non paged pool and for the pager thread.
 263     */
 264    if ((Consumer == MC_NPPOOL) || (Consumer == MC_SYSTEM) || MiIsBalancerThread())
 265    {
 266       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 267       Page = MmAllocPage(Consumer);
 268       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 269       if (Page == 0)
 270       {
 271          KeBugCheck(NO_PAGES_AVAILABLE);
 272       }
 273       *AllocatedPage = Page;
 274       if (MmAvailablePages <= MiMinimumAvailablePages &&
 275             MiBalancerThreadHandle != NULL)
 276       {
 277          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 278       }
 279       return(STATUS_SUCCESS);
 280    }
 281
 282    /*
 283     * Make sure we don't exceed global targets.
 284     */
 285    if (MmAvailablePages <= MiMinimumAvailablePages)
 286    {
 287       MM_ALLOCATION_REQUEST Request;
 288
 289       if (!CanWait)
 290       {
 291          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 292          return(STATUS_NO_MEMORY);
 293       }
 294
 295       /* Insert an allocation request. */
 296       Request.Page = 0;
 297
 298       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 299       (void)InterlockedIncrementUL(&MiPagesRequired);
 300
 301       KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 302
 303       if (MiBalancerThreadHandle != NULL)
 304       {
 305          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 306       }
 307       InsertTailList(&AllocationListHead, &Request.ListEntry);
 308       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 309
 310       KeWaitForSingleObject(&Request.Event,
 311                             0,
 312                             KernelMode,
 313                             FALSE,
 314                             NULL);
 315
 316       Page = Request.Page;
 317       if (Page == 0)
 318       {
 319          KeBugCheck(NO_PAGES_AVAILABLE);
 320       }
 321       /* Update the Consumer and make the page active */
 322       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 323       *AllocatedPage = Page;
 324       (void)InterlockedDecrementUL(&MiPagesRequired);
 325       return(STATUS_SUCCESS);
 326    }
 327
 328    /*
 329     * Actually allocate the page.
 330     */
 331    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 332    Page = MmAllocPage(Consumer);
 333    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 334    if (Page == 0)
 335    {
 336       KeBugCheck(NO_PAGES_AVAILABLE);
 337    }
 338    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 339    *AllocatedPage = Page;
 340
 341    return(STATUS_SUCCESS);
 342 }
 343
 344 VOID NTAPI
 345 MiBalancerThread(PVOID Unused)
 346 {
 347    PVOID WaitObjects[2];
 348    NTSTATUS Status;
 349    ULONG i;
 350    ULONG NrFreedPages;
 351    ULONG NrPagesUsed;
 352    ULONG Target;
 353    BOOLEAN ShouldRun;
 354
 355
 356    WaitObjects[0] = &MiBalancerEvent;
 357    WaitObjects[1] = &MiBalancerTimer;
 358
 359    while (1)
 360    {
 361       Status = KeWaitForMultipleObjects(2,
 362                                         WaitObjects,
 363                                         WaitAny,
 364                                         Executive,
 365                                         KernelMode,
 366                                         FALSE,
 367                                         NULL,
 368                                         NULL);
 369
 370       if (Status == STATUS_SUCCESS)
 371       {
 372          /* MiBalancerEvent */
 373          while (MmAvailablePages < MiMinimumAvailablePages + 5)
 374          {
 375             for (i = 0; i < MC_MAXIMUM; i++)
 376             {
 377                if (MiMemoryConsumers[i].Trim != NULL)
 378                {
 379                   NrFreedPages = 0;
 380                   Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
 381                   if (!NT_SUCCESS(Status))
 382                   {
 383                      KeBugCheck(MEMORY_MANAGEMENT);
 384                   }
 385                }
 386             }
 387          }
 388          InterlockedExchange(&MiBalancerWork, 0);
 389       }
 390       else if (Status == STATUS_SUCCESS + 1)
 391       {
 392          /* MiBalancerTimer */
 393          ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
 394          for (i = 0; i < MC_MAXIMUM; i++)
 395          {
 396             if (MiMemoryConsumers[i].Trim != NULL)
 397             {
 398                NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
 399                if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
 400                {
 401                   if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
 402                   {
 403                      Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
 404                                    MiMinimumPagesPerRun);
 405                   }
 406                   else
 407                   {
 408                      Target = MiMinimumPagesPerRun;
 409                   }
 410                   NrFreedPages = 0;
 411                   Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 412                   if (!NT_SUCCESS(Status))
 413                   {
 414                      KeBugCheck(MEMORY_MANAGEMENT);
 415                   }
 416                }
 417             }
 418          }
 419       }
 420       else
 421       {
 422          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 423          KeBugCheck(MEMORY_MANAGEMENT);
 424       }
 425    }
 426 }
 427
 428 VOID
 429 INIT_FUNCTION
 430 NTAPI
 431 MiInitBalancerThread(VOID)
 432 {
 433    KPRIORITY Priority;
 434    NTSTATUS Status;
 435 #if !defined(__GNUC__)
 436
 437    LARGE_INTEGER dummyJunkNeeded;
 438    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 439    ;
 440 #endif
 441
 442
 443    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 444    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 445    KeSetTimerEx(&MiBalancerTimer,
 446 #if defined(__GNUC__)
 447                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 448 #else
 449                 dummyJunkNeeded,
 450 #endif
 451                 2000,         /* 2 sec */
 452                 NULL);
 453
 454    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 455                                  THREAD_ALL_ACCESS,
 456                                  NULL,
 457                                  NULL,
 458                                  &MiBalancerThreadId,
 459                                  (PKSTART_ROUTINE) MiBalancerThread,
 460                                  NULL);
 461    if (!NT_SUCCESS(Status))
 462    {
 463       KeBugCheck(MEMORY_MANAGEMENT);
 464    }
 465
 466    Priority = LOW_REALTIME_PRIORITY + 1;
 467    NtSetInformationThread(MiBalancerThreadHandle,
 468                           ThreadPriority,
 469                           &Priority,
 470                           sizeof(Priority));
 471
 472 }
 473
 474
 475 /* EOF */