ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #if defined (ALLOC_PRAGMA)
  17 #pragma alloc_text(INIT, MmInitializeBalancer)
  18 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  19 #pragma alloc_text(INIT, MiInitBalancerThread)
  20 #endif
  21
  22
  23 /* TYPES ********************************************************************/
  24 typedef struct _MM_ALLOCATION_REQUEST
  25 {
  26    PFN_TYPE Page;
  27    LIST_ENTRY ListEntry;
  28    KEVENT Event;
  29 }
  30 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  31
  32 /* GLOBALS ******************************************************************/
  33
  34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  35 static ULONG MiMinimumAvailablePages;
  36 static ULONG MiNrTotalPages;
  37 static LIST_ENTRY AllocationListHead;
  38 static KSPIN_LOCK AllocationListLock;
  39 static ULONG MiPagesRequired = 0;
  40 static ULONG MiMinimumPagesPerRun = 10;
  41
  42 static CLIENT_ID MiBalancerThreadId;
  43 static HANDLE MiBalancerThreadHandle = NULL;
  44 static KEVENT MiBalancerEvent;
  45 static KTIMER MiBalancerTimer;
  46 static LONG MiBalancerWork = 0;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID MmPrintMemoryStatistic(VOID)
  51 {
  52    DbgPrint("MC_CACHE %d, MC_USER %d, MC_PPOOL %d, MC_NPPOOL %d, MmAvailablePages %d\n",
  53             MiMemoryConsumers[MC_CACHE].PagesUsed, MiMemoryConsumers[MC_USER].PagesUsed,
  54             MiMemoryConsumers[MC_PPOOL].PagesUsed, MiMemoryConsumers[MC_NPPOOL].PagesUsed,
  55             MmAvailablePages);
  56 }
  57
  58 VOID
  59 INIT_FUNCTION
  60 NTAPI
  61 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  62 {
  63    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  64    InitializeListHead(&AllocationListHead);
  65    KeInitializeSpinLock(&AllocationListLock);
  66
  67    MiNrTotalPages = NrAvailablePages;
  68
  69    /* Set up targets. */
  70    MiMinimumAvailablePages = 64;
  71     if ((NrAvailablePages + NrSystemPages) >= 8192)
  72     {
  73         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  74     }
  75     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  76     {
  77         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  78     }
  79     else
  80     {
  81         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  82     }
  83    MiMemoryConsumers[MC_USER].PagesTarget =
  84       NrAvailablePages - MiMinimumAvailablePages;
  85    MiMemoryConsumers[MC_PPOOL].PagesTarget = NrAvailablePages / 2;
  86    MiMemoryConsumers[MC_NPPOOL].PagesTarget = 0xFFFFFFFF;
  87    MiMemoryConsumers[MC_NPPOOL].PagesUsed = NrSystemPages;
  88    MiMemoryConsumers[MC_SYSTEM].PagesTarget = 0xFFFFFFFF;
  89    MiMemoryConsumers[MC_SYSTEM].PagesUsed = 0;
  90 }
  91
  92 VOID
  93 INIT_FUNCTION
  94 NTAPI
  95 MmInitializeMemoryConsumer(ULONG Consumer,
  96                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  97                                             PULONG NrFreed))
  98 {
  99    MiMemoryConsumers[Consumer].Trim = Trim;
 100 }
 101
 102 NTSTATUS
 103 NTAPI
 104 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_TYPE Page)
 105 {
 106    PMM_ALLOCATION_REQUEST Request;
 107    PLIST_ENTRY Entry;
 108    KIRQL OldIrql;
 109
 110    if (Page == 0)
 111    {
 112       DPRINT1("Tried to release page zero.\n");
 113       KeBugCheck(MEMORY_MANAGEMENT);
 114    }
 115
 116    KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 117    if (MmGetReferenceCountPage(Page) == 1)
 118    {
 119       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 120       if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
 121       {
 122          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 123          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 124          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 125          MmDereferencePage(Page);
 126          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 127       }
 128       else
 129       {
 130          Entry = RemoveHeadList(&AllocationListHead);
 131          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 132          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 133          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 134          MiZeroPage(Page);
 135          Request->Page = Page;
 136          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 137       }
 138    }
 139    else
 140    {
 141       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 142       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 143       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 144       MmDereferencePage(Page);
 145       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 146    }
 147
 148    return(STATUS_SUCCESS);
 149 }
 150
 151 VOID
 152 NTAPI
 153 MiTrimMemoryConsumer(ULONG Consumer)
 154 {
 155    LONG Target;
 156    ULONG NrFreedPages;
 157
 158    Target = MiMemoryConsumers[Consumer].PagesUsed -
 159             MiMemoryConsumers[Consumer].PagesTarget;
 160    if (Target < 1)
 161    {
 162       Target = 1;
 163    }
 164
 165    if (MiMemoryConsumers[Consumer].Trim != NULL)
 166    {
 167       MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 168    }
 169 }
 170
 171 NTSTATUS
 172 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 173 {
 174     PFN_TYPE CurrentPage;
 175     PFN_TYPE NextPage;
 176     NTSTATUS Status;
 177
 178     (*NrFreedPages) = 0;
 179
 180     CurrentPage = MmGetLRUFirstUserPage();
 181     while (CurrentPage != 0 && Target > 0)
 182     {
 183         NextPage = MmGetLRUNextUserPage(CurrentPage);
 184
 185         Status = MmPageOutPhysicalAddress(CurrentPage);
 186         if (NT_SUCCESS(Status))
 187         {
 188             DPRINT("Succeeded\n");
 189             Target--;
 190             (*NrFreedPages)++;
 191         }
 192
 193         CurrentPage = NextPage;
 194     }
 195     return(STATUS_SUCCESS);
 196 }
 197
 198 VOID
 199 NTAPI
 200 MmRebalanceMemoryConsumers(VOID)
 201 {
 202    LONG Target;
 203    ULONG i;
 204    ULONG NrFreedPages;
 205    NTSTATUS Status;
 206
 207    Target = (MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
 208    Target = max(Target, (LONG) MiMinimumPagesPerRun);
 209
 210    for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
 211    {
 212       if (MiMemoryConsumers[i].Trim != NULL)
 213       {
 214          Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 215          if (!NT_SUCCESS(Status))
 216          {
 217             KeBugCheck(MEMORY_MANAGEMENT);
 218          }
 219          Target = Target - NrFreedPages;
 220       }
 221    }
 222 }
 223
 224 static BOOLEAN
 225 MiIsBalancerThread(VOID)
 226 {
 227    return MiBalancerThreadHandle != NULL &&
 228           PsGetCurrentThread() == MiBalancerThreadId.UniqueThread;
 229 }
 230
 231 VOID NTAPI MiSetConsumer(IN PFN_TYPE Pfn, IN ULONG Consumer);
 232
 233 NTSTATUS
 234 NTAPI
 235 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 236                             PPFN_TYPE AllocatedPage)
 237 {
 238    ULONG OldUsed;
 239    PFN_TYPE Page;
 240    KIRQL OldIrql;
 241
 242    /*
 243     * Make sure we don't exceed our individual target.
 244     */
 245    OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 246    if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
 247          !MiIsBalancerThread())
 248    {
 249       if (!CanWait)
 250       {
 251          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 252          return(STATUS_NO_MEMORY);
 253       }
 254       MiTrimMemoryConsumer(Consumer);
 255    }
 256
 257    /*
 258     * Allocate always memory for the non paged pool and for the pager thread.
 259     */
 260    if ((Consumer == MC_NPPOOL) || (Consumer == MC_SYSTEM) || MiIsBalancerThread())
 261    {
 262       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 263       Page = MmAllocPage(Consumer);
 264       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 265       if (Page == 0)
 266       {
 267          KeBugCheck(NO_PAGES_AVAILABLE);
 268       }
 269       *AllocatedPage = Page;
 270       if (MmAvailablePages <= MiMinimumAvailablePages &&
 271             MiBalancerThreadHandle != NULL)
 272       {
 273          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 274       }
 275       return(STATUS_SUCCESS);
 276    }
 277
 278    /*
 279     * Make sure we don't exceed global targets.
 280     */
 281    if (MmAvailablePages <= MiMinimumAvailablePages)
 282    {
 283       MM_ALLOCATION_REQUEST Request;
 284
 285       if (!CanWait)
 286       {
 287          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 288          return(STATUS_NO_MEMORY);
 289       }
 290
 291       /* Insert an allocation request. */
 292       Request.Page = 0;
 293
 294       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 295       (void)InterlockedIncrementUL(&MiPagesRequired);
 296
 297       KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 298
 299       if (MiBalancerThreadHandle != NULL)
 300       {
 301          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 302       }
 303       InsertTailList(&AllocationListHead, &Request.ListEntry);
 304       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 305
 306       KeWaitForSingleObject(&Request.Event,
 307                             0,
 308                             KernelMode,
 309                             FALSE,
 310                             NULL);
 311
 312       Page = Request.Page;
 313       if (Page == 0)
 314       {
 315          KeBugCheck(NO_PAGES_AVAILABLE);
 316       }
 317       /* Update the Consumer and make the page active */
 318       MiSetConsumer(Page, Consumer);
 319       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 320       *AllocatedPage = Page;
 321       (void)InterlockedDecrementUL(&MiPagesRequired);
 322       return(STATUS_SUCCESS);
 323    }
 324
 325    /*
 326     * Actually allocate the page.
 327     */
 328    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 329    Page = MmAllocPage(Consumer);
 330    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 331    if (Page == 0)
 332    {
 333       KeBugCheck(NO_PAGES_AVAILABLE);
 334    }
 335    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 336    *AllocatedPage = Page;
 337
 338    return(STATUS_SUCCESS);
 339 }
 340
 341 VOID NTAPI
 342 MiBalancerThread(PVOID Unused)
 343 {
 344    PVOID WaitObjects[2];
 345    NTSTATUS Status;
 346    ULONG i;
 347    ULONG NrFreedPages;
 348    ULONG NrPagesUsed;
 349    ULONG Target;
 350    BOOLEAN ShouldRun;
 351
 352
 353    WaitObjects[0] = &MiBalancerEvent;
 354    WaitObjects[1] = &MiBalancerTimer;
 355
 356    while (1)
 357    {
 358       Status = KeWaitForMultipleObjects(2,
 359                                         WaitObjects,
 360                                         WaitAny,
 361                                         Executive,
 362                                         KernelMode,
 363                                         FALSE,
 364                                         NULL,
 365                                         NULL);
 366
 367       if (Status == STATUS_SUCCESS)
 368       {
 369          /* MiBalancerEvent */
 370          while (MmAvailablePages < MiMinimumAvailablePages + 5)
 371          {
 372             for (i = 0; i < MC_MAXIMUM; i++)
 373             {
 374                if (MiMemoryConsumers[i].Trim != NULL)
 375                {
 376                   NrFreedPages = 0;
 377                   Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
 378                   if (!NT_SUCCESS(Status))
 379                   {
 380                      KeBugCheck(MEMORY_MANAGEMENT);
 381                   }
 382                }
 383             }
 384          }
 385          InterlockedExchange(&MiBalancerWork, 0);
 386       }
 387       else if (Status == STATUS_SUCCESS + 1)
 388       {
 389          /* MiBalancerTimer */
 390          ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
 391          for (i = 0; i < MC_MAXIMUM; i++)
 392          {
 393             if (MiMemoryConsumers[i].Trim != NULL)
 394             {
 395                NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
 396                if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
 397                {
 398                   if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
 399                   {
 400                      Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
 401                                    MiMinimumPagesPerRun);
 402                   }
 403                   else
 404                   {
 405                      Target = MiMinimumPagesPerRun;
 406                   }
 407                   NrFreedPages = 0;
 408                   Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 409                   if (!NT_SUCCESS(Status))
 410                   {
 411                      KeBugCheck(MEMORY_MANAGEMENT);
 412                   }
 413                }
 414             }
 415          }
 416       }
 417       else
 418       {
 419          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 420          KeBugCheck(MEMORY_MANAGEMENT);
 421       }
 422    }
 423 }
 424
 425 VOID
 426 INIT_FUNCTION
 427 NTAPI
 428 MiInitBalancerThread(VOID)
 429 {
 430    KPRIORITY Priority;
 431    NTSTATUS Status;
 432 #if !defined(__GNUC__)
 433
 434    LARGE_INTEGER dummyJunkNeeded;
 435    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 436    ;
 437 #endif
 438
 439
 440    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 441    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 442    KeSetTimerEx(&MiBalancerTimer,
 443 #if defined(__GNUC__)
 444                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 445 #else
 446                 dummyJunkNeeded,
 447 #endif
 448                 2000,         /* 2 sec */
 449                 NULL);
 450
 451    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 452                                  THREAD_ALL_ACCESS,
 453                                  NULL,
 454                                  NULL,
 455                                  &MiBalancerThreadId,
 456                                  (PKSTART_ROUTINE) MiBalancerThread,
 457                                  NULL);
 458    if (!NT_SUCCESS(Status))
 459    {
 460       KeBugCheck(MEMORY_MANAGEMENT);
 461    }
 462
 463    Priority = LOW_REALTIME_PRIORITY + 1;
 464    NtSetInformationThread(MiBalancerThreadHandle,
 465                           ThreadPriority,
 466                           &Priority,
 467                           sizeof(Priority));
 468
 469 }
 470
 471
 472 /* EOF */