reactos/ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #if defined (ALLOC_PRAGMA)
  17 #pragma alloc_text(INIT, MmInitializeBalancer)
  18 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  19 #pragma alloc_text(INIT, MiInitBalancerThread)
  20 #endif
  21
  22
  23 /* TYPES ********************************************************************/
  24 typedef struct _MM_ALLOCATION_REQUEST
  25 {
  26    PFN_NUMBER Page;
  27    LIST_ENTRY ListEntry;
  28    KEVENT Event;
  29 }
  30 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  31
  32 /* GLOBALS ******************************************************************/
  33
  34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  35 static ULONG MiMinimumAvailablePages;
  36 static ULONG MiNrTotalPages;
  37 static LIST_ENTRY AllocationListHead;
  38 static KSPIN_LOCK AllocationListLock;
  39 static ULONG MiPagesRequired = 0;
  40 static ULONG MiMinimumPagesPerRun = 10;
  41
  42 static CLIENT_ID MiBalancerThreadId;
  43 static HANDLE MiBalancerThreadHandle = NULL;
  44 static KEVENT MiBalancerEvent;
  45 static KTIMER MiBalancerTimer;
  46 static LONG MiBalancerWork = 0;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID
  51 INIT_FUNCTION
  52 NTAPI
  53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  54 {
  55    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  56    InitializeListHead(&AllocationListHead);
  57    KeInitializeSpinLock(&AllocationListLock);
  58
  59    MiNrTotalPages = NrAvailablePages;
  60
  61    /* Set up targets. */
  62    MiMinimumAvailablePages = 64;
  63     if ((NrAvailablePages + NrSystemPages) >= 8192)
  64     {
  65         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  66     }
  67     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  68     {
  69         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  70     }
  71     else
  72     {
  73         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  74     }
  75    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  76 }
  77
  78 VOID
  79 INIT_FUNCTION
  80 NTAPI
  81 MmInitializeMemoryConsumer(ULONG Consumer,
  82                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  83                                             PULONG NrFreed))
  84 {
  85    MiMemoryConsumers[Consumer].Trim = Trim;
  86 }
  87
  88 VOID
  89 NTAPI
  90 MiZeroPhysicalPage(
  91     IN PFN_NUMBER PageFrameIndex
  92 );
  93
  94 NTSTATUS
  95 NTAPI
  96 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  97 {
  98    PMM_ALLOCATION_REQUEST Request;
  99    PLIST_ENTRY Entry;
 100    KIRQL OldIrql;
 101
 102    if (Page == 0)
 103    {
 104       DPRINT1("Tried to release page zero.\n");
 105       KeBugCheck(MEMORY_MANAGEMENT);
 106    }
 107
 108    KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 109    if (MmGetReferenceCountPage(Page) == 1)
 110    {
 111       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 112       if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
 113       {
 114          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 115          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 116          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 117          MmDereferencePage(Page);
 118          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 119       }
 120       else
 121       {
 122          Entry = RemoveHeadList(&AllocationListHead);
 123          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 124          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 125          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 126          MiZeroPhysicalPage(Page);
 127          Request->Page = Page;
 128          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 129       }
 130    }
 131    else
 132    {
 133       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 134       if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 135       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 136       MmDereferencePage(Page);
 137       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 138    }
 139
 140    return(STATUS_SUCCESS);
 141 }
 142
 143 VOID
 144 NTAPI
 145 MiTrimMemoryConsumer(ULONG Consumer)
 146 {
 147    LONG Target;
 148    ULONG NrFreedPages;
 149
 150    Target = MiMemoryConsumers[Consumer].PagesUsed -
 151             MiMemoryConsumers[Consumer].PagesTarget;
 152    if (Target < 1)
 153    {
 154       Target = 1;
 155    }
 156
 157    if (MiMemoryConsumers[Consumer].Trim != NULL)
 158    {
 159       MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 160    }
 161 }
 162
 163 NTSTATUS
 164 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 165 {
 166     PFN_NUMBER CurrentPage;
 167     PFN_NUMBER NextPage;
 168     NTSTATUS Status;
 169
 170     (*NrFreedPages) = 0;
 171
 172     CurrentPage = MmGetLRUFirstUserPage();
 173     while (CurrentPage != 0 && Target > 0)
 174     {
 175         NextPage = MmGetLRUNextUserPage(CurrentPage);
 176
 177         Status = MmPageOutPhysicalAddress(CurrentPage);
 178         if (NT_SUCCESS(Status))
 179         {
 180             DPRINT("Succeeded\n");
 181             Target--;
 182             (*NrFreedPages)++;
 183         }
 184
 185         CurrentPage = NextPage;
 186     }
 187     return(STATUS_SUCCESS);
 188 }
 189
 190 VOID
 191 NTAPI
 192 MmRebalanceMemoryConsumers(VOID)
 193 {
 194    LONG Target;
 195    ULONG i;
 196    ULONG NrFreedPages;
 197    NTSTATUS Status;
 198
 199    Target = (MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
 200    Target = max(Target, (LONG) MiMinimumPagesPerRun);
 201
 202    for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
 203    {
 204       if (MiMemoryConsumers[i].Trim != NULL)
 205       {
 206          Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 207          if (!NT_SUCCESS(Status))
 208          {
 209             KeBugCheck(MEMORY_MANAGEMENT);
 210          }
 211          Target = Target - NrFreedPages;
 212       }
 213    }
 214 }
 215
 216 static BOOLEAN
 217 MiIsBalancerThread(VOID)
 218 {
 219    return MiBalancerThreadHandle != NULL &&
 220           PsGetCurrentThread() == MiBalancerThreadId.UniqueThread;
 221 }
 222
 223 NTSTATUS
 224 NTAPI
 225 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 226                             PPFN_NUMBER AllocatedPage)
 227 {
 228    ULONG OldUsed;
 229    PFN_NUMBER Page;
 230    KIRQL OldIrql;
 231
 232    /*
 233     * Make sure we don't exceed our individual target.
 234     */
 235    OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 236    if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
 237          !MiIsBalancerThread())
 238    {
 239       if (!CanWait)
 240       {
 241          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 242          return(STATUS_NO_MEMORY);
 243       }
 244       MiTrimMemoryConsumer(Consumer);
 245    }
 246
 247    /*
 248     * Allocate always memory for the non paged pool and for the pager thread.
 249     */
 250    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 251    {
 252       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 253       Page = MmAllocPage(Consumer);
 254       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 255       if (Page == 0)
 256       {
 257          KeBugCheck(NO_PAGES_AVAILABLE);
 258       }
 259       *AllocatedPage = Page;
 260       if (MmAvailablePages <= MiMinimumAvailablePages &&
 261             MiBalancerThreadHandle != NULL)
 262       {
 263          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 264       }
 265       return(STATUS_SUCCESS);
 266    }
 267
 268    /*
 269     * Make sure we don't exceed global targets.
 270     */
 271    if (MmAvailablePages <= MiMinimumAvailablePages)
 272    {
 273       MM_ALLOCATION_REQUEST Request;
 274
 275       if (!CanWait)
 276       {
 277          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 278          return(STATUS_NO_MEMORY);
 279       }
 280
 281       /* Insert an allocation request. */
 282       Request.Page = 0;
 283
 284       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 285       (void)InterlockedIncrementUL(&MiPagesRequired);
 286
 287       KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 288
 289       if (MiBalancerThreadHandle != NULL)
 290       {
 291          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 292       }
 293       InsertTailList(&AllocationListHead, &Request.ListEntry);
 294       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 295
 296       KeWaitForSingleObject(&Request.Event,
 297                             0,
 298                             KernelMode,
 299                             FALSE,
 300                             NULL);
 301
 302       Page = Request.Page;
 303       if (Page == 0)
 304       {
 305          KeBugCheck(NO_PAGES_AVAILABLE);
 306       }
 307       /* Update the Consumer and make the page active */
 308       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 309       *AllocatedPage = Page;
 310       (void)InterlockedDecrementUL(&MiPagesRequired);
 311       return(STATUS_SUCCESS);
 312    }
 313
 314    /*
 315     * Actually allocate the page.
 316     */
 317    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 318    Page = MmAllocPage(Consumer);
 319    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 320    if (Page == 0)
 321    {
 322       KeBugCheck(NO_PAGES_AVAILABLE);
 323    }
 324    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 325    *AllocatedPage = Page;
 326
 327    return(STATUS_SUCCESS);
 328 }
 329
 330 VOID NTAPI
 331 MiBalancerThread(PVOID Unused)
 332 {
 333    PVOID WaitObjects[2];
 334    NTSTATUS Status;
 335    ULONG i;
 336    ULONG NrFreedPages;
 337    ULONG NrPagesUsed;
 338    ULONG Target;
 339    BOOLEAN ShouldRun;
 340
 341
 342    WaitObjects[0] = &MiBalancerEvent;
 343    WaitObjects[1] = &MiBalancerTimer;
 344
 345    while (1)
 346    {
 347       Status = KeWaitForMultipleObjects(2,
 348                                         WaitObjects,
 349                                         WaitAny,
 350                                         Executive,
 351                                         KernelMode,
 352                                         FALSE,
 353                                         NULL,
 354                                         NULL);
 355
 356       if (Status == STATUS_SUCCESS)
 357       {
 358          /* MiBalancerEvent */
 359          while (MmAvailablePages < MiMinimumAvailablePages + 5)
 360          {
 361             for (i = 0; i < MC_MAXIMUM; i++)
 362             {
 363                if (MiMemoryConsumers[i].Trim != NULL)
 364                {
 365                   NrFreedPages = 0;
 366                   Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
 367                   if (!NT_SUCCESS(Status))
 368                   {
 369                      KeBugCheck(MEMORY_MANAGEMENT);
 370                   }
 371                }
 372             }
 373          }
 374          InterlockedExchange(&MiBalancerWork, 0);
 375       }
 376       else if (Status == STATUS_SUCCESS + 1)
 377       {
 378          /* MiBalancerTimer */
 379          ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
 380          for (i = 0; i < MC_MAXIMUM; i++)
 381          {
 382             if (MiMemoryConsumers[i].Trim != NULL)
 383             {
 384                NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
 385                if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
 386                {
 387                   if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
 388                   {
 389                      Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
 390                                    MiMinimumPagesPerRun);
 391                   }
 392                   else
 393                   {
 394                      Target = MiMinimumPagesPerRun;
 395                   }
 396                   NrFreedPages = 0;
 397                   Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 398                   if (!NT_SUCCESS(Status))
 399                   {
 400                      KeBugCheck(MEMORY_MANAGEMENT);
 401                   }
 402                }
 403             }
 404          }
 405       }
 406       else
 407       {
 408          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 409          KeBugCheck(MEMORY_MANAGEMENT);
 410       }
 411    }
 412 }
 413
 414 VOID
 415 INIT_FUNCTION
 416 NTAPI
 417 MiInitBalancerThread(VOID)
 418 {
 419    KPRIORITY Priority;
 420    NTSTATUS Status;
 421 #if !defined(__GNUC__)
 422
 423    LARGE_INTEGER dummyJunkNeeded;
 424    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 425    ;
 426 #endif
 427
 428
 429    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 430    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 431    KeSetTimerEx(&MiBalancerTimer,
 432 #if defined(__GNUC__)
 433                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 434 #else
 435                 dummyJunkNeeded,
 436 #endif
 437                 2000,         /* 2 sec */
 438                 NULL);
 439
 440    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 441                                  THREAD_ALL_ACCESS,
 442                                  NULL,
 443                                  NULL,
 444                                  &MiBalancerThreadId,
 445                                  (PKSTART_ROUTINE) MiBalancerThread,
 446                                  NULL);
 447    if (!NT_SUCCESS(Status))
 448    {
 449       KeBugCheck(MEMORY_MANAGEMENT);
 450    }
 451
 452    Priority = LOW_REALTIME_PRIORITY + 1;
 453    NtSetInformationThread(MiBalancerThreadHandle,
 454                           ThreadPriority,
 455                           &Priority,
 456                           sizeof(Priority));
 457
 458 }
 459
 460
 461 /* EOF */