reactos/ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #if defined (ALLOC_PRAGMA)
  17 #pragma alloc_text(INIT, MmInitializeBalancer)
  18 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  19 #pragma alloc_text(INIT, MiInitBalancerThread)
  20 #endif
  21
  22
  23 /* TYPES ********************************************************************/
  24 typedef struct _MM_ALLOCATION_REQUEST
  25 {
  26    PFN_NUMBER Page;
  27    LIST_ENTRY ListEntry;
  28    KEVENT Event;
  29 }
  30 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  31
  32 /* GLOBALS ******************************************************************/
  33
  34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  35 static ULONG MiMinimumAvailablePages;
  36 static ULONG MiNrTotalPages;
  37 static LIST_ENTRY AllocationListHead;
  38 static KSPIN_LOCK AllocationListLock;
  39 static ULONG MiPagesRequired = 0;
  40 static ULONG MiMinimumPagesPerRun = 10;
  41
  42 static CLIENT_ID MiBalancerThreadId;
  43 static HANDLE MiBalancerThreadHandle = NULL;
  44 static KEVENT MiBalancerEvent;
  45 static KTIMER MiBalancerTimer;
  46 static LONG MiBalancerWork = 0;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID
  51 INIT_FUNCTION
  52 NTAPI
  53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  54 {
  55    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  56    InitializeListHead(&AllocationListHead);
  57    KeInitializeSpinLock(&AllocationListLock);
  58
  59    MiNrTotalPages = NrAvailablePages;
  60
  61    /* Set up targets. */
  62    MiMinimumAvailablePages = 64;
  63     if ((NrAvailablePages + NrSystemPages) >= 8192)
  64     {
  65         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  66     }
  67     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  68     {
  69         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  70     }
  71     else
  72     {
  73         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  74     }
  75    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  76 }
  77
  78 VOID
  79 INIT_FUNCTION
  80 NTAPI
  81 MmInitializeMemoryConsumer(ULONG Consumer,
  82                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  83                                             PULONG NrFreed))
  84 {
  85    MiMemoryConsumers[Consumer].Trim = Trim;
  86 }
  87
  88 VOID
  89 NTAPI
  90 MiZeroPhysicalPage(
  91     IN PFN_NUMBER PageFrameIndex
  92 );
  93
  94 NTSTATUS
  95 NTAPI
  96 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  97 {
  98    PMM_ALLOCATION_REQUEST Request;
  99    PLIST_ENTRY Entry;
 100    KIRQL OldIrql;
 101
 102    if (Page == 0)
 103    {
 104       DPRINT1("Tried to release page zero.\n");
 105       KeBugCheck(MEMORY_MANAGEMENT);
 106    }
 107
 108    KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 109    if (MmGetReferenceCountPage(Page) == 1)
 110    {
 111       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 112       if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
 113       {
 114          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 115          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 116          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 117          MmDereferencePage(Page);
 118          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 119       }
 120       else
 121       {
 122          Entry = RemoveHeadList(&AllocationListHead);
 123          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 124          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 125          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 126          MiZeroPhysicalPage(Page);
 127          Request->Page = Page;
 128          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 129       }
 130    }
 131    else
 132    {
 133       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 134       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 135       MmDereferencePage(Page);
 136       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 137    }
 138
 139    return(STATUS_SUCCESS);
 140 }
 141
 142 VOID
 143 NTAPI
 144 MiTrimMemoryConsumer(ULONG Consumer)
 145 {
 146    LONG Target;
 147    ULONG NrFreedPages;
 148
 149    Target = max(MiMinimumPagesPerRun,
 150                 MiMemoryConsumers[Consumer].PagesUsed -
 151                 MiMemoryConsumers[Consumer].PagesTarget);
 152
 153    if (MiMemoryConsumers[Consumer].Trim != NULL)
 154    {
 155       MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 156    }
 157 }
 158
 159 NTSTATUS
 160 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 161 {
 162     PFN_NUMBER CurrentPage;
 163     PFN_NUMBER NextPage;
 164     NTSTATUS Status;
 165
 166     (*NrFreedPages) = 0;
 167
 168     CurrentPage = MmGetLRUFirstUserPage();
 169     while (CurrentPage != 0 && Target > 0)
 170     {
 171         Status = MmPageOutPhysicalAddress(CurrentPage);
 172         if (NT_SUCCESS(Status))
 173         {
 174             DPRINT("Succeeded\n");
 175             Target--;
 176             (*NrFreedPages)++;
 177         }
 178
 179         NextPage = MmGetLRUNextUserPage(CurrentPage);
 180         if (NextPage <= CurrentPage)
 181         {
 182             /* We wrapped around, so we're done */
 183             break;
 184         }
 185         CurrentPage = NextPage;
 186     }
 187
 188     return STATUS_SUCCESS;
 189 }
 190
 191 VOID
 192 NTAPI
 193 MmRebalanceMemoryConsumers(VOID)
 194 {
 195    LONG Target;
 196    ULONG i;
 197    ULONG NrFreedPages;
 198    NTSTATUS Status;
 199
 200    Target = (ULONG)(MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
 201    Target = max(Target, (LONG) MiMinimumPagesPerRun);
 202
 203    for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
 204    {
 205       if (MiMemoryConsumers[i].Trim != NULL)
 206       {
 207          Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 208          if (!NT_SUCCESS(Status))
 209          {
 210             KeBugCheck(MEMORY_MANAGEMENT);
 211          }
 212          Target = Target - NrFreedPages;
 213       }
 214    }
 215 }
 216
 217 static BOOLEAN
 218 MiIsBalancerThread(VOID)
 219 {
 220    return (MiBalancerThreadHandle != NULL) &&
 221           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 222 }
 223
 224 NTSTATUS
 225 NTAPI
 226 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 227                             PPFN_NUMBER AllocatedPage)
 228 {
 229    ULONG OldUsed;
 230    PFN_NUMBER Page;
 231    KIRQL OldIrql;
 232
 233    /*
 234     * Make sure we don't exceed our individual target.
 235     */
 236    OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 237    if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
 238          !MiIsBalancerThread())
 239    {
 240       if (!CanWait)
 241       {
 242          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 243          return(STATUS_NO_MEMORY);
 244       }
 245       MiTrimMemoryConsumer(Consumer);
 246    }
 247
 248    /*
 249     * Allocate always memory for the non paged pool and for the pager thread.
 250     */
 251    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 252    {
 253       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 254       Page = MmAllocPage(Consumer);
 255       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 256       if (Page == 0)
 257       {
 258          KeBugCheck(NO_PAGES_AVAILABLE);
 259       }
 260       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 261       *AllocatedPage = Page;
 262       if (MmAvailablePages <= MiMinimumAvailablePages &&
 263           MiBalancerThreadHandle != NULL &&
 264           !MiIsBalancerThread())
 265       {
 266          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 267       }
 268       return(STATUS_SUCCESS);
 269    }
 270
 271    /*
 272     * Make sure we don't exceed global targets.
 273     */
 274    if (MmAvailablePages <= MiMinimumAvailablePages)
 275    {
 276       MM_ALLOCATION_REQUEST Request;
 277
 278       if (!CanWait)
 279       {
 280          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 281          return(STATUS_NO_MEMORY);
 282       }
 283
 284       /* Insert an allocation request. */
 285       Request.Page = 0;
 286
 287       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 288       (void)InterlockedIncrementUL(&MiPagesRequired);
 289
 290       KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 291
 292       if (MiBalancerThreadHandle != NULL)
 293       {
 294          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 295       }
 296       InsertTailList(&AllocationListHead, &Request.ListEntry);
 297       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 298
 299       KeWaitForSingleObject(&Request.Event,
 300                             0,
 301                             KernelMode,
 302                             FALSE,
 303                             NULL);
 304
 305       Page = Request.Page;
 306       if (Page == 0)
 307       {
 308          KeBugCheck(NO_PAGES_AVAILABLE);
 309       }
 310
 311       if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 312       *AllocatedPage = Page;
 313       (void)InterlockedDecrementUL(&MiPagesRequired);
 314
 315       if (MmAvailablePages <= MiMinimumAvailablePages &&
 316           MiBalancerThreadHandle != NULL &&
 317           !MiIsBalancerThread())
 318       {
 319           KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 320       }
 321
 322       return(STATUS_SUCCESS);
 323    }
 324
 325    /*
 326     * Actually allocate the page.
 327     */
 328    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 329    Page = MmAllocPage(Consumer);
 330    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 331    if (Page == 0)
 332    {
 333       KeBugCheck(NO_PAGES_AVAILABLE);
 334    }
 335    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 336    *AllocatedPage = Page;
 337
 338    if (MmAvailablePages <= MiMinimumAvailablePages &&
 339        MiBalancerThreadHandle != NULL &&
 340        !MiIsBalancerThread())
 341    {
 342        KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 343    }
 344
 345    return(STATUS_SUCCESS);
 346 }
 347
 348 VOID NTAPI
 349 MiBalancerThread(PVOID Unused)
 350 {
 351    PVOID WaitObjects[2];
 352    NTSTATUS Status;
 353    ULONG i;
 354    ULONG NrFreedPages;
 355    ULONG NrPagesUsed;
 356    ULONG Target;
 357    BOOLEAN ShouldRun;
 358
 359
 360    WaitObjects[0] = &MiBalancerEvent;
 361    WaitObjects[1] = &MiBalancerTimer;
 362
 363    while (1)
 364    {
 365       Status = KeWaitForMultipleObjects(2,
 366                                         WaitObjects,
 367                                         WaitAny,
 368                                         Executive,
 369                                         KernelMode,
 370                                         FALSE,
 371                                         NULL,
 372                                         NULL);
 373
 374       if (Status == STATUS_SUCCESS)
 375       {
 376          /* MiBalancerEvent */
 377          while (MmAvailablePages < MiMinimumAvailablePages + 5)
 378          {
 379             for (i = 0; i < MC_MAXIMUM; i++)
 380             {
 381                if (MiMemoryConsumers[i].Trim != NULL)
 382                {
 383                   NrFreedPages = 0;
 384                   Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
 385                   if (!NT_SUCCESS(Status))
 386                   {
 387                      KeBugCheck(MEMORY_MANAGEMENT);
 388                   }
 389                }
 390             }
 391          }
 392          InterlockedExchange(&MiBalancerWork, 0);
 393       }
 394       else if (Status == STATUS_SUCCESS + 1)
 395       {
 396          /* MiBalancerTimer */
 397          ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
 398          for (i = 0; i < MC_MAXIMUM; i++)
 399          {
 400             if (MiMemoryConsumers[i].Trim != NULL)
 401             {
 402                NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
 403                if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
 404                {
 405                   if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
 406                   {
 407                      Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
 408                                    MiMinimumPagesPerRun);
 409                   }
 410                   else
 411                   {
 412                      Target = MiMinimumPagesPerRun;
 413                   }
 414                   NrFreedPages = 0;
 415                   Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 416                   if (!NT_SUCCESS(Status))
 417                   {
 418                      KeBugCheck(MEMORY_MANAGEMENT);
 419                   }
 420                }
 421             }
 422          }
 423       }
 424       else
 425       {
 426          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 427          KeBugCheck(MEMORY_MANAGEMENT);
 428       }
 429    }
 430 }
 431
 432 VOID
 433 INIT_FUNCTION
 434 NTAPI
 435 MiInitBalancerThread(VOID)
 436 {
 437    KPRIORITY Priority;
 438    NTSTATUS Status;
 439 #if !defined(__GNUC__)
 440
 441    LARGE_INTEGER dummyJunkNeeded;
 442    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 443    ;
 444 #endif
 445
 446
 447    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 448    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 449    KeSetTimerEx(&MiBalancerTimer,
 450 #if defined(__GNUC__)
 451                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 452 #else
 453                 dummyJunkNeeded,
 454 #endif
 455                 2000,         /* 2 sec */
 456                 NULL);
 457
 458    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 459                                  THREAD_ALL_ACCESS,
 460                                  NULL,
 461                                  NULL,
 462                                  &MiBalancerThreadId,
 463                                  (PKSTART_ROUTINE) MiBalancerThread,
 464                                  NULL);
 465    if (!NT_SUCCESS(Status))
 466    {
 467       KeBugCheck(MEMORY_MANAGEMENT);
 468    }
 469
 470    Priority = LOW_REALTIME_PRIORITY + 1;
 471    NtSetInformationThread(MiBalancerThreadHandle,
 472                           ThreadPriority,
 473                           &Priority,
 474                           sizeof(Priority));
 475
 476 }
 477
 478
 479 /* EOF */