reactos/ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 #if defined (ALLOC_PRAGMA)
  17 #pragma alloc_text(INIT, MmInitializeBalancer)
  18 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  19 #pragma alloc_text(INIT, MiInitBalancerThread)
  20 #endif
  21
  22
  23 /* TYPES ********************************************************************/
  24 typedef struct _MM_ALLOCATION_REQUEST
  25 {
  26    PFN_NUMBER Page;
  27    LIST_ENTRY ListEntry;
  28    KEVENT Event;
  29 }
  30 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  31
  32 /* GLOBALS ******************************************************************/
  33
  34 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  35 static ULONG MiMinimumAvailablePages;
  36 static ULONG MiNrTotalPages;
  37 static LIST_ENTRY AllocationListHead;
  38 static KSPIN_LOCK AllocationListLock;
  39 static ULONG MiPagesRequired = 0;
  40 static ULONG MiMinimumPagesPerRun = 10;
  41
  42 static CLIENT_ID MiBalancerThreadId;
  43 static HANDLE MiBalancerThreadHandle = NULL;
  44 static KEVENT MiBalancerEvent;
  45 static KTIMER MiBalancerTimer;
  46 static LONG MiBalancerWork = 0;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID
  51 INIT_FUNCTION
  52 NTAPI
  53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  54 {
  55    memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  56    InitializeListHead(&AllocationListHead);
  57    KeInitializeSpinLock(&AllocationListLock);
  58
  59    MiNrTotalPages = NrAvailablePages;
  60
  61    /* Set up targets. */
  62    MiMinimumAvailablePages = 64;
  63     if ((NrAvailablePages + NrSystemPages) >= 8192)
  64     {
  65         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  66     }
  67     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  68     {
  69         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  70     }
  71     else
  72     {
  73         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  74     }
  75    MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  76 }
  77
  78 VOID
  79 INIT_FUNCTION
  80 NTAPI
  81 MmInitializeMemoryConsumer(ULONG Consumer,
  82                            NTSTATUS (*Trim)(ULONG Target, ULONG Priority,
  83                                             PULONG NrFreed))
  84 {
  85    MiMemoryConsumers[Consumer].Trim = Trim;
  86 }
  87
  88 VOID
  89 NTAPI
  90 MiZeroPhysicalPage(
  91     IN PFN_NUMBER PageFrameIndex
  92 );
  93
  94 NTSTATUS
  95 NTAPI
  96 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  97 {
  98    PMM_ALLOCATION_REQUEST Request;
  99    PLIST_ENTRY Entry;
 100    KIRQL OldIrql;
 101
 102    if (Page == 0)
 103    {
 104       DPRINT1("Tried to release page zero.\n");
 105       KeBugCheck(MEMORY_MANAGEMENT);
 106    }
 107
 108    KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 109    if (MmGetReferenceCountPage(Page) == 1)
 110    {
 111       (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 112       if (IsListEmpty(&AllocationListHead) || MmAvailablePages < MiMinimumAvailablePages)
 113       {
 114          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 115          if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 116          OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 117          MmDereferencePage(Page);
 118          KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 119       }
 120       else
 121       {
 122          Entry = RemoveHeadList(&AllocationListHead);
 123          Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 124          KeReleaseSpinLock(&AllocationListLock, OldIrql);
 125          MiZeroPhysicalPage(Page);
 126          Request->Page = Page;
 127          KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 128       }
 129    }
 130    else
 131    {
 132       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 133       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 134       MmDereferencePage(Page);
 135       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 136    }
 137
 138    return(STATUS_SUCCESS);
 139 }
 140
 141 VOID
 142 NTAPI
 143 MiTrimMemoryConsumer(ULONG Consumer)
 144 {
 145    LONG Target;
 146    ULONG NrFreedPages;
 147
 148    Target = MiMemoryConsumers[Consumer].PagesUsed -
 149             MiMemoryConsumers[Consumer].PagesTarget;
 150    if (Target < 1)
 151    {
 152       Target = 1;
 153    }
 154
 155    if (MiMemoryConsumers[Consumer].Trim != NULL)
 156    {
 157       MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 158    }
 159 }
 160
 161 NTSTATUS
 162 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 163 {
 164     PFN_NUMBER CurrentPage;
 165     PFN_NUMBER NextPage;
 166     NTSTATUS Status;
 167
 168     (*NrFreedPages) = 0;
 169
 170     CurrentPage = MmGetLRUFirstUserPage();
 171     while (CurrentPage != 0 && Target > 0)
 172     {
 173         Status = MmPageOutPhysicalAddress(CurrentPage);
 174         if (NT_SUCCESS(Status))
 175         {
 176             DPRINT("Succeeded\n");
 177             Target--;
 178             (*NrFreedPages)++;
 179         }
 180
 181         NextPage = MmGetLRUNextUserPage(CurrentPage);
 182         if (NextPage <= CurrentPage)
 183         {
 184             /* We wrapped around, so we're done */
 185             break;
 186         }
 187         CurrentPage = NextPage;
 188     }
 189
 190     return STATUS_SUCCESS;
 191 }
 192
 193 VOID
 194 NTAPI
 195 MmRebalanceMemoryConsumers(VOID)
 196 {
 197    LONG Target;
 198    ULONG i;
 199    ULONG NrFreedPages;
 200    NTSTATUS Status;
 201
 202    Target = (ULONG)(MiMinimumAvailablePages - MmAvailablePages) + MiPagesRequired;
 203    Target = max(Target, (LONG) MiMinimumPagesPerRun);
 204
 205    for (i = 0; i < MC_MAXIMUM && Target > 0; i++)
 206    {
 207       if (MiMemoryConsumers[i].Trim != NULL)
 208       {
 209          Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 210          if (!NT_SUCCESS(Status))
 211          {
 212             KeBugCheck(MEMORY_MANAGEMENT);
 213          }
 214          Target = Target - NrFreedPages;
 215       }
 216    }
 217 }
 218
 219 static BOOLEAN
 220 MiIsBalancerThread(VOID)
 221 {
 222    return (MiBalancerThreadHandle != NULL) &&
 223           (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 224 }
 225
 226 NTSTATUS
 227 NTAPI
 228 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 229                             PPFN_NUMBER AllocatedPage)
 230 {
 231    ULONG OldUsed;
 232    PFN_NUMBER Page;
 233    KIRQL OldIrql;
 234
 235    /*
 236     * Make sure we don't exceed our individual target.
 237     */
 238    OldUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 239    if (OldUsed >= (MiMemoryConsumers[Consumer].PagesTarget - 1) &&
 240          !MiIsBalancerThread())
 241    {
 242       if (!CanWait)
 243       {
 244          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 245          return(STATUS_NO_MEMORY);
 246       }
 247       MiTrimMemoryConsumer(Consumer);
 248    }
 249
 250    /*
 251     * Allocate always memory for the non paged pool and for the pager thread.
 252     */
 253    if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 254    {
 255       OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 256       Page = MmAllocPage(Consumer);
 257       KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 258       if (Page == 0)
 259       {
 260          KeBugCheck(NO_PAGES_AVAILABLE);
 261       }
 262       if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 263       *AllocatedPage = Page;
 264       if (MmAvailablePages <= MiMinimumAvailablePages &&
 265             MiBalancerThreadHandle != NULL)
 266       {
 267          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 268       }
 269       return(STATUS_SUCCESS);
 270    }
 271
 272    /*
 273     * Make sure we don't exceed global targets.
 274     */
 275    if (MmAvailablePages <= MiMinimumAvailablePages)
 276    {
 277       MM_ALLOCATION_REQUEST Request;
 278
 279       if (!CanWait)
 280       {
 281          (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 282          return(STATUS_NO_MEMORY);
 283       }
 284
 285       /* Insert an allocation request. */
 286       Request.Page = 0;
 287
 288       KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 289       (void)InterlockedIncrementUL(&MiPagesRequired);
 290
 291       KeAcquireSpinLock(&AllocationListLock, &OldIrql);
 292
 293       if (MiBalancerThreadHandle != NULL)
 294       {
 295          KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 296       }
 297       InsertTailList(&AllocationListHead, &Request.ListEntry);
 298       KeReleaseSpinLock(&AllocationListLock, OldIrql);
 299
 300       KeWaitForSingleObject(&Request.Event,
 301                             0,
 302                             KernelMode,
 303                             FALSE,
 304                             NULL);
 305
 306       Page = Request.Page;
 307       if (Page == 0)
 308       {
 309          KeBugCheck(NO_PAGES_AVAILABLE);
 310       }
 311
 312       *AllocatedPage = Page;
 313       (void)InterlockedDecrementUL(&MiPagesRequired);
 314
 315       return(STATUS_SUCCESS);
 316    }
 317
 318    /*
 319     * Actually allocate the page.
 320     */
 321    OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 322    Page = MmAllocPage(Consumer);
 323    KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 324    if (Page == 0)
 325    {
 326       KeBugCheck(NO_PAGES_AVAILABLE);
 327    }
 328    if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 329    *AllocatedPage = Page;
 330
 331    return(STATUS_SUCCESS);
 332 }
 333
 334 VOID NTAPI
 335 MiBalancerThread(PVOID Unused)
 336 {
 337    PVOID WaitObjects[2];
 338    NTSTATUS Status;
 339    ULONG i;
 340    ULONG NrFreedPages;
 341    ULONG NrPagesUsed;
 342    ULONG Target;
 343    BOOLEAN ShouldRun;
 344
 345
 346    WaitObjects[0] = &MiBalancerEvent;
 347    WaitObjects[1] = &MiBalancerTimer;
 348
 349    while (1)
 350    {
 351       Status = KeWaitForMultipleObjects(2,
 352                                         WaitObjects,
 353                                         WaitAny,
 354                                         Executive,
 355                                         KernelMode,
 356                                         FALSE,
 357                                         NULL,
 358                                         NULL);
 359
 360       if (Status == STATUS_SUCCESS)
 361       {
 362          /* MiBalancerEvent */
 363          while (MmAvailablePages < MiMinimumAvailablePages + 5)
 364          {
 365             for (i = 0; i < MC_MAXIMUM; i++)
 366             {
 367                if (MiMemoryConsumers[i].Trim != NULL)
 368                {
 369                   NrFreedPages = 0;
 370                   Status = MiMemoryConsumers[i].Trim(MiMinimumPagesPerRun, 0, &NrFreedPages);
 371                   if (!NT_SUCCESS(Status))
 372                   {
 373                      KeBugCheck(MEMORY_MANAGEMENT);
 374                   }
 375                }
 376             }
 377          }
 378          InterlockedExchange(&MiBalancerWork, 0);
 379       }
 380       else if (Status == STATUS_SUCCESS + 1)
 381       {
 382          /* MiBalancerTimer */
 383          ShouldRun = MmAvailablePages < MiMinimumAvailablePages + 5 ? TRUE : FALSE;
 384          for (i = 0; i < MC_MAXIMUM; i++)
 385          {
 386             if (MiMemoryConsumers[i].Trim != NULL)
 387             {
 388                NrPagesUsed = MiMemoryConsumers[i].PagesUsed;
 389                if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget || ShouldRun)
 390                {
 391                   if (NrPagesUsed > MiMemoryConsumers[i].PagesTarget)
 392                   {
 393                      Target = max (NrPagesUsed - MiMemoryConsumers[i].PagesTarget,
 394                                    MiMinimumPagesPerRun);
 395                   }
 396                   else
 397                   {
 398                      Target = MiMinimumPagesPerRun;
 399                   }
 400                   NrFreedPages = 0;
 401                   Status = MiMemoryConsumers[i].Trim(Target, 0, &NrFreedPages);
 402                   if (!NT_SUCCESS(Status))
 403                   {
 404                      KeBugCheck(MEMORY_MANAGEMENT);
 405                   }
 406                }
 407             }
 408          }
 409       }
 410       else
 411       {
 412          DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 413          KeBugCheck(MEMORY_MANAGEMENT);
 414       }
 415    }
 416 }
 417
 418 VOID
 419 INIT_FUNCTION
 420 NTAPI
 421 MiInitBalancerThread(VOID)
 422 {
 423    KPRIORITY Priority;
 424    NTSTATUS Status;
 425 #if !defined(__GNUC__)
 426
 427    LARGE_INTEGER dummyJunkNeeded;
 428    dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 429    ;
 430 #endif
 431
 432
 433    KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 434    KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 435    KeSetTimerEx(&MiBalancerTimer,
 436 #if defined(__GNUC__)
 437                 (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 438 #else
 439                 dummyJunkNeeded,
 440 #endif
 441                 2000,         /* 2 sec */
 442                 NULL);
 443
 444    Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 445                                  THREAD_ALL_ACCESS,
 446                                  NULL,
 447                                  NULL,
 448                                  &MiBalancerThreadId,
 449                                  (PKSTART_ROUTINE) MiBalancerThread,
 450                                  NULL);
 451    if (!NT_SUCCESS(Status))
 452    {
 453       KeBugCheck(MEMORY_MANAGEMENT);
 454    }
 455
 456    Priority = LOW_REALTIME_PRIORITY + 1;
 457    NtSetInformationThread(MiBalancerThreadHandle,
 458                           ThreadPriority,
 459                           &Priority,
 460                           sizeof(Priority));
 461
 462 }
 463
 464
 465 /* EOF */