reactos/ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Cameron Gutman (cameron.gutman@reactos.org)
   9  */
  10
  11 /* INCLUDES *****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #include "ARM3/miarm.h"
  18
  19 #if defined (ALLOC_PRAGMA)
  20 #pragma alloc_text(INIT, MmInitializeBalancer)
  21 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  22 #pragma alloc_text(INIT, MiInitBalancerThread)
  23 #endif
  24
  25
  26 /* TYPES ********************************************************************/
  27 typedef struct _MM_ALLOCATION_REQUEST
  28 {
  29     PFN_NUMBER Page;
  30     LIST_ENTRY ListEntry;
  31     KEVENT Event;
  32 }
  33 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  34 /* GLOBALS ******************************************************************/
  35
  36 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  37 static ULONG MiMinimumAvailablePages;
  38 static ULONG MiNrTotalPages;
  39 static LIST_ENTRY AllocationListHead;
  40 static KSPIN_LOCK AllocationListLock;
  41 static ULONG MiMinimumPagesPerRun;
  42
  43 static CLIENT_ID MiBalancerThreadId;
  44 static HANDLE MiBalancerThreadHandle = NULL;
  45 static KEVENT MiBalancerEvent;
  46 static KTIMER MiBalancerTimer;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID
  51 INIT_FUNCTION
  52 NTAPI
  53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  54 {
  55     memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  56     InitializeListHead(&AllocationListHead);
  57     KeInitializeSpinLock(&AllocationListLock);
  58
  59     MiNrTotalPages = NrAvailablePages;
  60
  61     /* Set up targets. */
  62     MiMinimumAvailablePages = 128;
  63     MiMinimumPagesPerRun = 256;
  64     if ((NrAvailablePages + NrSystemPages) >= 8192)
  65     {
  66         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  67     }
  68     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  69     {
  70         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  71     }
  72     else
  73     {
  74         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  75     }
  76     MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  77 }
  78
  79 VOID
  80 INIT_FUNCTION
  81 NTAPI
  82 MmInitializeMemoryConsumer(
  83     ULONG Consumer,
  84     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
  85 {
  86     MiMemoryConsumers[Consumer].Trim = Trim;
  87 }
  88
  89 VOID
  90 NTAPI
  91 MiZeroPhysicalPage(
  92     IN PFN_NUMBER PageFrameIndex
  93 );
  94
  95 NTSTATUS
  96 NTAPI
  97 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  98 {
  99     PMM_ALLOCATION_REQUEST Request;
 100     PLIST_ENTRY Entry;
 101
 102     if (Page == 0)
 103     {
 104         DPRINT1("Tried to release page zero.\n");
 105         KeBugCheck(MEMORY_MANAGEMENT);
 106     }
 107
 108     if (MmGetReferenceCountPage(Page) == 1)
 109     {
 110         if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 111         (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 112         if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL)
 113         {
 114             MmDereferencePage(Page);
 115         }
 116         else
 117         {
 118             Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 119             MiZeroPhysicalPage(Page);
 120             Request->Page = Page;
 121             KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 122         }
 123     }
 124     else
 125     {
 126         MmDereferencePage(Page);
 127     }
 128
 129     return(STATUS_SUCCESS);
 130 }
 131
 132 ULONG
 133 NTAPI
 134 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
 135 {
 136     ULONG Target = InitialTarget;
 137     ULONG NrFreedPages = 0;
 138     NTSTATUS Status;
 139
 140     /* Make sure we can trim this consumer */
 141     if (!MiMemoryConsumers[Consumer].Trim)
 142     {
 143         /* Return the unmodified initial target */
 144         return InitialTarget;
 145     }
 146
 147     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
 148     {
 149         /* Consumer page limit exceeded */
 150         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
 151     }
 152     if (MmAvailablePages < MiMinimumAvailablePages)
 153     {
 154         /* Global page limit exceeded */
 155         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
 156     }
 157
 158     if (Target)
 159     {
 160         if (!InitialTarget)
 161         {
 162             /* If there was no initial target,
 163              * swap at least MiMinimumPagesPerRun */
 164             Target = max(Target, MiMinimumPagesPerRun);
 165         }
 166
 167         /* Now swap the pages out */
 168         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 169
 170         DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
 171
 172         if (!NT_SUCCESS(Status))
 173         {
 174             KeBugCheck(MEMORY_MANAGEMENT);
 175         }
 176
 177         /* Update the target */
 178         if (NrFreedPages < Target)
 179             Target -= NrFreedPages;
 180         else
 181             Target = 0;
 182
 183         /* Return the remaining pages needed to meet the target */
 184         return Target;
 185     }
 186     else
 187     {
 188         /* Initial target is zero and we don't have anything else to add */
 189         return 0;
 190     }
 191 }
 192
 193 NTSTATUS
 194 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 195 {
 196     PFN_NUMBER CurrentPage;
 197     PFN_NUMBER NextPage;
 198     NTSTATUS Status;
 199
 200     (*NrFreedPages) = 0;
 201
 202     CurrentPage = MmGetLRUFirstUserPage();
 203     while (CurrentPage != 0 && Target > 0)
 204     {
 205         Status = MmPageOutPhysicalAddress(CurrentPage);
 206         if (NT_SUCCESS(Status))
 207         {
 208             DPRINT("Succeeded\n");
 209             Target--;
 210             (*NrFreedPages)++;
 211         }
 212
 213         NextPage = MmGetLRUNextUserPage(CurrentPage);
 214         if (NextPage <= CurrentPage)
 215         {
 216             /* We wrapped around, so we're done */
 217             break;
 218         }
 219         CurrentPage = NextPage;
 220     }
 221
 222     return STATUS_SUCCESS;
 223 }
 224
 225 static BOOLEAN
 226 MiIsBalancerThread(VOID)
 227 {
 228     return (MiBalancerThreadHandle != NULL) &&
 229            (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 230 }
 231
 232 VOID
 233 NTAPI
 234 MmRebalanceMemoryConsumers(VOID)
 235 {
 236     if (MiBalancerThreadHandle != NULL &&
 237         !MiIsBalancerThread())
 238     {
 239         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 240     }
 241 }
 242
 243 NTSTATUS
 244 NTAPI
 245 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 246                             PPFN_NUMBER AllocatedPage)
 247 {
 248     ULONG PagesUsed;
 249     PFN_NUMBER Page;
 250
 251     /*
 252      * Make sure we don't exceed our individual target.
 253      */
 254     PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 255     if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
 256             !MiIsBalancerThread())
 257     {
 258         MmRebalanceMemoryConsumers();
 259     }
 260
 261     /*
 262      * Allocate always memory for the non paged pool and for the pager thread.
 263      */
 264     if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 265     {
 266         Page = MmAllocPage(Consumer);
 267         if (Page == 0)
 268         {
 269             KeBugCheck(NO_PAGES_AVAILABLE);
 270         }
 271         if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 272         *AllocatedPage = Page;
 273         if (MmAvailablePages < MiMinimumAvailablePages)
 274             MmRebalanceMemoryConsumers();
 275         return(STATUS_SUCCESS);
 276     }
 277
 278     /*
 279      * Make sure we don't exceed global targets.
 280      */
 281     if (MmAvailablePages < MiMinimumAvailablePages)
 282     {
 283         MM_ALLOCATION_REQUEST Request;
 284
 285         if (!CanWait)
 286         {
 287             (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 288             MmRebalanceMemoryConsumers();
 289             return(STATUS_NO_MEMORY);
 290         }
 291
 292         /* Insert an allocation request. */
 293         Request.Page = 0;
 294         KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 295
 296         ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
 297         MmRebalanceMemoryConsumers();
 298
 299         KeWaitForSingleObject(&Request.Event,
 300                               0,
 301                               KernelMode,
 302                               FALSE,
 303                               NULL);
 304
 305         Page = Request.Page;
 306         if (Page == 0)
 307         {
 308             KeBugCheck(NO_PAGES_AVAILABLE);
 309         }
 310
 311         if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 312         *AllocatedPage = Page;
 313
 314         if (MmAvailablePages < MiMinimumAvailablePages)
 315         {
 316             MmRebalanceMemoryConsumers();
 317         }
 318
 319         return(STATUS_SUCCESS);
 320     }
 321
 322     /*
 323      * Actually allocate the page.
 324      */
 325     Page = MmAllocPage(Consumer);
 326     if (Page == 0)
 327     {
 328         KeBugCheck(NO_PAGES_AVAILABLE);
 329     }
 330     if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 331     *AllocatedPage = Page;
 332
 333     if (MmAvailablePages < MiMinimumAvailablePages)
 334     {
 335         MmRebalanceMemoryConsumers();
 336     }
 337
 338     return(STATUS_SUCCESS);
 339 }
 340
 341
 342 VOID NTAPI
 343 MiBalancerThread(PVOID Unused)
 344 {
 345     PVOID WaitObjects[2];
 346     NTSTATUS Status;
 347     ULONG i;
 348
 349     WaitObjects[0] = &MiBalancerEvent;
 350     WaitObjects[1] = &MiBalancerTimer;
 351
 352     while (1)
 353     {
 354         Status = KeWaitForMultipleObjects(2,
 355                                           WaitObjects,
 356                                           WaitAny,
 357                                           Executive,
 358                                           KernelMode,
 359                                           FALSE,
 360                                           NULL,
 361                                           NULL);
 362
 363         if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
 364         {
 365             ULONG InitialTarget = 0;
 366
 367 #if (_MI_PAGING_LEVELS == 2)
 368             if (!MiIsBalancerThread())
 369             {
 370                 /* Clean up the unused PDEs */
 371                 ULONG_PTR Address;
 372                 PEPROCESS Process = PsGetCurrentProcess();
 373
 374                 /* Acquire PFN lock */
 375                 KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 376                 PMMPDE pointerPde;
 377                 for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
 378                         Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
 379                         Address += (PAGE_SIZE * PTE_COUNT))
 380                 {
 381                     if (MiQueryPageTableReferences((PVOID)Address) == 0)
 382                     {
 383                         pointerPde = MiAddressToPde(Address);
 384                         if (pointerPde->u.Hard.Valid)
 385                             MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
 386                         ASSERT(pointerPde->u.Hard.Valid == 0);
 387                     }
 388                 }
 389                 /* Release lock */
 390                 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 391             }
 392 #endif
 393             do
 394             {
 395                 ULONG OldTarget = InitialTarget;
 396
 397                 /* Trim each consumer */
 398                 for (i = 0; i < MC_MAXIMUM; i++)
 399                 {
 400                     InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
 401                 }
 402
 403                 /* No pages left to swap! */
 404                 if (InitialTarget != 0 &&
 405                         InitialTarget == OldTarget)
 406                 {
 407                     /* Game over */
 408                     KeBugCheck(NO_PAGES_AVAILABLE);
 409                 }
 410             }
 411             while (InitialTarget != 0);
 412         }
 413         else
 414         {
 415             DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 416             KeBugCheck(MEMORY_MANAGEMENT);
 417         }
 418     }
 419 }
 420
 421 VOID
 422 INIT_FUNCTION
 423 NTAPI
 424 MiInitBalancerThread(VOID)
 425 {
 426     KPRIORITY Priority;
 427     NTSTATUS Status;
 428 #if !defined(__GNUC__)
 429
 430     LARGE_INTEGER dummyJunkNeeded;
 431     dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 432     ;
 433 #endif
 434
 435
 436     KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 437     KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 438     KeSetTimerEx(&MiBalancerTimer,
 439 #if defined(__GNUC__)
 440                  (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 441 #else
 442                  dummyJunkNeeded,
 443 #endif
 444                  2000,         /* 2 sec */
 445                  NULL);
 446
 447     Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 448                                   THREAD_ALL_ACCESS,
 449                                   NULL,
 450                                   NULL,
 451                                   &MiBalancerThreadId,
 452                                   MiBalancerThread,
 453                                   NULL);
 454     if (!NT_SUCCESS(Status))
 455     {
 456         KeBugCheck(MEMORY_MANAGEMENT);
 457     }
 458
 459     Priority = LOW_REALTIME_PRIORITY + 1;
 460     NtSetInformationThread(MiBalancerThreadHandle,
 461                            ThreadPriority,
 462                            &Priority,
 463                            sizeof(Priority));
 464
 465 }
 466
 467
 468 /* EOF */