reactos/ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Cameron Gutman (cameron.gutman@reactos.org)
   9  */
  10
  11 /* INCLUDES *****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #include "ARM3/miarm.h"
  18
  19 #if defined (ALLOC_PRAGMA)
  20 #pragma alloc_text(INIT, MmInitializeBalancer)
  21 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
  22 #pragma alloc_text(INIT, MiInitBalancerThread)
  23 #endif
  24
  25
  26 /* TYPES ********************************************************************/
  27 typedef struct _MM_ALLOCATION_REQUEST
  28 {
  29     PFN_NUMBER Page;
  30     LIST_ENTRY ListEntry;
  31     KEVENT Event;
  32 }
  33 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  34 /* GLOBALS ******************************************************************/
  35
  36 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  37 static ULONG MiMinimumAvailablePages;
  38 static ULONG MiNrTotalPages;
  39 static LIST_ENTRY AllocationListHead;
  40 static KSPIN_LOCK AllocationListLock;
  41 static ULONG MiMinimumPagesPerRun;
  42
  43 static CLIENT_ID MiBalancerThreadId;
  44 static HANDLE MiBalancerThreadHandle = NULL;
  45 static KEVENT MiBalancerEvent;
  46 static KTIMER MiBalancerTimer;
  47
  48 /* FUNCTIONS ****************************************************************/
  49
  50 VOID
  51 INIT_FUNCTION
  52 NTAPI
  53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  54 {
  55     memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  56     InitializeListHead(&AllocationListHead);
  57     KeInitializeSpinLock(&AllocationListLock);
  58
  59     MiNrTotalPages = NrAvailablePages;
  60
  61     /* Set up targets. */
  62     MiMinimumAvailablePages = 128;
  63     MiMinimumPagesPerRun = 256;
  64     if ((NrAvailablePages + NrSystemPages) >= 8192)
  65     {
  66         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
  67     }
  68     else if ((NrAvailablePages + NrSystemPages) >= 4096)
  69     {
  70         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
  71     }
  72     else
  73     {
  74         MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
  75     }
  76     MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
  77 }
  78
  79 VOID
  80 INIT_FUNCTION
  81 NTAPI
  82 MmInitializeMemoryConsumer(
  83     ULONG Consumer,
  84     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
  85 {
  86     MiMemoryConsumers[Consumer].Trim = Trim;
  87 }
  88
  89 VOID
  90 NTAPI
  91 MiZeroPhysicalPage(
  92     IN PFN_NUMBER PageFrameIndex
  93 );
  94
  95 NTSTATUS
  96 NTAPI
  97 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  98 {
  99     PMM_ALLOCATION_REQUEST Request;
 100     PLIST_ENTRY Entry;
 101
 102     if (Page == 0)
 103     {
 104         DPRINT1("Tried to release page zero.\n");
 105         KeBugCheck(MEMORY_MANAGEMENT);
 106     }
 107
 108     if (MmGetReferenceCountPage(Page) == 1)
 109     {
 110         if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
 111         (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 112         if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL)
 113         {
 114             MmDereferencePage(Page);
 115         }
 116         else
 117         {
 118             Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 119             MiZeroPhysicalPage(Page);
 120             Request->Page = Page;
 121             KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 122         }
 123     }
 124     else
 125     {
 126         MmDereferencePage(Page);
 127     }
 128
 129     return(STATUS_SUCCESS);
 130 }
 131
 132 ULONG
 133 NTAPI
 134 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
 135 {
 136     ULONG Target = InitialTarget;
 137     ULONG NrFreedPages = 0;
 138     NTSTATUS Status;
 139
 140     /* Make sure we can trim this consumer */
 141     if (!MiMemoryConsumers[Consumer].Trim)
 142     {
 143         /* Return the unmodified initial target */
 144         return InitialTarget;
 145     }
 146
 147     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
 148     {
 149         /* Consumer page limit exceeded */
 150         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
 151     }
 152     if (MmAvailablePages < MiMinimumAvailablePages)
 153     {
 154         /* Global page limit exceeded */
 155         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
 156     }
 157
 158     if (Target)
 159     {
 160         if (!InitialTarget)
 161         {
 162             /* If there was no initial target,
 163              * swap at least MiMinimumPagesPerRun */
 164             Target = max(Target, MiMinimumPagesPerRun);
 165         }
 166
 167         /* Now swap the pages out */
 168         Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
 169
 170         DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
 171
 172         if (!NT_SUCCESS(Status))
 173         {
 174             KeBugCheck(MEMORY_MANAGEMENT);
 175         }
 176
 177         /* Update the target */
 178         if (NrFreedPages < Target)
 179             Target -= NrFreedPages;
 180         else
 181             Target = 0;
 182
 183         /* Return the remaining pages needed to meet the target */
 184         return Target;
 185     }
 186     else
 187     {
 188         /* Initial target is zero and we don't have anything else to add */
 189         return 0;
 190     }
 191 }
 192
 193 NTSTATUS
 194 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 195 {
 196     PFN_NUMBER CurrentPage;
 197     PFN_NUMBER NextPage;
 198     NTSTATUS Status;
 199
 200     (*NrFreedPages) = 0;
 201
 202     CurrentPage = MmGetLRUFirstUserPage();
 203     while (CurrentPage != 0 && Target > 0)
 204     {
 205         Status = MmPageOutPhysicalAddress(CurrentPage);
 206         if (NT_SUCCESS(Status))
 207         {
 208             DPRINT("Succeeded\n");
 209             Target--;
 210             (*NrFreedPages)++;
 211         }
 212
 213         NextPage = MmGetLRUNextUserPage(CurrentPage);
 214         if (NextPage <= CurrentPage)
 215         {
 216             /* We wrapped around, so we're done */
 217             break;
 218         }
 219         CurrentPage = NextPage;
 220     }
 221
 222     return STATUS_SUCCESS;
 223 }
 224
 225 static BOOLEAN
 226 MiIsBalancerThread(VOID)
 227 {
 228     return (MiBalancerThreadHandle != NULL) &&
 229            (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 230 }
 231
 232 VOID
 233 NTAPI
 234 MiDeletePte(IN PMMPTE PointerPte,
 235             IN PVOID VirtualAddress,
 236             IN PEPROCESS CurrentProcess,
 237             IN PMMPTE PrototypePte);
 238
 239 VOID
 240 NTAPI
 241 MmRebalanceMemoryConsumers(VOID)
 242 {
 243     if (MiBalancerThreadHandle != NULL &&
 244         !MiIsBalancerThread())
 245     {
 246         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 247     }
 248 }
 249
 250 NTSTATUS
 251 NTAPI
 252 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 253                             PPFN_NUMBER AllocatedPage)
 254 {
 255     ULONG PagesUsed;
 256     PFN_NUMBER Page;
 257
 258     /*
 259      * Make sure we don't exceed our individual target.
 260      */
 261     PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 262     if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
 263             !MiIsBalancerThread())
 264     {
 265         MmRebalanceMemoryConsumers();
 266     }
 267
 268     /*
 269      * Allocate always memory for the non paged pool and for the pager thread.
 270      */
 271     if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
 272     {
 273         Page = MmAllocPage(Consumer);
 274         if (Page == 0)
 275         {
 276             KeBugCheck(NO_PAGES_AVAILABLE);
 277         }
 278         if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 279         *AllocatedPage = Page;
 280         if (MmAvailablePages < MiMinimumAvailablePages)
 281             MmRebalanceMemoryConsumers();
 282         return(STATUS_SUCCESS);
 283     }
 284
 285     /*
 286      * Make sure we don't exceed global targets.
 287      */
 288     if (MmAvailablePages < MiMinimumAvailablePages)
 289     {
 290         MM_ALLOCATION_REQUEST Request;
 291
 292         if (!CanWait)
 293         {
 294             (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 295             MmRebalanceMemoryConsumers();
 296             return(STATUS_NO_MEMORY);
 297         }
 298
 299         /* Insert an allocation request. */
 300         Request.Page = 0;
 301         KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
 302
 303         ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
 304         MmRebalanceMemoryConsumers();
 305
 306         KeWaitForSingleObject(&Request.Event,
 307                               0,
 308                               KernelMode,
 309                               FALSE,
 310                               NULL);
 311
 312         Page = Request.Page;
 313         if (Page == 0)
 314         {
 315             KeBugCheck(NO_PAGES_AVAILABLE);
 316         }
 317
 318         if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 319         *AllocatedPage = Page;
 320
 321         if (MmAvailablePages < MiMinimumAvailablePages)
 322         {
 323             MmRebalanceMemoryConsumers();
 324         }
 325
 326         return(STATUS_SUCCESS);
 327     }
 328
 329     /*
 330      * Actually allocate the page.
 331      */
 332     Page = MmAllocPage(Consumer);
 333     if (Page == 0)
 334     {
 335         KeBugCheck(NO_PAGES_AVAILABLE);
 336     }
 337     if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
 338     *AllocatedPage = Page;
 339
 340     if (MmAvailablePages < MiMinimumAvailablePages)
 341     {
 342         MmRebalanceMemoryConsumers();
 343     }
 344
 345     return(STATUS_SUCCESS);
 346 }
 347
 348
 349 VOID NTAPI
 350 MiBalancerThread(PVOID Unused)
 351 {
 352     PVOID WaitObjects[2];
 353     NTSTATUS Status;
 354     ULONG i;
 355
 356     WaitObjects[0] = &MiBalancerEvent;
 357     WaitObjects[1] = &MiBalancerTimer;
 358
 359     while (1)
 360     {
 361         Status = KeWaitForMultipleObjects(2,
 362                                           WaitObjects,
 363                                           WaitAny,
 364                                           Executive,
 365                                           KernelMode,
 366                                           FALSE,
 367                                           NULL,
 368                                           NULL);
 369
 370         if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
 371         {
 372             ULONG InitialTarget = 0;
 373
 374 #if (_MI_PAGING_LEVELS == 2)
 375             if (!MiIsBalancerThread())
 376             {
 377                 /* Clean up the unused PDEs */
 378                 ULONG_PTR Address;
 379                 PEPROCESS Process = PsGetCurrentProcess();
 380
 381                 /* Acquire PFN lock */
 382                 KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
 383                 PMMPDE pointerPde;
 384                 for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
 385                         Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
 386                         Address += (PAGE_SIZE * PTE_COUNT))
 387                 {
 388                     if (MiQueryPageTableReferences((PVOID)Address) == 0)
 389                     {
 390                         pointerPde = MiAddressToPde(Address);
 391                         if (pointerPde->u.Hard.Valid)
 392                             MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
 393                         ASSERT(pointerPde->u.Hard.Valid == 0);
 394                     }
 395                 }
 396                 /* Release lock */
 397                 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
 398             }
 399 #endif
 400             do
 401             {
 402                 ULONG OldTarget = InitialTarget;
 403
 404                 /* Trim each consumer */
 405                 for (i = 0; i < MC_MAXIMUM; i++)
 406                 {
 407                     InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
 408                 }
 409
 410                 /* No pages left to swap! */
 411                 if (InitialTarget != 0 &&
 412                         InitialTarget == OldTarget)
 413                 {
 414                     /* Game over */
 415                     KeBugCheck(NO_PAGES_AVAILABLE);
 416                 }
 417             }
 418             while (InitialTarget != 0);
 419         }
 420         else
 421         {
 422             DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 423             KeBugCheck(MEMORY_MANAGEMENT);
 424         }
 425     }
 426 }
 427
 428 VOID
 429 INIT_FUNCTION
 430 NTAPI
 431 MiInitBalancerThread(VOID)
 432 {
 433     KPRIORITY Priority;
 434     NTSTATUS Status;
 435 #if !defined(__GNUC__)
 436
 437     LARGE_INTEGER dummyJunkNeeded;
 438     dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
 439     ;
 440 #endif
 441
 442
 443     KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 444     KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 445     KeSetTimerEx(&MiBalancerTimer,
 446 #if defined(__GNUC__)
 447                  (LARGE_INTEGER)(LONGLONG)-20000000LL,     /* 2 sec */
 448 #else
 449                  dummyJunkNeeded,
 450 #endif
 451                  2000,         /* 2 sec */
 452                  NULL);
 453
 454     Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 455                                   THREAD_ALL_ACCESS,
 456                                   NULL,
 457                                   NULL,
 458                                   &MiBalancerThreadId,
 459                                   MiBalancerThread,
 460                                   NULL);
 461     if (!NT_SUCCESS(Status))
 462     {
 463         KeBugCheck(MEMORY_MANAGEMENT);
 464     }
 465
 466     Priority = LOW_REALTIME_PRIORITY + 1;
 467     NtSetInformationThread(MiBalancerThreadHandle,
 468                            ThreadPriority,
 469                            &Priority,
 470                            sizeof(Priority));
 471
 472 }
 473
 474
 475 /* EOF */