ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Cameron Gutman (cameron.gutman@reactos.org)
   9  */
  10
  11 /* INCLUDES *****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #include "ARM3/miarm.h"
  18
  19 /* TYPES ********************************************************************/
  20 typedef struct _MM_ALLOCATION_REQUEST
  21 {
  22     PFN_NUMBER Page;
  23     LIST_ENTRY ListEntry;
  24     KEVENT Event;
  25 }
  26 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  27 /* GLOBALS ******************************************************************/
  28
  29 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  30 static ULONG MiMinimumAvailablePages;
  31 static LIST_ENTRY AllocationListHead;
  32 static KSPIN_LOCK AllocationListLock;
  33 static ULONG MiMinimumPagesPerRun;
  34
  35 static CLIENT_ID MiBalancerThreadId;
  36 static HANDLE MiBalancerThreadHandle = NULL;
  37 static KEVENT MiBalancerEvent;
  38 static KTIMER MiBalancerTimer;
  39
  40 /* FUNCTIONS ****************************************************************/
  41
  42 CODE_SEG("INIT")
  43 VOID
  44 NTAPI
  45 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  46 {
  47     memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  48     InitializeListHead(&AllocationListHead);
  49     KeInitializeSpinLock(&AllocationListLock);
  50
  51     /* Set up targets. */
  52     MiMinimumAvailablePages = 256;
  53     MiMinimumPagesPerRun = 256;
  54     MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2;
  55 }
  56
  57 CODE_SEG("INIT")
  58 VOID
  59 NTAPI
  60 MmInitializeMemoryConsumer(
  61     ULONG Consumer,
  62     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
  63 {
  64     MiMemoryConsumers[Consumer].Trim = Trim;
  65 }
  66
  67 VOID
  68 NTAPI
  69 MiZeroPhysicalPage(
  70     IN PFN_NUMBER PageFrameIndex
  71 );
  72
  73 NTSTATUS
  74 NTAPI
  75 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  76 {
  77     if (Page == 0)
  78     {
  79         DPRINT1("Tried to release page zero.\n");
  80         KeBugCheck(MEMORY_MANAGEMENT);
  81     }
  82
  83     (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
  84
  85     MmDereferencePage(Page);
  86
  87     return(STATUS_SUCCESS);
  88 }
  89
  90 ULONG
  91 NTAPI
  92 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
  93 {
  94     ULONG Target = InitialTarget;
  95     ULONG NrFreedPages = 0;
  96     NTSTATUS Status;
  97
  98     /* Make sure we can trim this consumer */
  99     if (!MiMemoryConsumers[Consumer].Trim)
 100     {
 101         /* Return the unmodified initial target */
 102         return InitialTarget;
 103     }
 104
 105     if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
 106     {
 107         /* Consumer page limit exceeded */
 108         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
 109     }
 110     if (MmAvailablePages < MiMinimumAvailablePages)
 111     {
 112         /* Global page limit exceeded */
 113         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
 114     }
 115
 116     /* Don't be too greedy in one run */
 117     Target = min(Target, 256);
 118
 119     if (Target)
 120     {
 121         /* Now swap the pages out */
 122         Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages);
 123
 124         DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
 125
 126         if (!NT_SUCCESS(Status))
 127         {
 128             KeBugCheck(MEMORY_MANAGEMENT);
 129         }
 130     }
 131
 132     /* Return the page count needed to be freed to meet the initial target */
 133     return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0;
 134 }
 135
 136 NTSTATUS
 137 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 138 {
 139     PFN_NUMBER CurrentPage;
 140     NTSTATUS Status;
 141
 142     (*NrFreedPages) = 0;
 143
 144     DPRINT1("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!");
 145
 146     CurrentPage = MmGetLRUFirstUserPage();
 147     while (CurrentPage != 0 && Target > 0)
 148     {
 149         if (Priority)
 150         {
 151             Status = MmPageOutPhysicalAddress(CurrentPage);
 152             if (NT_SUCCESS(Status))
 153             {
 154                 DPRINT("Succeeded\n");
 155                 Target--;
 156                 (*NrFreedPages)++;
 157             }
 158         }
 159         else
 160         {
 161             /* When not paging-out agressively, just reset the accessed bit */
 162             PEPROCESS Process = NULL;
 163             PVOID Address = NULL;
 164             BOOLEAN Accessed = FALSE;
 165
 166             /*
 167              * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space.
 168              * So we must use circonvoluted loops.
 169              * Well...
 170              */
 171             while (TRUE)
 172             {
 173                 KAPC_STATE ApcState;
 174                 KIRQL OldIrql = MiAcquirePfnLock();
 175                 PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage);
 176                 while (Entry)
 177                 {
 178                     if (RMAP_IS_SEGMENT(Entry->Address))
 179                     {
 180                         Entry = Entry->Next;
 181                         continue;
 182                     }
 183
 184                     /* Check that we didn't treat this entry before */
 185                     if (Entry->Address < Address)
 186                     {
 187                         Entry = Entry->Next;
 188                         continue;
 189                     }
 190
 191                     if ((Entry->Address == Address) && (Entry->Process <= Process))
 192                     {
 193                         Entry = Entry->Next;
 194                         continue;
 195                     }
 196
 197                     break;
 198                 }
 199
 200                 if (!Entry)
 201                 {
 202                     MiReleasePfnLock(OldIrql);
 203                     break;
 204                 }
 205
 206                 Process = Entry->Process;
 207                 Address = Entry->Address;
 208
 209                 MiReleasePfnLock(OldIrql);
 210
 211                 KeStackAttachProcess(&Process->Pcb, &ApcState);
 212
 213                 MmLockAddressSpace(&Process->Vm);
 214
 215                 /* Be sure this is still valid. */
 216                 PMMPTE Pte = MiAddressToPte(Address);
 217                 if (Pte->u.Hard.Valid)
 218                 {
 219                     Accessed = Accessed || Pte->u.Hard.Accessed;
 220                     Pte->u.Hard.Accessed = 0;
 221
 222                     /* There is no need to invalidate, the balancer thread is never on a user process */
 223                     //KeInvalidateTlbEntry(Address);
 224                 }
 225
 226                 MmUnlockAddressSpace(&Process->Vm);
 227
 228                 KeUnstackDetachProcess(&ApcState);
 229             }
 230
 231             if (!Accessed)
 232             {
 233                 /* Nobody accessed this page since the last time we check. Time to clean up */
 234
 235                 Status = MmPageOutPhysicalAddress(CurrentPage);
 236                 // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No");
 237                 (void)Status;
 238             }
 239
 240             /* Done for this page. */
 241             Target--;
 242         }
 243
 244         CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
 245     }
 246
 247     if (CurrentPage)
 248     {
 249         KIRQL OldIrql = MiAcquirePfnLock();
 250         MmDereferencePage(CurrentPage);
 251         MiReleasePfnLock(OldIrql);
 252     }
 253
 254     return STATUS_SUCCESS;
 255 }
 256
 257 static BOOLEAN
 258 MiIsBalancerThread(VOID)
 259 {
 260     return (MiBalancerThreadHandle != NULL) &&
 261            (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 262 }
 263
 264 VOID
 265 NTAPI
 266 MmRebalanceMemoryConsumers(VOID)
 267 {
 268     if (MiBalancerThreadHandle != NULL &&
 269         !MiIsBalancerThread())
 270     {
 271         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 272     }
 273 }
 274
 275 NTSTATUS
 276 NTAPI
 277 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 278                             PPFN_NUMBER AllocatedPage)
 279 {
 280     PFN_NUMBER Page;
 281
 282     /* Update the target */
 283     InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 284
 285     /*
 286      * Actually allocate the page.
 287      */
 288     Page = MmAllocPage(Consumer);
 289     if (Page == 0)
 290     {
 291         KeBugCheck(NO_PAGES_AVAILABLE);
 292     }
 293     *AllocatedPage = Page;
 294
 295     return(STATUS_SUCCESS);
 296 }
 297
 298
 299 VOID NTAPI
 300 MiBalancerThread(PVOID Unused)
 301 {
 302     PVOID WaitObjects[2];
 303     NTSTATUS Status;
 304     ULONG i;
 305
 306     WaitObjects[0] = &MiBalancerEvent;
 307     WaitObjects[1] = &MiBalancerTimer;
 308
 309     while (1)
 310     {
 311         Status = KeWaitForMultipleObjects(2,
 312                                           WaitObjects,
 313                                           WaitAny,
 314                                           Executive,
 315                                           KernelMode,
 316                                           FALSE,
 317                                           NULL,
 318                                           NULL);
 319
 320         if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
 321         {
 322             ULONG InitialTarget = 0;
 323
 324 #if (_MI_PAGING_LEVELS == 2)
 325             if (!MiIsBalancerThread())
 326             {
 327                 /* Clean up the unused PDEs */
 328                 ULONG_PTR Address;
 329                 PEPROCESS Process = PsGetCurrentProcess();
 330
 331                 /* Acquire PFN lock */
 332                 KIRQL OldIrql = MiAcquirePfnLock();
 333                 PMMPDE pointerPde;
 334                 for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
 335                      Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
 336                      Address += PTE_PER_PAGE * PAGE_SIZE)
 337                 {
 338                     if (MiQueryPageTableReferences((PVOID)Address) == 0)
 339                     {
 340                         pointerPde = MiAddressToPde(Address);
 341                         if (pointerPde->u.Hard.Valid)
 342                             MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
 343                         ASSERT(pointerPde->u.Hard.Valid == 0);
 344                     }
 345                 }
 346                 /* Release lock */
 347                 MiReleasePfnLock(OldIrql);
 348             }
 349 #endif
 350             do
 351             {
 352                 ULONG OldTarget = InitialTarget;
 353
 354                 /* Trim each consumer */
 355                 for (i = 0; i < MC_MAXIMUM; i++)
 356                 {
 357                     InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
 358                 }
 359
 360                 /* No pages left to swap! */
 361                 if (InitialTarget != 0 &&
 362                         InitialTarget == OldTarget)
 363                 {
 364                     /* Game over */
 365                     KeBugCheck(NO_PAGES_AVAILABLE);
 366                 }
 367             }
 368             while (InitialTarget != 0);
 369         }
 370         else
 371         {
 372             DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 373             KeBugCheck(MEMORY_MANAGEMENT);
 374         }
 375     }
 376 }
 377
 378 BOOLEAN MmRosNotifyAvailablePage(PFN_NUMBER Page)
 379 {
 380     PLIST_ENTRY Entry;
 381     PMM_ALLOCATION_REQUEST Request;
 382     PMMPFN Pfn1;
 383
 384     /* Make sure the PFN lock is held */
 385     MI_ASSERT_PFN_LOCK_HELD();
 386
 387     if (!MiMinimumAvailablePages)
 388     {
 389         /* Dirty way to know if we were initialized. */
 390         return FALSE;
 391     }
 392
 393     Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock);
 394     if (!Entry)
 395         return FALSE;
 396
 397     Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 398     MiZeroPhysicalPage(Page);
 399     Request->Page = Page;
 400
 401     Pfn1 = MiGetPfnEntry(Page);
 402     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 403     Pfn1->u3.e2.ReferenceCount = 1;
 404     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 405
 406     /* This marks the PFN as a ReactOS PFN */
 407     Pfn1->u4.AweAllocation = TRUE;
 408
 409     /* Allocate the extra ReactOS Data and zero it out */
 410     Pfn1->u1.SwapEntry = 0;
 411     Pfn1->RmapListHead = NULL;
 412
 413     KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 414
 415     return TRUE;
 416 }
 417
 418 CODE_SEG("INIT")
 419 VOID
 420 NTAPI
 421 MiInitBalancerThread(VOID)
 422 {
 423     KPRIORITY Priority;
 424     NTSTATUS Status;
 425     LARGE_INTEGER Timeout;
 426
 427     KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 428     KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 429
 430     Timeout.QuadPart = -20000000; /* 2 sec */
 431     KeSetTimerEx(&MiBalancerTimer,
 432                  Timeout,
 433                  2000,         /* 2 sec */
 434                  NULL);
 435
 436     Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 437                                   THREAD_ALL_ACCESS,
 438                                   NULL,
 439                                   NULL,
 440                                   &MiBalancerThreadId,
 441                                   MiBalancerThread,
 442                                   NULL);
 443     if (!NT_SUCCESS(Status))
 444     {
 445         KeBugCheck(MEMORY_MANAGEMENT);
 446     }
 447
 448     Priority = LOW_REALTIME_PRIORITY + 1;
 449     NtSetInformationThread(MiBalancerThreadHandle,
 450                            ThreadPriority,
 451                            &Priority,
 452                            sizeof(Priority));
 453
 454 }
 455
 456
 457 /* EOF */