ntoskrnl/mm/balance.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/mm/balance.c
   5  * PURPOSE:         kernel memory managment functions
   6  *
   7  * PROGRAMMERS:     David Welch (welch@cwcom.net)
   8  *                  Cameron Gutman (cameron.gutman@reactos.org)
   9  */
  10
  11 /* INCLUDES *****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 #include "ARM3/miarm.h"
  18
  19
  20 /* TYPES ********************************************************************/
  21 typedef struct _MM_ALLOCATION_REQUEST
  22 {
  23     PFN_NUMBER Page;
  24     LIST_ENTRY ListEntry;
  25     KEVENT Event;
  26 }
  27 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
  28 /* GLOBALS ******************************************************************/
  29
  30 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
  31 static ULONG MiMinimumAvailablePages;
  32 static LIST_ENTRY AllocationListHead;
  33 static KSPIN_LOCK AllocationListLock;
  34 static ULONG MiMinimumPagesPerRun;
  35
  36 static CLIENT_ID MiBalancerThreadId;
  37 static HANDLE MiBalancerThreadHandle = NULL;
  38 static KEVENT MiBalancerEvent;
  39 static KTIMER MiBalancerTimer;
  40
  41 static LONG PageOutThreadActive;
  42
  43 /* FUNCTIONS ****************************************************************/
  44
  45 CODE_SEG("INIT")
  46 VOID
  47 NTAPI
  48 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
  49 {
  50     memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
  51     InitializeListHead(&AllocationListHead);
  52     KeInitializeSpinLock(&AllocationListLock);
  53
  54     /* Set up targets. */
  55     MiMinimumAvailablePages = 256;
  56     MiMinimumPagesPerRun = 256;
  57     MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages / 2;
  58 }
  59
  60 CODE_SEG("INIT")
  61 VOID
  62 NTAPI
  63 MmInitializeMemoryConsumer(
  64     ULONG Consumer,
  65     NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
  66 {
  67     MiMemoryConsumers[Consumer].Trim = Trim;
  68 }
  69
  70 VOID
  71 NTAPI
  72 MiZeroPhysicalPage(
  73     IN PFN_NUMBER PageFrameIndex
  74 );
  75
  76 NTSTATUS
  77 NTAPI
  78 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
  79 {
  80     if (Page == 0)
  81     {
  82         DPRINT1("Tried to release page zero.\n");
  83         KeBugCheck(MEMORY_MANAGEMENT);
  84     }
  85
  86     (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
  87
  88     MmDereferencePage(Page);
  89
  90     return(STATUS_SUCCESS);
  91 }
  92
  93 ULONG
  94 NTAPI
  95 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
  96 {
  97     ULONG Target = InitialTarget;
  98     ULONG NrFreedPages = 0;
  99     NTSTATUS Status;
 100
 101     /* Make sure we can trim this consumer */
 102     if (!MiMemoryConsumers[Consumer].Trim)
 103     {
 104         /* Return the unmodified initial target */
 105         return InitialTarget;
 106     }
 107
 108     if (MmAvailablePages < MiMinimumAvailablePages)
 109     {
 110         /* Global page limit exceeded */
 111         Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
 112     }
 113     else if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
 114     {
 115         /* Consumer page limit exceeded */
 116         Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
 117     }
 118
 119     if (Target)
 120     {
 121         /* Now swap the pages out */
 122         Status = MiMemoryConsumers[Consumer].Trim(Target, MmAvailablePages < MiMinimumAvailablePages, &NrFreedPages);
 123
 124         DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
 125
 126         if (!NT_SUCCESS(Status))
 127         {
 128             KeBugCheck(MEMORY_MANAGEMENT);
 129         }
 130     }
 131
 132     /* Return the page count needed to be freed to meet the initial target */
 133     return (InitialTarget > NrFreedPages) ? (InitialTarget - NrFreedPages) : 0;
 134 }
 135
 136 NTSTATUS
 137 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
 138 {
 139     PFN_NUMBER CurrentPage;
 140     NTSTATUS Status;
 141
 142     (*NrFreedPages) = 0;
 143
 144     DPRINT1("MM BALANCER: %s\n", Priority ? "Paging out!" : "Removing access bit!");
 145
 146     CurrentPage = MmGetLRUFirstUserPage();
 147     while (CurrentPage != 0 && Target > 0)
 148     {
 149         if (Priority)
 150         {
 151             Status = MmPageOutPhysicalAddress(CurrentPage);
 152             if (NT_SUCCESS(Status))
 153             {
 154                 DPRINT("Succeeded\n");
 155                 Target--;
 156                 (*NrFreedPages)++;
 157             }
 158         }
 159         else
 160         {
 161             /* When not paging-out agressively, just reset the accessed bit */
 162             PEPROCESS Process = NULL;
 163             PVOID Address = NULL;
 164             BOOLEAN Accessed = FALSE;
 165
 166             /*
 167              * We have a lock-ordering problem here. We cant lock the PFN DB before the Process address space.
 168              * So we must use circonvoluted loops.
 169              * Well...
 170              */
 171             while (TRUE)
 172             {
 173                 KAPC_STATE ApcState;
 174                 KIRQL OldIrql = MiAcquirePfnLock();
 175                 PMM_RMAP_ENTRY Entry = MmGetRmapListHeadPage(CurrentPage);
 176                 while (Entry)
 177                 {
 178                     if (RMAP_IS_SEGMENT(Entry->Address))
 179                     {
 180                         Entry = Entry->Next;
 181                         continue;
 182                     }
 183
 184                     /* Check that we didn't treat this entry before */
 185                     if (Entry->Address < Address)
 186                     {
 187                         Entry = Entry->Next;
 188                         continue;
 189                     }
 190
 191                     if ((Entry->Address == Address) && (Entry->Process <= Process))
 192                     {
 193                         Entry = Entry->Next;
 194                         continue;
 195                     }
 196
 197                     break;
 198                 }
 199
 200                 if (!Entry)
 201                 {
 202                     MiReleasePfnLock(OldIrql);
 203                     break;
 204                 }
 205
 206                 Process = Entry->Process;
 207                 Address = Entry->Address;
 208
 209                 MiReleasePfnLock(OldIrql);
 210
 211                 KeStackAttachProcess(&Process->Pcb, &ApcState);
 212
 213                 MmLockAddressSpace(&Process->Vm);
 214
 215                 /* Be sure this is still valid. */
 216                 PMMPTE Pte = MiAddressToPte(Address);
 217                 if (Pte->u.Hard.Valid)
 218                 {
 219                     Accessed = Accessed || Pte->u.Hard.Accessed;
 220                     Pte->u.Hard.Accessed = 0;
 221
 222                     /* There is no need to invalidate, the balancer thread is never on a user process */
 223                     //KeInvalidateTlbEntry(Address);
 224                 }
 225
 226                 MmUnlockAddressSpace(&Process->Vm);
 227
 228                 KeUnstackDetachProcess(&ApcState);
 229             }
 230
 231             if (!Accessed)
 232             {
 233                 /* Nobody accessed this page since the last time we check. Time to clean up */
 234
 235                 Status = MmPageOutPhysicalAddress(CurrentPage);
 236                 // DPRINT1("Paged-out one page: %s\n", NT_SUCCESS(Status) ? "Yes" : "No");
 237                 (void)Status;
 238             }
 239
 240             /* Done for this page. */
 241             Target--;
 242         }
 243
 244         CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
 245     }
 246
 247     if (CurrentPage)
 248     {
 249         KIRQL OldIrql = MiAcquirePfnLock();
 250         MmDereferencePage(CurrentPage);
 251         MiReleasePfnLock(OldIrql);
 252     }
 253
 254     return STATUS_SUCCESS;
 255 }
 256
 257 static BOOLEAN
 258 MiIsBalancerThread(VOID)
 259 {
 260     return (MiBalancerThreadHandle != NULL) &&
 261            (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
 262 }
 263
 264 VOID
 265 NTAPI
 266 MmRebalanceMemoryConsumers(VOID)
 267 {
 268     // if (InterlockedCompareExchange(&PageOutThreadActive, 0, 1) == 0)
 269     {
 270         KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
 271     }
 272 }
 273
 274 NTSTATUS
 275 NTAPI
 276 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
 277                             PPFN_NUMBER AllocatedPage)
 278 {
 279     PFN_NUMBER Page;
 280
 281     /* Update the target */
 282     InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
 283
 284     /*
 285      * Actually allocate the page.
 286      */
 287     Page = MmAllocPage(Consumer);
 288     if (Page == 0)
 289     {
 290         KeBugCheck(NO_PAGES_AVAILABLE);
 291     }
 292     *AllocatedPage = Page;
 293
 294     return(STATUS_SUCCESS);
 295 }
 296
 297
 298 VOID NTAPI
 299 MiBalancerThread(PVOID Unused)
 300 {
 301     PVOID WaitObjects[2];
 302     NTSTATUS Status;
 303     ULONG i;
 304
 305     WaitObjects[0] = &MiBalancerEvent;
 306     WaitObjects[1] = &MiBalancerTimer;
 307
 308     while (1)
 309     {
 310         Status = KeWaitForMultipleObjects(2,
 311                                           WaitObjects,
 312                                           WaitAny,
 313                                           Executive,
 314                                           KernelMode,
 315                                           FALSE,
 316                                           NULL,
 317                                           NULL);
 318
 319         if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
 320         {
 321             ULONG InitialTarget = 0;
 322
 323 #if (_MI_PAGING_LEVELS == 2)
 324             if (!MiIsBalancerThread())
 325             {
 326                 /* Clean up the unused PDEs */
 327                 ULONG_PTR Address;
 328                 PEPROCESS Process = PsGetCurrentProcess();
 329
 330                 /* Acquire PFN lock */
 331                 KIRQL OldIrql = MiAcquirePfnLock();
 332                 PMMPDE pointerPde;
 333                 for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
 334                      Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
 335                      Address += PTE_PER_PAGE * PAGE_SIZE)
 336                 {
 337                     if (MiQueryPageTableReferences((PVOID)Address) == 0)
 338                     {
 339                         pointerPde = MiAddressToPde(Address);
 340                         if (pointerPde->u.Hard.Valid)
 341                             MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
 342                         ASSERT(pointerPde->u.Hard.Valid == 0);
 343                     }
 344                 }
 345                 /* Release lock */
 346                 MiReleasePfnLock(OldIrql);
 347             }
 348 #endif
 349             do
 350             {
 351                 ULONG OldTarget = InitialTarget;
 352
 353                 /* Trim each consumer */
 354                 for (i = 0; i < MC_MAXIMUM; i++)
 355                 {
 356                     InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
 357                 }
 358
 359                 /* No pages left to swap! */
 360                 if (InitialTarget != 0 &&
 361                         InitialTarget == OldTarget)
 362                 {
 363                     /* Game over */
 364                     KeBugCheck(NO_PAGES_AVAILABLE);
 365                 }
 366             }
 367             while (InitialTarget != 0);
 368
 369             if (Status == STATUS_WAIT_0)
 370                 InterlockedDecrement(&PageOutThreadActive);
 371         }
 372         else
 373         {
 374             DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
 375             KeBugCheck(MEMORY_MANAGEMENT);
 376         }
 377     }
 378 }
 379
 380 BOOLEAN MmRosNotifyAvailablePage(PFN_NUMBER Page)
 381 {
 382     PLIST_ENTRY Entry;
 383     PMM_ALLOCATION_REQUEST Request;
 384     PMMPFN Pfn1;
 385
 386     /* Make sure the PFN lock is held */
 387     MI_ASSERT_PFN_LOCK_HELD();
 388
 389     if (!MiMinimumAvailablePages)
 390     {
 391         /* Dirty way to know if we were initialized. */
 392         return FALSE;
 393     }
 394
 395     Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock);
 396     if (!Entry)
 397         return FALSE;
 398
 399     Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
 400     MiZeroPhysicalPage(Page);
 401     Request->Page = Page;
 402
 403     Pfn1 = MiGetPfnEntry(Page);
 404     ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
 405     Pfn1->u3.e2.ReferenceCount = 1;
 406     Pfn1->u3.e1.PageLocation = ActiveAndValid;
 407
 408     /* This marks the PFN as a ReactOS PFN */
 409     Pfn1->u4.AweAllocation = TRUE;
 410
 411     /* Allocate the extra ReactOS Data and zero it out */
 412     Pfn1->u1.SwapEntry = 0;
 413     Pfn1->RmapListHead = NULL;
 414
 415     KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
 416
 417     return TRUE;
 418 }
 419
 420 CODE_SEG("INIT")
 421 VOID
 422 NTAPI
 423 MiInitBalancerThread(VOID)
 424 {
 425     KPRIORITY Priority;
 426     NTSTATUS Status;
 427     LARGE_INTEGER Timeout;
 428
 429     KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
 430     KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
 431
 432     Timeout.QuadPart = -20000000; /* 2 sec */
 433     KeSetTimerEx(&MiBalancerTimer,
 434                  Timeout,
 435                  2000,         /* 2 sec */
 436                  NULL);
 437
 438     Status = PsCreateSystemThread(&MiBalancerThreadHandle,
 439                                   THREAD_ALL_ACCESS,
 440                                   NULL,
 441                                   NULL,
 442                                   &MiBalancerThreadId,
 443                                   MiBalancerThread,
 444                                   NULL);
 445     if (!NT_SUCCESS(Status))
 446     {
 447         KeBugCheck(MEMORY_MANAGEMENT);
 448     }
 449
 450     Priority = LOW_REALTIME_PRIORITY + 1;
 451     NtSetInformationThread(MiBalancerThreadHandle,
 452                            ThreadPriority,
 453                            &Priority,
 454                            sizeof(Priority));
 455
 456 }
 457
 458
 459 /* EOF */