a82906b0cec2c10c1d33fbecd54c5491ff746c4b
[reactos.git] / reactos / ntoskrnl / mm / balance.c
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS kernel
4 * FILE: ntoskrnl/mm/balance.c
5 * PURPOSE: kernel memory managment functions
6 *
7 * PROGRAMMERS: David Welch (welch@cwcom.net)
8 * Cameron Gutman (cameron.gutman@reactos.org)
9 */
10
11 /* INCLUDES *****************************************************************/
12
13 #include <ntoskrnl.h>
14 #define NDEBUG
15 #include <debug.h>
16
17 #include "ARM3/miarm.h"
18
19 #if defined (ALLOC_PRAGMA)
20 #pragma alloc_text(INIT, MmInitializeBalancer)
21 #pragma alloc_text(INIT, MmInitializeMemoryConsumer)
22 #pragma alloc_text(INIT, MiInitBalancerThread)
23 #endif
24
25
26 /* TYPES ********************************************************************/
27 typedef struct _MM_ALLOCATION_REQUEST
28 {
29 PFN_NUMBER Page;
30 LIST_ENTRY ListEntry;
31 KEVENT Event;
32 }
33 MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST;
34 /* GLOBALS ******************************************************************/
35
36 MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM];
37 static ULONG MiMinimumAvailablePages;
38 static ULONG MiNrTotalPages;
39 static LIST_ENTRY AllocationListHead;
40 static KSPIN_LOCK AllocationListLock;
41 static ULONG MiMinimumPagesPerRun;
42
43 static CLIENT_ID MiBalancerThreadId;
44 static HANDLE MiBalancerThreadHandle = NULL;
45 static KEVENT MiBalancerEvent;
46 static KTIMER MiBalancerTimer;
47
48 /* FUNCTIONS ****************************************************************/
49
50 VOID
51 INIT_FUNCTION
52 NTAPI
53 MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages)
54 {
55 memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers));
56 InitializeListHead(&AllocationListHead);
57 KeInitializeSpinLock(&AllocationListLock);
58
59 MiNrTotalPages = NrAvailablePages;
60
61 /* Set up targets. */
62 MiMinimumAvailablePages = 128;
63 MiMinimumPagesPerRun = 256;
64 if ((NrAvailablePages + NrSystemPages) >= 8192)
65 {
66 MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3;
67 }
68 else if ((NrAvailablePages + NrSystemPages) >= 4096)
69 {
70 MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2;
71 }
72 else
73 {
74 MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8;
75 }
76 MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages;
77 }
78
79 VOID
80 INIT_FUNCTION
81 NTAPI
82 MmInitializeMemoryConsumer(
83 ULONG Consumer,
84 NTSTATUS (*Trim)(ULONG Target, ULONG Priority, PULONG NrFreed))
85 {
86 MiMemoryConsumers[Consumer].Trim = Trim;
87 }
88
89 VOID
90 NTAPI
91 MiZeroPhysicalPage(
92 IN PFN_NUMBER PageFrameIndex
93 );
94
95 NTSTATUS
96 NTAPI
97 MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
98 {
99 PMM_ALLOCATION_REQUEST Request;
100 PLIST_ENTRY Entry;
101
102 if (Page == 0)
103 {
104 DPRINT1("Tried to release page zero.\n");
105 KeBugCheck(MEMORY_MANAGEMENT);
106 }
107
108 if (MmGetReferenceCountPage(Page) == 1)
109 {
110 if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
111 (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
112 if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL)
113 {
114 MmDereferencePage(Page);
115 }
116 else
117 {
118 Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry);
119 MiZeroPhysicalPage(Page);
120 Request->Page = Page;
121 KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE);
122 }
123 }
124 else
125 {
126 MmDereferencePage(Page);
127 }
128
129 return(STATUS_SUCCESS);
130 }
131
132 ULONG
133 NTAPI
134 MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget)
135 {
136 ULONG Target = InitialTarget;
137 ULONG NrFreedPages = 0;
138 NTSTATUS Status;
139
140 /* Make sure we can trim this consumer */
141 if (!MiMemoryConsumers[Consumer].Trim)
142 {
143 /* Return the unmodified initial target */
144 return InitialTarget;
145 }
146
147 if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget)
148 {
149 /* Consumer page limit exceeded */
150 Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget);
151 }
152 if (MmAvailablePages < MiMinimumAvailablePages)
153 {
154 /* Global page limit exceeded */
155 Target = (ULONG)max(Target, MiMinimumAvailablePages - MmAvailablePages);
156 }
157
158 if (Target)
159 {
160 if (!InitialTarget)
161 {
162 /* If there was no initial target,
163 * swap at least MiMinimumPagesPerRun */
164 Target = max(Target, MiMinimumPagesPerRun);
165 }
166
167 /* Now swap the pages out */
168 Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages);
169
170 DPRINT("Trimming consumer %lu: Freed %lu pages with a target of %lu pages\n", Consumer, NrFreedPages, Target);
171
172 if (!NT_SUCCESS(Status))
173 {
174 KeBugCheck(MEMORY_MANAGEMENT);
175 }
176
177 /* Update the target */
178 if (NrFreedPages < Target)
179 Target -= NrFreedPages;
180 else
181 Target = 0;
182
183 /* Return the remaining pages needed to meet the target */
184 return Target;
185 }
186 else
187 {
188 /* Initial target is zero and we don't have anything else to add */
189 return 0;
190 }
191 }
192
193 NTSTATUS
194 MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
195 {
196 PFN_NUMBER CurrentPage;
197 PFN_NUMBER NextPage;
198 NTSTATUS Status;
199
200 (*NrFreedPages) = 0;
201
202 CurrentPage = MmGetLRUFirstUserPage();
203 while (CurrentPage != 0 && Target > 0)
204 {
205 Status = MmPageOutPhysicalAddress(CurrentPage);
206 if (NT_SUCCESS(Status))
207 {
208 DPRINT("Succeeded\n");
209 Target--;
210 (*NrFreedPages)++;
211 }
212
213 NextPage = MmGetLRUNextUserPage(CurrentPage);
214 if (NextPage <= CurrentPage)
215 {
216 /* We wrapped around, so we're done */
217 break;
218 }
219 CurrentPage = NextPage;
220 }
221
222 return STATUS_SUCCESS;
223 }
224
225 static BOOLEAN
226 MiIsBalancerThread(VOID)
227 {
228 return (MiBalancerThreadHandle != NULL) &&
229 (PsGetCurrentThreadId() == MiBalancerThreadId.UniqueThread);
230 }
231
232 VOID
233 NTAPI
234 MmRebalanceMemoryConsumers(VOID)
235 {
236 if (MiBalancerThreadHandle != NULL &&
237 !MiIsBalancerThread())
238 {
239 KeSetEvent(&MiBalancerEvent, IO_NO_INCREMENT, FALSE);
240 }
241 }
242
243 NTSTATUS
244 NTAPI
245 MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
246 PPFN_NUMBER AllocatedPage)
247 {
248 ULONG PagesUsed;
249 PFN_NUMBER Page;
250
251 /*
252 * Make sure we don't exceed our individual target.
253 */
254 PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
255 if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget &&
256 !MiIsBalancerThread())
257 {
258 MmRebalanceMemoryConsumers();
259 }
260
261 /*
262 * Allocate always memory for the non paged pool and for the pager thread.
263 */
264 if ((Consumer == MC_SYSTEM) || MiIsBalancerThread())
265 {
266 Page = MmAllocPage(Consumer);
267 if (Page == 0)
268 {
269 KeBugCheck(NO_PAGES_AVAILABLE);
270 }
271 if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
272 *AllocatedPage = Page;
273 if (MmAvailablePages < MiMinimumAvailablePages)
274 MmRebalanceMemoryConsumers();
275 return(STATUS_SUCCESS);
276 }
277
278 /*
279 * Make sure we don't exceed global targets.
280 */
281 if (MmAvailablePages < MiMinimumAvailablePages)
282 {
283 MM_ALLOCATION_REQUEST Request;
284
285 if (!CanWait)
286 {
287 (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
288 MmRebalanceMemoryConsumers();
289 return(STATUS_NO_MEMORY);
290 }
291
292 /* Insert an allocation request. */
293 Request.Page = 0;
294 KeInitializeEvent(&Request.Event, NotificationEvent, FALSE);
295
296 ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock);
297 MmRebalanceMemoryConsumers();
298
299 KeWaitForSingleObject(&Request.Event,
300 0,
301 KernelMode,
302 FALSE,
303 NULL);
304
305 Page = Request.Page;
306 if (Page == 0)
307 {
308 KeBugCheck(NO_PAGES_AVAILABLE);
309 }
310
311 if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
312 *AllocatedPage = Page;
313
314 if (MmAvailablePages < MiMinimumAvailablePages)
315 {
316 MmRebalanceMemoryConsumers();
317 }
318
319 return(STATUS_SUCCESS);
320 }
321
322 /*
323 * Actually allocate the page.
324 */
325 Page = MmAllocPage(Consumer);
326 if (Page == 0)
327 {
328 KeBugCheck(NO_PAGES_AVAILABLE);
329 }
330 if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
331 *AllocatedPage = Page;
332
333 if (MmAvailablePages < MiMinimumAvailablePages)
334 {
335 MmRebalanceMemoryConsumers();
336 }
337
338 return(STATUS_SUCCESS);
339 }
340
341
342 VOID NTAPI
343 MiBalancerThread(PVOID Unused)
344 {
345 PVOID WaitObjects[2];
346 NTSTATUS Status;
347 ULONG i;
348
349 WaitObjects[0] = &MiBalancerEvent;
350 WaitObjects[1] = &MiBalancerTimer;
351
352 while (1)
353 {
354 Status = KeWaitForMultipleObjects(2,
355 WaitObjects,
356 WaitAny,
357 Executive,
358 KernelMode,
359 FALSE,
360 NULL,
361 NULL);
362
363 if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1)
364 {
365 ULONG InitialTarget = 0;
366
367 #if (_MI_PAGING_LEVELS == 2)
368 if (!MiIsBalancerThread())
369 {
370 /* Clean up the unused PDEs */
371 ULONG_PTR Address;
372 PEPROCESS Process = PsGetCurrentProcess();
373
374 /* Acquire PFN lock */
375 KIRQL OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
376 PMMPDE pointerPde;
377 for (Address = (ULONG_PTR)MI_LOWEST_VAD_ADDRESS;
378 Address < (ULONG_PTR)MM_HIGHEST_VAD_ADDRESS;
379 Address += (PAGE_SIZE * PTE_COUNT))
380 {
381 if (MiQueryPageTableReferences((PVOID)Address) == 0)
382 {
383 pointerPde = MiAddressToPde(Address);
384 if (pointerPde->u.Hard.Valid)
385 MiDeletePte(pointerPde, MiPdeToPte(pointerPde), Process, NULL);
386 ASSERT(pointerPde->u.Hard.Valid == 0);
387 }
388 }
389 /* Release lock */
390 KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql);
391 }
392 #endif
393 do
394 {
395 ULONG OldTarget = InitialTarget;
396
397 /* Trim each consumer */
398 for (i = 0; i < MC_MAXIMUM; i++)
399 {
400 InitialTarget = MiTrimMemoryConsumer(i, InitialTarget);
401 }
402
403 /* No pages left to swap! */
404 if (InitialTarget != 0 &&
405 InitialTarget == OldTarget)
406 {
407 /* Game over */
408 KeBugCheck(NO_PAGES_AVAILABLE);
409 }
410 }
411 while (InitialTarget != 0);
412 }
413 else
414 {
415 DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status);
416 KeBugCheck(MEMORY_MANAGEMENT);
417 }
418 }
419 }
420
421 VOID
422 INIT_FUNCTION
423 NTAPI
424 MiInitBalancerThread(VOID)
425 {
426 KPRIORITY Priority;
427 NTSTATUS Status;
428 #if !defined(__GNUC__)
429
430 LARGE_INTEGER dummyJunkNeeded;
431 dummyJunkNeeded.QuadPart = -20000000; /* 2 sec */
432 ;
433 #endif
434
435
436 KeInitializeEvent(&MiBalancerEvent, SynchronizationEvent, FALSE);
437 KeInitializeTimerEx(&MiBalancerTimer, SynchronizationTimer);
438 KeSetTimerEx(&MiBalancerTimer,
439 #if defined(__GNUC__)
440 (LARGE_INTEGER)(LONGLONG)-20000000LL, /* 2 sec */
441 #else
442 dummyJunkNeeded,
443 #endif
444 2000, /* 2 sec */
445 NULL);
446
447 Status = PsCreateSystemThread(&MiBalancerThreadHandle,
448 THREAD_ALL_ACCESS,
449 NULL,
450 NULL,
451 &MiBalancerThreadId,
452 MiBalancerThread,
453 NULL);
454 if (!NT_SUCCESS(Status))
455 {
456 KeBugCheck(MEMORY_MANAGEMENT);
457 }
458
459 Priority = LOW_REALTIME_PRIORITY + 1;
460 NtSetInformationThread(MiBalancerThreadHandle,
461 ThreadPriority,
462 &Priority,
463 sizeof(Priority));
464
465 }
466
467
468 /* EOF */