ntoskrnl/cc/lazywrite.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS kernel
   4  * FILE:            ntoskrnl/cc/lazywrite.c
   5  * PURPOSE:         Cache manager
   6  *
   7  * PROGRAMMERS:     Pierre Schweitzer (pierre@reactos.org)
   8  */
   9
  10 /* INCLUDES *****************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #define NDEBUG
  14 #include <debug.h>
  15
  16 /* Counters:
  17  * - Amount of pages flushed by lazy writer
  18  * - Number of times lazy writer ran
  19  */
  20 ULONG CcLazyWritePages = 0;
  21 ULONG CcLazyWriteIos = 0;
  22
  23 /* Internal vars (MS):
  24  * - Lazy writer status structure
  25  * - Lookaside list where to allocate work items
  26  * - Queue for high priority work items (read ahead)
  27  * - Queue for regular work items
  28  * - Available worker threads
  29  * - Queue for stuff to be queued after lazy writer is done
  30  * - Marker for throttling queues
  31  * - Number of ongoing workers
  32  * - Three seconds delay for lazy writer
  33  * - One second delay for lazy writer
  34  * - Zero delay for lazy writer
  35  * - Number of worker threads
  36  */
  37 LAZY_WRITER LazyWriter;
  38 NPAGED_LOOKASIDE_LIST CcTwilightLookasideList;
  39 LIST_ENTRY CcExpressWorkQueue;
  40 LIST_ENTRY CcRegularWorkQueue;
  41 LIST_ENTRY CcIdleWorkerThreadList;
  42 LIST_ENTRY CcPostTickWorkQueue;
  43 BOOLEAN CcQueueThrottle = FALSE;
  44 ULONG CcNumberActiveWorkerThreads = 0;
  45 LARGE_INTEGER CcFirstDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)-1*3000*1000*10);
  46 LARGE_INTEGER CcIdleDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)-1*1000*1000*10);
  47 LARGE_INTEGER CcNoDelay = RTL_CONSTANT_LARGE_INTEGER((LONGLONG)0);
  48 ULONG CcNumberWorkerThreads;
  49
  50 /* FUNCTIONS *****************************************************************/
  51
  52 VOID
  53 CcPostWorkQueue(
  54     IN PWORK_QUEUE_ENTRY WorkItem,
  55     IN PLIST_ENTRY WorkQueue)
  56 {
  57     KIRQL OldIrql;
  58     PWORK_QUEUE_ITEM ThreadToSpawn;
  59
  60     /* First of all, insert the item in the queue */
  61     OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock);
  62     InsertTailList(WorkQueue, &WorkItem->WorkQueueLinks);
  63
  64     /* Now, define whether we have to spawn a new work thread
  65      * We will spawn a new one if:
  66      * - There's no throttle in action
  67      * - There's still at least one idle thread
  68      */
  69     ThreadToSpawn = NULL;
  70     if (!CcQueueThrottle && !IsListEmpty(&CcIdleWorkerThreadList))
  71     {
  72         PLIST_ENTRY ListEntry;
  73
  74         /* Get the idle thread */
  75         ListEntry = RemoveHeadList(&CcIdleWorkerThreadList);
  76         ThreadToSpawn = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ITEM, List);
  77
  78         /* We're going to have one more! */
  79         CcNumberActiveWorkerThreads += 1;
  80     }
  81
  82     KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
  83
  84     /* If we have a thread to spawn, do it! */
  85     if (ThreadToSpawn != NULL)
  86     {
  87         /* We NULLify it to be consistent with initialization */
  88         ThreadToSpawn->List.Flink = NULL;
  89         ExQueueWorkItem(ThreadToSpawn, CriticalWorkQueue);
  90     }
  91 }
  92
  93 VOID
  94 NTAPI
  95 CcScanDpc(
  96     IN PKDPC Dpc,
  97     IN PVOID DeferredContext,
  98     IN PVOID SystemArgument1,
  99     IN PVOID SystemArgument2)
 100 {
 101     PWORK_QUEUE_ENTRY WorkItem;
 102
 103     /* Allocate a work item */
 104     WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList);
 105     if (WorkItem == NULL)
 106     {
 107         LazyWriter.ScanActive = FALSE;
 108         return;
 109     }
 110
 111     /* And post it, it will be for lazy write */
 112     WorkItem->Function = LazyScan;
 113     CcPostWorkQueue(WorkItem, &CcRegularWorkQueue);
 114 }
 115
 116 VOID
 117 CcWriteBehind(VOID)
 118 {
 119     ULONG Target, Count;
 120
 121     Target = CcTotalDirtyPages / 8;
 122     if (Target != 0)
 123     {
 124         /* Flush! */
 125         DPRINT("Lazy writer starting (%d)\n", Target);
 126         CcRosFlushDirtyPages(Target, &Count, FALSE, TRUE);
 127
 128         /* And update stats */
 129         CcLazyWritePages += Count;
 130         ++CcLazyWriteIos;
 131         DPRINT("Lazy writer done (%d)\n", Count);
 132     }
 133
 134     /* Make sure we're not throttling writes after this */
 135     while (MmAvailablePages < MmThrottleTop)
 136     {
 137         /* Break if we can't even find one to free */
 138         if (!CcRosFreeOneUnusedVacb())
 139         {
 140             break;
 141         }
 142     }
 143 }
 144
 145 VOID
 146 CcLazyWriteScan(VOID)
 147 {
 148     ULONG Target;
 149     KIRQL OldIrql;
 150     PLIST_ENTRY ListEntry;
 151     LIST_ENTRY ToPost;
 152     PWORK_QUEUE_ENTRY WorkItem;
 153
 154     /* Do we have entries to queue after we're done? */
 155     InitializeListHead(&ToPost);
 156     OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
 157     if (LazyWriter.OtherWork)
 158     {
 159         while (!IsListEmpty(&CcPostTickWorkQueue))
 160         {
 161             ListEntry = RemoveHeadList(&CcPostTickWorkQueue);
 162             WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks);
 163             InsertTailList(&ToPost, &WorkItem->WorkQueueLinks);
 164         }
 165         LazyWriter.OtherWork = FALSE;
 166     }
 167     KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql);
 168
 169     /* Our target is one-eighth of the dirty pages */
 170     Target = CcTotalDirtyPages / 8;
 171     if (Target != 0)
 172     {
 173         /* There is stuff to flush, schedule a write-behind operation */
 174
 175         /* Allocate a work item */
 176         WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList);
 177         if (WorkItem != NULL)
 178         {
 179             WorkItem->Function = WriteBehind;
 180             CcPostWorkQueue(WorkItem, &CcRegularWorkQueue);
 181         }
 182     }
 183
 184     /* Post items that were due for end of run */
 185     while (!IsListEmpty(&ToPost))
 186     {
 187         ListEntry = RemoveHeadList(&ToPost);
 188         WorkItem = CONTAINING_RECORD(ListEntry, WORK_QUEUE_ENTRY, WorkQueueLinks);
 189         CcPostWorkQueue(WorkItem, &CcRegularWorkQueue);
 190     }
 191
 192     /* If we have deferred writes, try them now! */
 193     if (!IsListEmpty(&CcDeferredWrites))
 194     {
 195         CcPostDeferredWrites();
 196         /* Reschedule immediately a lazy writer run
 197          * Keep us active to have short idle delay
 198          */
 199         CcScheduleLazyWriteScan(FALSE);
 200     }
 201     else
 202     {
 203         /* We're no longer active */
 204         OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
 205         LazyWriter.ScanActive = FALSE;
 206         KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql);
 207     }
 208 }
 209
 210 VOID CcScheduleLazyWriteScan(
 211     IN BOOLEAN NoDelay)
 212 {
 213     /* If no delay, immediately start lazy writer,
 214      * no matter it was already started
 215      */
 216     if (NoDelay)
 217     {
 218         LazyWriter.ScanActive = TRUE;
 219         KeSetTimer(&LazyWriter.ScanTimer, CcNoDelay, &LazyWriter.ScanDpc);
 220     }
 221     /* Otherwise, if it's not running, just wait three seconds to start it */
 222     else if (!LazyWriter.ScanActive)
 223     {
 224         LazyWriter.ScanActive = TRUE;
 225         KeSetTimer(&LazyWriter.ScanTimer, CcFirstDelay, &LazyWriter.ScanDpc);
 226     }
 227     /* Finally, already running, so queue for the next second */
 228     else
 229     {
 230         KeSetTimer(&LazyWriter.ScanTimer, CcIdleDelay, &LazyWriter.ScanDpc);
 231     }
 232 }
 233
 234 VOID
 235 NTAPI
 236 CcWorkerThread(
 237     IN PVOID Parameter)
 238 {
 239     KIRQL OldIrql;
 240     BOOLEAN DropThrottle, WritePerformed;
 241     PWORK_QUEUE_ITEM Item;
 242 #if DBG
 243     PIRP TopLevel;
 244 #endif
 245
 246     /* Get back our thread item */
 247     Item = Parameter;
 248     /* And by default, don't touch throttle */
 249     DropThrottle = FALSE;
 250     /* No write performed */
 251     WritePerformed =  FALSE;
 252
 253 #if DBG
 254     /* Top level IRP should be clean when started
 255      * Save it to catch buggy drivers (or bugs!)
 256      */
 257     TopLevel = IoGetTopLevelIrp();
 258     if (TopLevel != NULL)
 259     {
 260         DPRINT1("(%p) TopLevel IRP for this thread: %p\n", PsGetCurrentThread(), TopLevel);
 261     }
 262 #endif
 263
 264     /* Loop till we have jobs */
 265     while (TRUE)
 266     {
 267         PWORK_QUEUE_ENTRY WorkItem;
 268
 269         /* Lock queues */
 270         OldIrql = KeAcquireQueuedSpinLock(LockQueueWorkQueueLock);
 271
 272         /* If we have to touch throttle, reset it now! */
 273         if (DropThrottle)
 274         {
 275             CcQueueThrottle = FALSE;
 276             DropThrottle = FALSE;
 277         }
 278
 279         /* Check first if we have read ahead to do */
 280         if (IsListEmpty(&CcExpressWorkQueue))
 281         {
 282             /* If not, check regular queue */
 283             if (IsListEmpty(&CcRegularWorkQueue))
 284             {
 285                 break;
 286             }
 287             else
 288             {
 289                 WorkItem = CONTAINING_RECORD(CcRegularWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks);
 290             }
 291         }
 292         else
 293         {
 294             WorkItem = CONTAINING_RECORD(CcExpressWorkQueue.Flink, WORK_QUEUE_ENTRY, WorkQueueLinks);
 295         }
 296
 297         /* Get our work item, if someone is waiting for us to finish
 298          * and we're not the only thread in queue
 299          * then, quit running to let the others do
 300          * and throttle so that noone starts till current activity is over
 301          */
 302         if (WorkItem->Function == SetDone && CcNumberActiveWorkerThreads > 1)
 303         {
 304             CcQueueThrottle = TRUE;
 305             break;
 306         }
 307
 308         /* Otherwise, remove current entry */
 309         RemoveEntryList(&WorkItem->WorkQueueLinks);
 310         KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
 311
 312         /* And handle it */
 313         switch (WorkItem->Function)
 314         {
 315             case ReadAhead:
 316                 CcPerformReadAhead(WorkItem->Parameters.Read.FileObject);
 317                 break;
 318
 319             case WriteBehind:
 320                 PsGetCurrentThread()->MemoryMaker = 1;
 321                 CcWriteBehind();
 322                 PsGetCurrentThread()->MemoryMaker = 0;
 323                 WritePerformed = TRUE;
 324                 break;
 325
 326             case LazyScan:
 327                 CcLazyWriteScan();
 328                 break;
 329
 330             case SetDone:
 331                 KeSetEvent(WorkItem->Parameters.Event.Event, IO_NO_INCREMENT, FALSE);
 332                 DropThrottle = TRUE;
 333                 break;
 334
 335             default:
 336                 DPRINT1("Ignored item: %p (%d)\n", WorkItem, WorkItem->Function);
 337                 break;
 338         }
 339
 340         /* And release the item */
 341         ExFreeToNPagedLookasideList(&CcTwilightLookasideList, WorkItem);
 342     }
 343
 344     /* Our thread is available again */
 345     InsertTailList(&CcIdleWorkerThreadList, &Item->List);
 346     /* One less worker */
 347     --CcNumberActiveWorkerThreads;
 348     KeReleaseQueuedSpinLock(LockQueueWorkQueueLock, OldIrql);
 349
 350     /* If there are pending write openations and we have at least 20 dirty pages */
 351     if (!IsListEmpty(&CcDeferredWrites) && CcTotalDirtyPages >= 20)
 352     {
 353         /* And if we performed a write operation previously, then
 354          * stress the system a bit and reschedule a scan to find
 355          * stuff to write
 356          */
 357         if (WritePerformed)
 358         {
 359             CcLazyWriteScan();
 360         }
 361     }
 362
 363 #if DBG
 364     /* Top level shouldn't have changed */
 365     if (TopLevel != IoGetTopLevelIrp())
 366     {
 367         DPRINT1("(%p) Mismatching TopLevel: %p, %p\n", PsGetCurrentThread(), TopLevel, IoGetTopLevelIrp());
 368     }
 369 #endif
 370 }
 371
 372 /*
 373  * @implemented
 374  */
 375 NTSTATUS
 376 NTAPI
 377 CcWaitForCurrentLazyWriterActivity (
 378     VOID)
 379 {
 380     KIRQL OldIrql;
 381     KEVENT WaitEvent;
 382     PWORK_QUEUE_ENTRY WorkItem;
 383
 384     /* Allocate a work item */
 385     WorkItem = ExAllocateFromNPagedLookasideList(&CcTwilightLookasideList);
 386     if (WorkItem == NULL)
 387     {
 388         return STATUS_INSUFFICIENT_RESOURCES;
 389     }
 390
 391     /* We want lazy writer to set our event */
 392     WorkItem->Function = SetDone;
 393     KeInitializeEvent(&WaitEvent, NotificationEvent, FALSE);
 394     WorkItem->Parameters.Event.Event = &WaitEvent;
 395
 396     /* Use the post tick queue */
 397     OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock);
 398     InsertTailList(&CcPostTickWorkQueue, &WorkItem->WorkQueueLinks);
 399
 400     /* Inform the lazy writer it will have to handle the post tick queue */
 401     LazyWriter.OtherWork = TRUE;
 402     /* And if it's not running, queue a lazy writer run
 403      * And start it NOW, we want the response now
 404      */
 405     if (!LazyWriter.ScanActive)
 406     {
 407         CcScheduleLazyWriteScan(TRUE);
 408     }
 409
 410     KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql);
 411
 412     /* And now, wait until lazy writer replies */
 413     return KeWaitForSingleObject(&WaitEvent, Executive, KernelMode, FALSE, NULL);
 414 }