2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/cache/pinsup.c
5 * PURPOSE: Logging and configuration routines
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
10 /* INCLUDES *******************************************************************/
14 #include "section/newmm.h"
18 /* The following is a test mode that only works with modified filesystems.
19 * it maps the cache sections read only until they're pinned writable, and then
20 * turns them readonly again when they're unpinned.
21 * This helped me determine that a certain bug was not a memory overwrite. */
22 //#define PIN_WRITE_ONLY
24 /* GLOBALS ********************************************************************/
26 #define TAG_MAP_SEC TAG('C', 'c', 'S', 'x')
27 #define TAG_MAP_READ TAG('M', 'c', 'p', 'y')
28 #define TAG_MAP_BCB TAG('B', 'c', 'b', ' ')
30 NOCC_BCB CcCacheSections
[CACHE_NUM_SECTIONS
];
31 CHAR CcpBitmapBuffer
[sizeof(RTL_BITMAP
) + ROUND_UP((CACHE_NUM_SECTIONS
), 32) / 8];
32 PRTL_BITMAP CcCacheBitmap
= (PRTL_BITMAP
)&CcpBitmapBuffer
;
35 KEVENT CcFinalizeEvent
;
36 ULONG CcCacheClockHand
;
37 LONG CcOutstandingDeletes
;
39 /* FUNCTIONS ******************************************************************/
42 VOID
_CcpLock(const char *file
, int line
)
44 //DPRINT("<<<---<<< CC In Mutex(%s:%d %x)!\n", file, line, PsGetCurrentThread());
45 ExAcquireFastMutex(&CcMutex
);
48 VOID
_CcpUnlock(const char *file
, int line
)
50 ExReleaseFastMutex(&CcMutex
);
51 //DPRINT(">>>--->>> CC Exit Mutex!\n", file, line);
56 MmGetDeviceObjectForFile(IN PFILE_OBJECT FileObject
);
58 NTSTATUS CcpAllocateSection
59 (PFILE_OBJECT FileObject
,
62 PMM_CACHE_SECTION_SEGMENT
*Result
)
65 LARGE_INTEGER MaxSize
;
67 MaxSize
.QuadPart
= Length
;
69 DPRINT("Making Section for File %x\n", FileObject
);
70 DPRINT("File name %wZ\n", &FileObject
->FileName
);
71 Status
= MmCreateCacheSection
73 STANDARD_RIGHTS_REQUIRED
,
77 SEC_RESERVE
| SEC_CACHE
,
83 typedef struct _WORK_QUEUE_WITH_CONTEXT
85 WORK_QUEUE_ITEM WorkItem
;
87 LARGE_INTEGER FileOffset
;
88 LARGE_INTEGER MapSize
;
89 PMM_CACHE_SECTION_SEGMENT ToDeref
;
90 PACQUIRE_FOR_LAZY_WRITE AcquireForLazyWrite
;
91 PRELEASE_FROM_LAZY_WRITE ReleaseFromLazyWrite
;
94 } WORK_QUEUE_WITH_CONTEXT
, *PWORK_QUEUE_WITH_CONTEXT
;
97 CcpUnmapCache(PVOID Context
)
99 PWORK_QUEUE_WITH_CONTEXT WorkItem
= (PWORK_QUEUE_WITH_CONTEXT
)Context
;
100 DPRINT("Unmapping (finally) %x\n", WorkItem
->ToUnmap
);
101 WorkItem
->AcquireForLazyWrite(WorkItem
->LazyContext
, TRUE
);
102 MiFlushMappedSection(WorkItem
->ToUnmap
, &WorkItem
->FileOffset
, &WorkItem
->MapSize
, WorkItem
->Dirty
);
103 WorkItem
->ReleaseFromLazyWrite(WorkItem
->LazyContext
);
104 MmUnmapCacheViewInSystemSpace(WorkItem
->ToUnmap
);
105 MmFinalizeSegment(WorkItem
->ToDeref
);
106 ExFreePool(WorkItem
);
110 /* Must have acquired the mutex */
111 VOID
CcpDereferenceCache(ULONG Start
, BOOLEAN Immediate
)
116 LARGE_INTEGER MappedSize
;
117 LARGE_INTEGER BaseOffset
;
118 PWORK_QUEUE_WITH_CONTEXT WorkItem
;
120 DPRINT("CcpDereferenceCache(#%x)\n", Start
);
122 Bcb
= &CcCacheSections
[Start
];
125 ToUnmap
= Bcb
->BaseAddress
;
126 BaseOffset
= Bcb
->FileOffset
;
127 MappedSize
= Bcb
->Map
->FileSizes
.ValidDataLength
;
129 DPRINT("Dereference #%x (count %d)\n", Start
, Bcb
->RefCount
);
130 ASSERT(Bcb
->SectionObject
);
131 ASSERT(Bcb
->RefCount
== 1);
133 DPRINT("Firing work item for %x\n", Bcb
->BaseAddress
);
137 PMM_CACHE_SECTION_SEGMENT ToDeref
= Bcb
->SectionObject
;
138 BOOLEAN Dirty
= Bcb
->Dirty
;
141 Bcb
->SectionObject
= NULL
;
142 Bcb
->BaseAddress
= NULL
;
143 Bcb
->FileOffset
.QuadPart
= 0;
147 RemoveEntryList(&Bcb
->ThisFileList
);
150 MiFlushMappedSection(ToUnmap
, &BaseOffset
, &MappedSize
, Dirty
);
151 MmUnmapCacheViewInSystemSpace(ToUnmap
);
152 MmFinalizeSegment(ToDeref
);
157 WorkItem
= ExAllocatePool(NonPagedPool
, sizeof(*WorkItem
));
158 if (!WorkItem
) KeBugCheck(0);
159 WorkItem
->ToUnmap
= Bcb
->BaseAddress
;
160 WorkItem
->FileOffset
= Bcb
->FileOffset
;
161 WorkItem
->Dirty
= Bcb
->Dirty
;
162 WorkItem
->MapSize
= MappedSize
;
163 WorkItem
->ToDeref
= Bcb
->SectionObject
;
164 WorkItem
->AcquireForLazyWrite
= Bcb
->Map
->Callbacks
.AcquireForLazyWrite
;
165 WorkItem
->ReleaseFromLazyWrite
= Bcb
->Map
->Callbacks
.ReleaseFromLazyWrite
;
166 WorkItem
->LazyContext
= Bcb
->Map
->LazyContext
;
168 ExInitializeWorkItem(((PWORK_QUEUE_ITEM
)WorkItem
), (PWORKER_THREAD_ROUTINE
)CcpUnmapCache
, WorkItem
);
171 Bcb
->SectionObject
= NULL
;
172 Bcb
->BaseAddress
= NULL
;
173 Bcb
->FileOffset
.QuadPart
= 0;
177 RemoveEntryList(&Bcb
->ThisFileList
);
180 ExQueueWorkItem((PWORK_QUEUE_ITEM
)WorkItem
, DelayedWorkQueue
);
187 ULONG CcpAllocateCacheSections
188 (PFILE_OBJECT FileObject
,
189 PMM_CACHE_SECTION_SEGMENT SectionObject
)
191 ULONG i
= INVALID_CACHE
;
195 DPRINT("AllocateCacheSections: FileObject %x\n", FileObject
);
197 if (!FileObject
->SectionObjectPointer
)
198 return INVALID_CACHE
;
200 Map
= (PNOCC_CACHE_MAP
)FileObject
->SectionObjectPointer
->SharedCacheMap
;
203 return INVALID_CACHE
;
205 DPRINT("Allocating Cache Section\n");
207 i
= RtlFindClearBitsAndSet(CcCacheBitmap
, 1, CcCacheClockHand
);
208 CcCacheClockHand
= (i
+ 1) % CACHE_NUM_SECTIONS
;
210 if (i
!= INVALID_CACHE
)
212 DPRINT("Setting up Bcb #%x\n", i
);
214 Bcb
= &CcCacheSections
[i
];
216 ASSERT(Bcb
->RefCount
< 2);
218 if (Bcb
->RefCount
> 0)
220 CcpDereferenceCache(i
, FALSE
);
223 ASSERT(!Bcb
->RefCount
);
226 DPRINT("Bcb #%x RefCount %d\n", Bcb
- CcCacheSections
, Bcb
->RefCount
);
228 if (!RtlTestBit(CcCacheBitmap
, i
))
230 DPRINT("Somebody stoeled BCB #%x\n", i
);
232 ASSERT(RtlTestBit(CcCacheBitmap
, i
));
234 DPRINT("Allocated #%x\n", i
);
235 ASSERT(CcCacheSections
[i
].RefCount
);
239 DPRINT("Failed to allocate cache segment\n");
244 /* Must have acquired the mutex */
245 VOID
CcpReferenceCache(ULONG Start
)
248 Bcb
= &CcCacheSections
[Start
];
249 ASSERT(Bcb
->SectionObject
);
251 RtlSetBit(CcCacheBitmap
, Start
);
255 VOID
CcpMarkForExclusive(ULONG Start
)
258 Bcb
= &CcCacheSections
[Start
];
259 Bcb
->ExclusiveWaiter
++;
262 /* Must not have the mutex */
263 VOID
CcpReferenceCacheExclusive(ULONG Start
)
265 PNOCC_BCB Bcb
= &CcCacheSections
[Start
];
267 KeWaitForSingleObject(&Bcb
->ExclusiveWait
, Executive
, KernelMode
, FALSE
, NULL
);
269 ASSERT(Bcb
->ExclusiveWaiter
);
270 ASSERT(Bcb
->SectionObject
);
271 Bcb
->Exclusive
= TRUE
;
272 Bcb
->ExclusiveWaiter
--;
273 RtlSetBit(CcCacheBitmap
, Start
);
277 /* Find a map that encompasses the target range */
278 /* Must have the mutex */
279 ULONG
CcpFindMatchingMap(PLIST_ENTRY Head
, PLARGE_INTEGER FileOffset
, ULONG Length
)
282 //DPRINT("Find Matching Map: (%x) %x:%x\n", FileOffset->LowPart, Length);
283 for (Entry
= Head
->Flink
; Entry
!= Head
; Entry
= Entry
->Flink
)
285 //DPRINT("Link @%x\n", Entry);
286 PNOCC_BCB Bcb
= CONTAINING_RECORD(Entry
, NOCC_BCB
, ThisFileList
);
287 //DPRINT("Selected BCB %x #%x\n", Bcb, Bcb - CcCacheSections);
288 //DPRINT("This File: %x:%x\n", Bcb->FileOffset.LowPart, Bcb->Length);
289 if (FileOffset
->QuadPart
>= Bcb
->FileOffset
.QuadPart
&&
290 FileOffset
->QuadPart
< Bcb
->FileOffset
.QuadPart
+ CACHE_STRIPE
)
292 //DPRINT("Found match at #%x\n", Bcb - CcCacheSections);
293 return Bcb
- CcCacheSections
;
297 //DPRINT("This region isn't mapped\n");
299 return INVALID_CACHE
;
305 (IN PFILE_OBJECT FileObject
,
306 IN PLARGE_INTEGER FileOffset
,
309 OUT PVOID
*BcbResult
,
312 BOOLEAN Success
= FALSE
, FaultIn
= FALSE
;
313 /* Note: windows 2000 drivers treat this as a bool */
314 //BOOLEAN Wait = (Flags & MAP_WAIT) || (Flags == TRUE);
315 LARGE_INTEGER Target
, EndInterval
;
317 PNOCC_BCB Bcb
= NULL
;
318 PMM_CACHE_SECTION_SEGMENT SectionObject
= NULL
;
320 PNOCC_CACHE_MAP Map
= (PNOCC_CACHE_MAP
)FileObject
->SectionObjectPointer
->SharedCacheMap
;
324 DPRINT1("File object was not mapped\n");
328 DPRINT("CcMapData(F->%x,%08x%08x:%d)\n", FileObject
, FileOffset
->HighPart
, FileOffset
->LowPart
, Length
);
330 ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL
);
332 Target
.HighPart
= FileOffset
->HighPart
;
333 Target
.LowPart
= CACHE_ROUND_DOWN(FileOffset
->LowPart
);
337 /* Find out if any range is a superset of what we want */
338 /* Find an accomodating section */
339 BcbHead
= CcpFindMatchingMap(&Map
->AssociatedBcb
, FileOffset
, Length
);
341 if (BcbHead
!= INVALID_CACHE
)
343 Bcb
= &CcCacheSections
[BcbHead
];
346 *Buffer
= ((PCHAR
)Bcb
->BaseAddress
) + (int)(FileOffset
->QuadPart
- Bcb
->FileOffset
.QuadPart
);
348 ("Bcb #%x Buffer maps (%08x%08x) At %x Length %x (Getting %x:%x) %wZ\n",
349 Bcb
- CcCacheSections
,
350 Bcb
->FileOffset
.HighPart
,
351 Bcb
->FileOffset
.LowPart
,
356 &FileObject
->FileName
);
363 DPRINT("File size %08x%08x\n", Map
->FileSizes
.ValidDataLength
.HighPart
, Map
->FileSizes
.ValidDataLength
.LowPart
);
365 if (Map
->FileSizes
.ValidDataLength
.QuadPart
)
367 SectionSize
= min(CACHE_STRIPE
, Map
->FileSizes
.ValidDataLength
.QuadPart
- Target
.QuadPart
);
371 SectionSize
= CACHE_STRIPE
;
374 DPRINT("Allocating a cache stripe at %x:%d\n",
375 Target
.LowPart
, SectionSize
);
376 //ASSERT(SectionSize <= CACHE_STRIPE);
379 Status
= CcpAllocateSection
382 #ifdef PIN_WRITE_ONLY
390 if (!NT_SUCCESS(Status
))
394 DPRINT1("End %08x\n", Status
);
399 /* Returns a reference */
400 DPRINT("Allocating cache sections: %wZ\n", &FileObject
->FileName
);
401 BcbHead
= CcpAllocateCacheSections(FileObject
, SectionObject
);
402 if (BcbHead
== INVALID_CACHE
)
405 DbgPrint("Cache Map:");
406 for (i
= 0; i
< CACHE_NUM_SECTIONS
; i
++)
408 if (!(i
% 64)) DbgPrint("\n");
409 DbgPrint("%c", CcCacheSections
[i
].RefCount
+ (RtlTestBit(CcCacheBitmap
, i
) ? '@' : '`'));
412 KeWaitForSingleObject(&CcDeleteEvent
, Executive
, KernelMode
, FALSE
, NULL
);
416 DPRINT("BcbHead #%x (final)\n", BcbHead
);
418 if (BcbHead
== INVALID_CACHE
)
426 DPRINT("Selected BCB #%x\n", BcbHead
);
427 ULONG ViewSize
= CACHE_STRIPE
;
429 Bcb
= &CcCacheSections
[BcbHead
];
430 Status
= MmMapCacheViewInSystemSpaceAtOffset
436 if (!NT_SUCCESS(Status
))
440 MmFinalizeSegment(SectionObject
);
441 RemoveEntryList(&Bcb
->ThisFileList
);
442 RtlZeroMemory(Bcb
, sizeof(*Bcb
));
443 RtlClearBit(CcCacheBitmap
, BcbHead
);
444 DPRINT1("Failed to map\n");
451 Bcb
->Length
= MIN(Map
->FileSizes
.ValidDataLength
.QuadPart
- Target
.QuadPart
, CACHE_STRIPE
);
452 Bcb
->SectionObject
= SectionObject
;
454 Bcb
->FileOffset
= Target
;
455 InsertTailList(&Map
->AssociatedBcb
, &Bcb
->ThisFileList
);
457 *BcbResult
= &CcCacheSections
[BcbHead
];
458 *Buffer
= ((PCHAR
)Bcb
->BaseAddress
) + (int)(FileOffset
->QuadPart
- Bcb
->FileOffset
.QuadPart
);
462 ("Bcb #%x Buffer maps (%08x%08x) At %x Length %x (Getting %x:%x) %wZ\n",
463 Bcb
- CcCacheSections
,
464 Bcb
->FileOffset
.HighPart
,
465 Bcb
->FileOffset
.LowPart
,
470 &FileObject
->FileName
);
472 EndInterval
.QuadPart
= Bcb
->FileOffset
.QuadPart
+ Bcb
->Length
- 1;
473 ASSERT((EndInterval
.QuadPart
& ~(CACHE_STRIPE
- 1)) == (Bcb
->FileOffset
.QuadPart
& ~(CACHE_STRIPE
- 1)));
483 // Fault in the pages. This forces reads to happen now.
486 PCHAR FaultIn
= Bcb
->BaseAddress
;
488 ("Faulting in pages at this point: file %wZ %08x%08x:%x\n",
489 &FileObject
->FileName
,
490 Bcb
->FileOffset
.HighPart
,
491 Bcb
->FileOffset
.LowPart
,
493 for (i
= 0; i
< Bcb
->Length
; i
++)
498 ASSERT(Bcb
>= CcCacheSections
&& Bcb
< (CcCacheSections
+ CACHE_NUM_SECTIONS
));
511 (IN PFILE_OBJECT FileObject
,
512 IN PLARGE_INTEGER FileOffset
,
515 OUT PVOID
*BcbResult
,
530 PNOCC_BCB Bcb
= (PNOCC_BCB
)*BcbResult
;
531 ASSERT(Bcb
>= CcCacheSections
&& Bcb
< CcCacheSections
+ CACHE_NUM_SECTIONS
);
532 ASSERT(Bcb
->BaseAddress
);
534 CcpReferenceCache(Bcb
- CcCacheSections
);
543 CcpPinMappedData(IN PNOCC_CACHE_MAP Map
,
544 IN PLARGE_INTEGER FileOffset
,
549 BOOLEAN Exclusive
= Flags
& PIN_EXCLUSIVE
;
555 ASSERT(Map
->AssociatedBcb
.Flink
== &Map
->AssociatedBcb
|| (CONTAINING_RECORD(Map
->AssociatedBcb
.Flink
, NOCC_BCB
, ThisFileList
) >= CcCacheSections
&& CONTAINING_RECORD(Map
->AssociatedBcb
.Flink
, NOCC_BCB
, ThisFileList
) < CcCacheSections
+ CACHE_NUM_SECTIONS
));
556 BcbHead
= CcpFindMatchingMap(&Map
->AssociatedBcb
, FileOffset
, Length
);
557 if (BcbHead
== INVALID_CACHE
)
563 TheBcb
= &CcCacheSections
[BcbHead
];
567 DPRINT("Requesting #%x Exclusive\n", BcbHead
);
568 CcpMarkForExclusive(BcbHead
);
572 DPRINT("Reference #%x\n", BcbHead
);
573 CcpReferenceCache(BcbHead
);
577 CcpReferenceCacheExclusive(BcbHead
);
587 CcPinMappedData(IN PFILE_OBJECT FileObject
,
588 IN PLARGE_INTEGER FileOffset
,
594 PNOCC_CACHE_MAP Map
= (PNOCC_CACHE_MAP
)FileObject
->SectionObjectPointer
->SharedCacheMap
;
598 DPRINT1("Not cached\n");
602 if (CcpMapData(FileObject
, FileOffset
, Length
, Flags
, Bcb
, &Buffer
))
604 return CcpPinMappedData(Map
, FileOffset
, Length
, Flags
, Bcb
);
608 DPRINT1("could not map\n");
615 CcPinRead(IN PFILE_OBJECT FileObject
,
616 IN PLARGE_INTEGER FileOffset
,
625 Result
= CcPinMappedData
636 *Buffer
= ((PCHAR
)RealBcb
->BaseAddress
) + (int)(FileOffset
->QuadPart
- RealBcb
->FileOffset
.QuadPart
);
645 CcPreparePinWrite(IN PFILE_OBJECT FileObject
,
646 IN PLARGE_INTEGER FileOffset
,
655 #ifdef PIN_WRITE_ONLY
657 SIZE_T NumberOfBytes
;
661 DPRINT1("CcPreparePinWrite(%x:%x)\n", Buffer
, Length
);
676 #ifdef PIN_WRITE_ONLY
677 BaseAddress
= RealBcb
->BaseAddress
;
678 NumberOfBytes
= RealBcb
->Length
;
680 MiProtectVirtualMemory
689 RealBcb
->Dirty
= TRUE
;
694 ("Zero fill #%x %08x%08x:%x Buffer %x %wZ\n",
695 RealBcb
- CcCacheSections
,
696 FileOffset
->u
.HighPart
,
697 FileOffset
->u
.LowPart
,
700 &FileObject
->FileName
);
702 DPRINT1("RtlZeroMemory(%x,%x)\n", *Buffer
, Length
);
703 RtlZeroMemory(*Buffer
, Length
);
712 CcpUnpinData(IN PNOCC_BCB RealBcb
, BOOLEAN ReleaseBit
)
714 if (RealBcb
->RefCount
<= 2)
716 RealBcb
->Exclusive
= FALSE
;
717 if (RealBcb
->ExclusiveWaiter
)
719 DPRINT("Triggering exclusive waiter\n");
720 KeSetEvent(&RealBcb
->ExclusiveWait
, IO_NO_INCREMENT
, FALSE
);
724 if (RealBcb
->RefCount
== 2 && !ReleaseBit
)
726 if (RealBcb
->RefCount
> 1)
728 DPRINT("Removing one reference #%x\n", RealBcb
- CcCacheSections
);
730 KeSetEvent(&CcDeleteEvent
, IO_DISK_INCREMENT
, FALSE
);
732 if (RealBcb
->RefCount
== 1)
734 DPRINT("Clearing allocation bit #%x\n", RealBcb
- CcCacheSections
);
736 RtlClearBit(CcCacheBitmap
, RealBcb
- CcCacheSections
);
738 #ifdef PIN_WRITE_ONLY
739 PVOID BaseAddress
= RealBcb
->BaseAddress
;
740 SIZE_T NumberOfBytes
= RealBcb
->Length
;
743 MiProtectVirtualMemory
757 CcUnpinData(IN PVOID Bcb
)
759 PNOCC_BCB RealBcb
= (PNOCC_BCB
)Bcb
;
760 ULONG Selected
= RealBcb
- CcCacheSections
;
763 ASSERT(RealBcb
>= CcCacheSections
&& RealBcb
- CcCacheSections
< CACHE_NUM_SECTIONS
);
764 DPRINT("CcUnpinData Bcb #%x (RefCount %d)\n", Selected
, RealBcb
->RefCount
);
767 Released
= CcpUnpinData(RealBcb
, FALSE
);
771 MiFlushMappedSection(RealBcb
->BaseAddress
, &RealBcb
->FileOffset
, &RealBcb
->Map
->FileSizes
.FileSize
, RealBcb
->Dirty
);
773 CcpUnpinData(RealBcb
, TRUE
);
780 CcSetBcbOwnerPointer(IN PVOID Bcb
,
781 IN PVOID OwnerPointer
)
783 PNOCC_BCB RealBcb
= (PNOCC_BCB
)Bcb
;
785 CcpReferenceCache(RealBcb
- CcCacheSections
);
786 RealBcb
->OwnerPointer
= OwnerPointer
;
792 CcUnpinDataForThread(IN PVOID Bcb
,
793 IN ERESOURCE_THREAD ResourceThreadId
)