* Sync up to trunk head (r60691).
[reactos.git] / ntoskrnl / cache / pinsup.c
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/cache/pinsup.c
5 * PURPOSE: Logging and configuration routines
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 * Art Yerkes
8 */
9
10 /* INCLUDES *******************************************************************/
11
12 #include <ntoskrnl.h>
13 #include "newcc.h"
14 #include "section/newmm.h"
15 #define NDEBUG
16 #include <debug.h>
17
18 /* The following is a test mode that only works with modified filesystems.
19 * it maps the cache sections read only until they're pinned writable, and then
20 * turns them readonly again when they're unpinned.
21 * This helped me determine that a certain bug was not a memory overwrite. */
22
23 //#define PIN_WRITE_ONLY
24
25 /*
26
27 Pinsup implements the core of NewCC.
28
29 A couple of things about this code:
30
31 I wrote this code over the course of about 2 years, often referring to Rajeev
32 Nagar's Filesystem Internals, book, the msdn pages on the Cc interface, and
33 a few NT filesystems that are open sourced. I went to fairly great lengths to
34 achieve a couple of goals.
35
36 1) To make a strictly layered facility that relies entirely on Mm to provide
37 maps. There were many ways in which data segments in the legacy Mm were unable
38 to provide what I needed; page maps were only 4 gig, and all offsets were in
39 ULONG, so no mapping at an offset greater than 4 gig was possible. Worse than
40 that, due to a convoluted set of dependencies, it would have been impossible to
41 support any two mappings farther apart than 4 gig, even if the above was
42 corrected. Along with that, the cache system's ownership of some pages was
43 integral to the operation of legacy Mm. All of the above problems, along with
44 an ambiguity about when the size of a file for mapping purposes is acquired,
45 and its inability to allow a file to be resized when any mappings were active
46 led me to rewrite data sections (and all other kinds of sections in the
47 original version), and use that layer to implement the Cc API without regard
48 to any internal, undocumented parts.
49
50 2) To write the simplest possible code that implements the Cc interface as
51 documented. Again this is without regard to any information that might be
52 gained through reverse engineering the real Cc. All conclusions about workings
53 of Cc here are mine, any failures are mine, any differences to the documented
54 interface were introduced by me due to misreading, misunderstanding or mis
55 remembering while implementing the code. I also implemented some obvious, but
56 not actually specified behaviors of Cc, for example that each cache stripe is
57 represented by a distinct BCB that the user can make decisions about as an
58 opaque pointer.
59
60 3) To make real filesystems work properly.
61
62 So about how it works:
63
64 CcCacheSections is the collection of cache sections that are currently mapped.
65 The cache ranges which are allocated and contain pages is larger, due to the
66 addition of sections containing rmaps and page references, but this array
67 determines the actual mapped pages on behalf of all mapped files for Cc's use.
68 All BCB pointers yielded to a driver are a pointer to one of these cache stripe
69 structures. The data structure is specified as opaque and so it contains
70 information convenient to NEWCC's implementation here. Free entries are
71 summarized in CcpBitmapBuffer, for which bits are set when the entry may be
72 safely evicted and redirected for use by another client. Note that the
73 reference count for an evictable cache section will generally be 1, since
74 we'll keep a reference to wait for any subsequent mapping of the same stripe.
75 We use CcCacheClockHand as a hint to start checking free bits at a point that
76 walks around the cache stripe list, so that we might evict a different stripe
77 every time even if all are awaiting reuse. This is a way to avoid thrashing.
78
79 CcpBitmapBuffer is the RTL_BITMAP that allows us to quickly decide what buffer
80 to allocate from the mapped buffer set.
81
82 CcDeleteEvent is an event used to wait for a cache stripe reference count to
83 go to 1, thus making the stripe eligible for eviction. It's used by CcpMapData
84 to wait for a free map when we can't fail.
85
86 All in all, use of Mm by Cc makes this code into a simple manager that wields
87 sections on behalf of filesystems. As such, its code is fairly high level and
88 no architecture specific changes should be necessary.
89
90 */
91
92 /* GLOBALS ********************************************************************/
93
94 #define TAG_MAP_SEC TAG('C', 'c', 'S', 'x')
95 #define TAG_MAP_READ TAG('M', 'c', 'p', 'y')
96 #define TAG_MAP_BCB TAG('B', 'c', 'b', ' ')
97
98 NOCC_BCB CcCacheSections[CACHE_NUM_SECTIONS];
99 CHAR CcpBitmapBuffer[sizeof(RTL_BITMAP) + ROUND_UP((CACHE_NUM_SECTIONS), 32) / 8];
100 PRTL_BITMAP CcCacheBitmap = (PRTL_BITMAP)&CcpBitmapBuffer;
101 FAST_MUTEX CcMutex;
102 KEVENT CcDeleteEvent;
103 KEVENT CcFinalizeEvent;
104 ULONG CcCacheClockHand;
105 LONG CcOutstandingDeletes;
106
107 /* FUNCTIONS ******************************************************************/
108
109 PETHREAD LastThread;
110
111 VOID
112 _CcpLock(const char *file,
113 int line)
114 {
115 //DPRINT("<<<---<<< CC In Mutex(%s:%d %x)!\n", file, line, PsGetCurrentThread());
116 ExAcquireFastMutex(&CcMutex);
117 }
118
119 VOID
120 _CcpUnlock(const char *file,
121 int line)
122 {
123 ExReleaseFastMutex(&CcMutex);
124 //DPRINT(">>>--->>> CC Exit Mutex!\n", file, line);
125 }
126
127 PDEVICE_OBJECT
128 NTAPI
129 MmGetDeviceObjectForFile(IN PFILE_OBJECT FileObject);
130
131 /*
132
133 Allocate an almost ordinary section object for use by the cache system.
134 The special internal SEC_CACHE flag is used to indicate that the section
135 should not count when determining whether the file can be resized.
136
137 */
138
139 NTSTATUS
140 CcpAllocateSection(PFILE_OBJECT FileObject,
141 ULONG Length,
142 ULONG Protect,
143 PROS_SECTION_OBJECT *Result)
144 {
145 NTSTATUS Status;
146 LARGE_INTEGER MaxSize;
147
148 MaxSize.QuadPart = Length;
149
150 DPRINT("Making Section for File %x\n", FileObject);
151 DPRINT("File name %wZ\n", &FileObject->FileName);
152
153 Status = MmCreateSection((PVOID*)Result,
154 STANDARD_RIGHTS_REQUIRED,
155 NULL,
156 &MaxSize,
157 Protect,
158 SEC_RESERVE | SEC_CACHE,
159 NULL,
160 FileObject);
161
162 return Status;
163 }
164
165 typedef struct _WORK_QUEUE_WITH_CONTEXT
166 {
167 WORK_QUEUE_ITEM WorkItem;
168 PVOID ToUnmap;
169 LARGE_INTEGER FileOffset;
170 LARGE_INTEGER MapSize;
171 PROS_SECTION_OBJECT ToDeref;
172 PACQUIRE_FOR_LAZY_WRITE AcquireForLazyWrite;
173 PRELEASE_FROM_LAZY_WRITE ReleaseFromLazyWrite;
174 PVOID LazyContext;
175 BOOLEAN Dirty;
176 } WORK_QUEUE_WITH_CONTEXT, *PWORK_QUEUE_WITH_CONTEXT;
177
178 /*
179
180 Unmap a cache stripe. Note that cache stripes aren't unmapped when their
181 last reference disappears. We enter this code only if cache for the file
182 is uninitialized in the last file object, or a cache stripe is evicted.
183
184 */
185
186 VOID
187 CcpUnmapCache(PVOID Context)
188 {
189 PWORK_QUEUE_WITH_CONTEXT WorkItem = (PWORK_QUEUE_WITH_CONTEXT)Context;
190 DPRINT("Unmapping (finally) %x\n", WorkItem->ToUnmap);
191 MmUnmapCacheViewInSystemSpace(WorkItem->ToUnmap);
192 ObDereferenceObject(WorkItem->ToDeref);
193 ExFreePool(WorkItem);
194 DPRINT("Done\n");
195 }
196
197 /*
198
199 Somewhat deceptively named function which removes the last reference to a
200 cache stripe and completely removes it using CcUnmapCache. This may be
201 done either inline (if the Immediate BOOLEAN is set), or using a work item
202 at a later time. Whether this is called to unmap immeidately is mainly
203 determined by whether the caller is calling from a place in filesystem code
204 where a deadlock may occur if immediate flushing is required.
205
206 It's always safe to reuse the Bcb at CcCacheSections[Start] after calling
207 this.
208
209 */
210
211 /* Must have acquired the mutex */
212 VOID
213 CcpDereferenceCache(ULONG Start,
214 BOOLEAN Immediate)
215 {
216 PVOID ToUnmap;
217 PNOCC_BCB Bcb;
218 BOOLEAN Dirty;
219 LARGE_INTEGER MappedSize;
220 LARGE_INTEGER BaseOffset;
221 PWORK_QUEUE_WITH_CONTEXT WorkItem;
222
223 DPRINT("CcpDereferenceCache(#%x)\n", Start);
224
225 Bcb = &CcCacheSections[Start];
226
227 Dirty = Bcb->Dirty;
228 ToUnmap = Bcb->BaseAddress;
229 BaseOffset = Bcb->FileOffset;
230 MappedSize = Bcb->Map->FileSizes.ValidDataLength;
231
232 DPRINT("Dereference #%x (count %d)\n", Start, Bcb->RefCount);
233 ASSERT(Bcb->SectionObject);
234 ASSERT(Bcb->RefCount == 1);
235
236 DPRINT("Firing work item for %x\n", Bcb->BaseAddress);
237
238 if (Dirty) {
239 CcpUnlock();
240 Bcb->RefCount++;
241 MiFlushMappedSection(ToUnmap, &BaseOffset, &MappedSize, Dirty);
242 Bcb->RefCount--;
243 CcpLock();
244 }
245
246 if (Immediate)
247 {
248 PROS_SECTION_OBJECT ToDeref = Bcb->SectionObject;
249 Bcb->Map = NULL;
250 Bcb->SectionObject = NULL;
251 Bcb->BaseAddress = NULL;
252 Bcb->FileOffset.QuadPart = 0;
253 Bcb->Length = 0;
254 Bcb->RefCount = 0;
255 Bcb->Dirty = FALSE;
256 RemoveEntryList(&Bcb->ThisFileList);
257
258 CcpUnlock();
259 MmUnmapCacheViewInSystemSpace(ToUnmap);
260 ObDereferenceObject(ToDeref);
261 CcpLock();
262 }
263 else
264 {
265 WorkItem = ExAllocatePool(NonPagedPool, sizeof(*WorkItem));
266 if (!WorkItem) KeBugCheck(0);
267 WorkItem->ToUnmap = Bcb->BaseAddress;
268 WorkItem->FileOffset = Bcb->FileOffset;
269 WorkItem->Dirty = Bcb->Dirty;
270 WorkItem->MapSize = MappedSize;
271 WorkItem->ToDeref = Bcb->SectionObject;
272 WorkItem->AcquireForLazyWrite = Bcb->Map->Callbacks.AcquireForLazyWrite;
273 WorkItem->ReleaseFromLazyWrite = Bcb->Map->Callbacks.ReleaseFromLazyWrite;
274 WorkItem->LazyContext = Bcb->Map->LazyContext;
275
276 ExInitializeWorkItem(((PWORK_QUEUE_ITEM)WorkItem),
277 (PWORKER_THREAD_ROUTINE)CcpUnmapCache,
278 WorkItem);
279
280 Bcb->Map = NULL;
281 Bcb->SectionObject = NULL;
282 Bcb->BaseAddress = NULL;
283 Bcb->FileOffset.QuadPart = 0;
284 Bcb->Length = 0;
285 Bcb->RefCount = 0;
286 Bcb->Dirty = FALSE;
287 RemoveEntryList(&Bcb->ThisFileList);
288
289 CcpUnlock();
290 ExQueueWorkItem((PWORK_QUEUE_ITEM)WorkItem, DelayedWorkQueue);
291 CcpLock();
292 }
293 DPRINT("Done\n");
294 }
295
296 /*
297
298 CcpAllocateCacheSections is called by CcpMapData to obtain a cache stripe,
299 possibly evicting an old stripe by calling CcpDereferenceCache in order to
300 obtain an empty Bcb.
301
302 This function was named plural due to a question I had at the beginning of
303 this endeavor about whether a map may span a 256k stripe boundary. It can't
304 so this function can only return the index of one Bcb. Returns INVALID_CACHE
305 on failure.
306
307 */
308 /* Needs mutex */
309 ULONG
310 CcpAllocateCacheSections(PFILE_OBJECT FileObject,
311 PROS_SECTION_OBJECT SectionObject)
312 {
313 ULONG i = INVALID_CACHE;
314 PNOCC_CACHE_MAP Map;
315 PNOCC_BCB Bcb;
316
317 DPRINT("AllocateCacheSections: FileObject %x\n", FileObject);
318
319 if (!FileObject->SectionObjectPointer)
320 return INVALID_CACHE;
321
322 Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap;
323
324 if (!Map)
325 return INVALID_CACHE;
326
327 DPRINT("Allocating Cache Section\n");
328
329 i = RtlFindClearBitsAndSet(CcCacheBitmap, 1, CcCacheClockHand);
330 CcCacheClockHand = (i + 1) % CACHE_NUM_SECTIONS;
331
332 if (i != INVALID_CACHE)
333 {
334 DPRINT("Setting up Bcb #%x\n", i);
335
336 Bcb = &CcCacheSections[i];
337
338 ASSERT(Bcb->RefCount < 2);
339
340 if (Bcb->RefCount > 0)
341 {
342 CcpDereferenceCache(i, FALSE);
343 }
344
345 ASSERT(!Bcb->RefCount);
346 Bcb->RefCount = 1;
347
348 DPRINT("Bcb #%x RefCount %d\n", Bcb - CcCacheSections, Bcb->RefCount);
349
350 if (!RtlTestBit(CcCacheBitmap, i))
351 {
352 DPRINT1("Somebody stoeled BCB #%x\n", i);
353 }
354 ASSERT(RtlTestBit(CcCacheBitmap, i));
355
356 DPRINT("Allocated #%x\n", i);
357 ASSERT(CcCacheSections[i].RefCount);
358 }
359 else
360 {
361 DPRINT1("Failed to allocate cache segment\n");
362 }
363 return i;
364 }
365
366 /* Must have acquired the mutex */
367 VOID
368 CcpReferenceCache(ULONG Start)
369 {
370 PNOCC_BCB Bcb;
371 Bcb = &CcCacheSections[Start];
372 ASSERT(Bcb->SectionObject);
373 Bcb->RefCount++;
374 RtlSetBit(CcCacheBitmap, Start);
375
376 }
377
378 VOID
379 CcpMarkForExclusive(ULONG Start)
380 {
381 PNOCC_BCB Bcb;
382 Bcb = &CcCacheSections[Start];
383 Bcb->ExclusiveWaiter++;
384 }
385
386 /*
387
388 Cache stripes have an idea of exclusive access, which would be hard to support
389 properly in the previous code. In our case, it's fairly easy, since we have
390 an event that indicates that the previous exclusive waiter has returned in each
391 Bcb.
392
393 */
394 /* Must not have the mutex */
395 VOID
396 CcpReferenceCacheExclusive(ULONG Start)
397 {
398 PNOCC_BCB Bcb = &CcCacheSections[Start];
399
400 KeWaitForSingleObject(&Bcb->ExclusiveWait,
401 Executive,
402 KernelMode,
403 FALSE,
404 NULL);
405
406 CcpLock();
407 ASSERT(Bcb->ExclusiveWaiter);
408 ASSERT(Bcb->SectionObject);
409 Bcb->Exclusive = TRUE;
410 Bcb->ExclusiveWaiter--;
411 RtlSetBit(CcCacheBitmap, Start);
412 CcpUnlock();
413 }
414
415 /*
416
417 Find a map that encompasses the target range. This function does not check
418 whether the desired range is partly outside the stripe. This could be
419 implemented with a generic table, but we generally aren't carring around a lot
420 of segments at once for a particular file.
421
422 When this returns a map for a given file address, then that address is by
423 definition already mapped and can be operated on.
424
425 Returns a valid index or INVALID_CACHE.
426
427 */
428 /* Must have the mutex */
429 ULONG
430 CcpFindMatchingMap(PLIST_ENTRY Head,
431 PLARGE_INTEGER FileOffset,
432 ULONG Length)
433 {
434 PLIST_ENTRY Entry;
435 //DPRINT("Find Matching Map: (%x) %x:%x\n", FileOffset->LowPart, Length);
436 for (Entry = Head->Flink; Entry != Head; Entry = Entry->Flink)
437 {
438 //DPRINT("Link @%x\n", Entry);
439 PNOCC_BCB Bcb = CONTAINING_RECORD(Entry, NOCC_BCB, ThisFileList);
440 //DPRINT("Selected BCB %x #%x\n", Bcb, Bcb - CcCacheSections);
441 //DPRINT("This File: %x:%x\n", Bcb->FileOffset.LowPart, Bcb->Length);
442 if (FileOffset->QuadPart >= Bcb->FileOffset.QuadPart &&
443 FileOffset->QuadPart < Bcb->FileOffset.QuadPart + CACHE_STRIPE)
444 {
445 //DPRINT("Found match at #%x\n", Bcb - CcCacheSections);
446 return Bcb - CcCacheSections;
447 }
448 }
449
450 //DPRINT("This region isn't mapped\n");
451
452 return INVALID_CACHE;
453 }
454
455 /*
456
457 Internal function that's used by all pinning functions.
458 It causes a mapped region to exist and prefaults the pages in it if possible,
459 possibly evicting another stripe in order to get our stripe.
460
461 */
462
463 BOOLEAN
464 NTAPI
465 CcpMapData(IN PFILE_OBJECT FileObject,
466 IN PLARGE_INTEGER FileOffset,
467 IN ULONG Length,
468 IN ULONG Flags,
469 OUT PVOID *BcbResult,
470 OUT PVOID *Buffer)
471 {
472 BOOLEAN Success = FALSE, FaultIn = FALSE;
473 /* Note: windows 2000 drivers treat this as a bool */
474 //BOOLEAN Wait = (Flags & MAP_WAIT) || (Flags == TRUE);
475 LARGE_INTEGER Target, EndInterval;
476 ULONG BcbHead, SectionSize, ViewSize;
477 PNOCC_BCB Bcb = NULL;
478 PROS_SECTION_OBJECT SectionObject = NULL;
479 NTSTATUS Status;
480 PNOCC_CACHE_MAP Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap;
481 ViewSize = CACHE_STRIPE;
482
483 if (!Map)
484 {
485 DPRINT1("File object was not mapped\n");
486 return FALSE;
487 }
488
489 DPRINT("CcMapData(F->%x,%08x%08x:%d)\n",
490 FileObject,
491 FileOffset->HighPart,
492 FileOffset->LowPart,
493 Length);
494
495 ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL);
496
497 Target.HighPart = FileOffset->HighPart;
498 Target.LowPart = CACHE_ROUND_DOWN(FileOffset->LowPart);
499
500 CcpLock();
501
502 /* Find out if any range is a superset of what we want */
503 /* Find an accomodating section */
504 BcbHead = CcpFindMatchingMap(&Map->AssociatedBcb, FileOffset, Length);
505
506 if (BcbHead != INVALID_CACHE)
507 {
508 Bcb = &CcCacheSections[BcbHead];
509 Success = TRUE;
510 *BcbResult = Bcb;
511 *Buffer = ((PCHAR)Bcb->BaseAddress) + (int)(FileOffset->QuadPart - Bcb->FileOffset.QuadPart);
512
513 DPRINT("Bcb #%x Buffer maps (%08x%08x) At %x Length %x (Getting %x:%x) %wZ\n",
514 Bcb - CcCacheSections,
515 Bcb->FileOffset.HighPart,
516 Bcb->FileOffset.LowPart,
517 Bcb->BaseAddress,
518 Bcb->Length,
519 *Buffer,
520 Length,
521 &FileObject->FileName);
522
523 DPRINT("w1n\n");
524 goto cleanup;
525 }
526
527 DPRINT("File size %08x%08x\n",
528 Map->FileSizes.ValidDataLength.HighPart,
529 Map->FileSizes.ValidDataLength.LowPart);
530
531 /* Not all files have length, in fact filesystems often use stream file
532 objects for various internal purposes and are loose about the file
533 length, since the filesystem promises itself to write the right number
534 of bytes to the internal stream. In these cases, we just allow the file
535 to have the full stripe worth of space. */
536 if (Map->FileSizes.ValidDataLength.QuadPart)
537 {
538 SectionSize = min(CACHE_STRIPE,
539 Map->FileSizes.ValidDataLength.QuadPart - Target.QuadPart);
540 }
541 else
542 {
543 SectionSize = CACHE_STRIPE;
544 }
545
546 DPRINT("Allocating a cache stripe at %x:%d\n",
547 Target.LowPart, SectionSize);
548
549 //ASSERT(SectionSize <= CACHE_STRIPE);
550
551 CcpUnlock();
552 /* CcpAllocateSection doesn't need the lock, so we'll give other action
553 a chance in here. */
554 Status = CcpAllocateSection(FileObject,
555 SectionSize,
556 #ifdef PIN_WRITE_ONLY
557 PAGE_READONLY,
558 #else
559 PAGE_READWRITE,
560 #endif
561 &SectionObject);
562 CcpLock();
563
564 if (!NT_SUCCESS(Status))
565 {
566 *BcbResult = NULL;
567 *Buffer = NULL;
568 DPRINT1("End %08x\n", Status);
569 goto cleanup;
570 }
571
572 retry:
573 /* Returns a reference */
574 DPRINT("Allocating cache sections: %wZ\n", &FileObject->FileName);
575 BcbHead = CcpAllocateCacheSections(FileObject, SectionObject);
576 /* XXX todo: we should handle the immediate fail case here, but don't */
577 if (BcbHead == INVALID_CACHE)
578 {
579 ULONG i;
580 DbgPrint("Cache Map:");
581 for (i = 0; i < CACHE_NUM_SECTIONS; i++)
582 {
583 if (!(i % 64)) DbgPrint("\n");
584 DbgPrint("%c",
585 CcCacheSections[i].RefCount + (RtlTestBit(CcCacheBitmap, i) ? '@' : '`'));
586 }
587 DbgPrint("\n");
588
589 KeWaitForSingleObject(&CcDeleteEvent,
590 Executive,
591 KernelMode,
592 FALSE,
593 NULL);
594
595 goto retry;
596 }
597
598 DPRINT("BcbHead #%x (final)\n", BcbHead);
599
600 if (BcbHead == INVALID_CACHE)
601 {
602 *BcbResult = NULL;
603 *Buffer = NULL;
604 DPRINT1("End\n");
605 goto cleanup;
606 }
607
608 DPRINT("Selected BCB #%x\n", BcbHead);
609 ViewSize = CACHE_STRIPE;
610
611 Bcb = &CcCacheSections[BcbHead];
612 /* MmMapCacheViewInSystemSpaceAtOffset is one of three methods of Mm
613 that are specific to NewCC. In this case, it's implementation
614 exactly mirrors MmMapViewInSystemSpace, but allows an offset to
615 be specified. */
616 Status = MmMapCacheViewInSystemSpaceAtOffset(SectionObject->Segment,
617 &Bcb->BaseAddress,
618 &Target,
619 &ViewSize);
620
621 /* Summary: Failure. Dereference our section and tell the user we failed */
622 if (!NT_SUCCESS(Status))
623 {
624 *BcbResult = NULL;
625 *Buffer = NULL;
626 ObDereferenceObject(SectionObject);
627 RemoveEntryList(&Bcb->ThisFileList);
628 RtlZeroMemory(Bcb, sizeof(*Bcb));
629 RtlClearBit(CcCacheBitmap, BcbHead);
630 DPRINT1("Failed to map\n");
631 goto cleanup;
632 }
633
634 /* Summary: Success. Put together a valid Bcb and link it with the others
635 * in the NOCC_CACHE_MAP.
636 */
637 Success = TRUE;
638 //DPRINT("w1n\n");
639
640 Bcb->Length = MIN(Map->FileSizes.ValidDataLength.QuadPart - Target.QuadPart,
641 CACHE_STRIPE);
642
643 Bcb->SectionObject = SectionObject;
644 Bcb->Map = Map;
645 Bcb->FileOffset = Target;
646 InsertTailList(&Map->AssociatedBcb, &Bcb->ThisFileList);
647
648 *BcbResult = &CcCacheSections[BcbHead];
649 *Buffer = ((PCHAR)Bcb->BaseAddress) + (int)(FileOffset->QuadPart - Bcb->FileOffset.QuadPart);
650 FaultIn = TRUE;
651
652 DPRINT("Bcb #%x Buffer maps (%08x%08x) At %x Length %x (Getting %x:%x) %wZ\n",
653 Bcb - CcCacheSections,
654 Bcb->FileOffset.HighPart,
655 Bcb->FileOffset.LowPart,
656 Bcb->BaseAddress,
657 Bcb->Length,
658 *Buffer,
659 Length,
660 &FileObject->FileName);
661
662 EndInterval.QuadPart = Bcb->FileOffset.QuadPart + Bcb->Length - 1;
663 ASSERT((EndInterval.QuadPart & ~(CACHE_STRIPE - 1)) ==
664 (Bcb->FileOffset.QuadPart & ~(CACHE_STRIPE - 1)));
665
666 //DPRINT("TERM!\n");
667
668 cleanup:
669 CcpUnlock();
670 if (Success)
671 {
672 if (FaultIn)
673 {
674 /* Fault in the pages. This forces reads to happen now. */
675 ULONG i;
676 PCHAR FaultIn = Bcb->BaseAddress;
677
678 DPRINT("Faulting in pages at this point: file %wZ %08x%08x:%x\n",
679 &FileObject->FileName,
680 Bcb->FileOffset.HighPart,
681 Bcb->FileOffset.LowPart,
682 Bcb->Length);
683
684 for (i = 0; i < Bcb->Length; i += PAGE_SIZE)
685 {
686 FaultIn[i] ^= 0;
687 }
688 }
689 ASSERT(Bcb >= CcCacheSections &&
690 Bcb < (CcCacheSections + CACHE_NUM_SECTIONS));
691 }
692 else
693 {
694 ASSERT(FALSE);
695 }
696
697 return Success;
698 }
699
700 BOOLEAN
701 NTAPI
702 CcMapData(IN PFILE_OBJECT FileObject,
703 IN PLARGE_INTEGER FileOffset,
704 IN ULONG Length,
705 IN ULONG Flags,
706 OUT PVOID *BcbResult,
707 OUT PVOID *Buffer)
708 {
709 BOOLEAN Result;
710
711 Result = CcpMapData(FileObject,
712 FileOffset,
713 Length,
714 Flags,
715 BcbResult,
716 Buffer);
717
718 if (Result)
719 {
720 PNOCC_BCB Bcb = (PNOCC_BCB)*BcbResult;
721
722 ASSERT(Bcb >= CcCacheSections &&
723 Bcb < CcCacheSections + CACHE_NUM_SECTIONS);
724
725 ASSERT(Bcb->BaseAddress);
726 CcpLock();
727 CcpReferenceCache(Bcb - CcCacheSections);
728 CcpUnlock();
729 }
730
731 return Result;
732 }
733
734 /* Used by functions that repin data, CcpPinMappedData does not alter the map,
735 but finds the appropriate stripe and update the accounting. */
736 BOOLEAN
737 NTAPI
738 CcpPinMappedData(IN PNOCC_CACHE_MAP Map,
739 IN PLARGE_INTEGER FileOffset,
740 IN ULONG Length,
741 IN ULONG Flags,
742 IN OUT PVOID *Bcb)
743 {
744 BOOLEAN Exclusive = Flags & PIN_EXCLUSIVE;
745 ULONG BcbHead;
746 PNOCC_BCB TheBcb;
747
748 CcpLock();
749
750 ASSERT(Map->AssociatedBcb.Flink == &Map->AssociatedBcb || (CONTAINING_RECORD(Map->AssociatedBcb.Flink, NOCC_BCB, ThisFileList) >= CcCacheSections && CONTAINING_RECORD(Map->AssociatedBcb.Flink, NOCC_BCB, ThisFileList) < CcCacheSections + CACHE_NUM_SECTIONS));
751 BcbHead = CcpFindMatchingMap(&Map->AssociatedBcb, FileOffset, Length);
752 if (BcbHead == INVALID_CACHE)
753 {
754 CcpUnlock();
755 return FALSE;
756 }
757
758 TheBcb = &CcCacheSections[BcbHead];
759
760 if (Exclusive)
761 {
762 DPRINT("Requesting #%x Exclusive\n", BcbHead);
763 CcpMarkForExclusive(BcbHead);
764 }
765 else
766 {
767 DPRINT("Reference #%x\n", BcbHead);
768 CcpReferenceCache(BcbHead);
769 }
770
771 if (Exclusive)
772 CcpReferenceCacheExclusive(BcbHead);
773
774 CcpUnlock();
775
776 *Bcb = TheBcb;
777 return TRUE;
778 }
779
780 BOOLEAN
781 NTAPI
782 CcPinMappedData(IN PFILE_OBJECT FileObject,
783 IN PLARGE_INTEGER FileOffset,
784 IN ULONG Length,
785 IN ULONG Flags,
786 IN OUT PVOID *Bcb)
787 {
788 PVOID Buffer;
789 PNOCC_CACHE_MAP Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap;
790
791 if (!Map)
792 {
793 DPRINT1("Not cached\n");
794 return FALSE;
795 }
796
797 if (CcpMapData(FileObject, FileOffset, Length, Flags, Bcb, &Buffer))
798 {
799 return CcpPinMappedData(Map, FileOffset, Length, Flags, Bcb);
800 }
801 else
802 {
803 DPRINT1("could not map\n");
804 return FALSE;
805 }
806 }
807
808 BOOLEAN
809 NTAPI
810 CcPinRead(IN PFILE_OBJECT FileObject,
811 IN PLARGE_INTEGER FileOffset,
812 IN ULONG Length,
813 IN ULONG Flags,
814 OUT PVOID *Bcb,
815 OUT PVOID *Buffer)
816 {
817 PNOCC_BCB RealBcb;
818 BOOLEAN Result;
819
820 Result = CcPinMappedData(FileObject, FileOffset, Length, Flags, Bcb);
821
822 if (Result)
823 {
824 CcpLock();
825 RealBcb = *Bcb;
826 *Buffer = ((PCHAR)RealBcb->BaseAddress) + (int)(FileOffset->QuadPart - RealBcb->FileOffset.QuadPart);
827 CcpUnlock();
828 }
829
830 return Result;
831 }
832
833 BOOLEAN
834 NTAPI
835 CcPreparePinWrite(IN PFILE_OBJECT FileObject,
836 IN PLARGE_INTEGER FileOffset,
837 IN ULONG Length,
838 IN BOOLEAN Zero,
839 IN ULONG Flags,
840 OUT PVOID *Bcb,
841 OUT PVOID *Buffer)
842 {
843 BOOLEAN Result;
844 PNOCC_BCB RealBcb;
845 #ifdef PIN_WRITE_ONLY
846 PVOID BaseAddress;
847 SIZE_T NumberOfBytes;
848 ULONG OldProtect;
849 #endif
850
851 DPRINT("CcPreparePinWrite(%x:%x)\n", Buffer, Length);
852
853 Result = CcPinRead(FileObject, FileOffset, Length, Flags, Bcb, Buffer);
854
855 if (Result)
856 {
857 CcpLock();
858 RealBcb = *Bcb;
859
860 #ifdef PIN_WRITE_ONLY
861 BaseAddress = RealBcb->BaseAddress;
862 NumberOfBytes = RealBcb->Length;
863
864 MiProtectVirtualMemory(NULL,
865 &BaseAddress,
866 &NumberOfBytes,
867 PAGE_READWRITE,
868 &OldProtect);
869 #endif
870
871 CcpUnlock();
872 RealBcb->Dirty = TRUE;
873
874 if (Zero)
875 {
876 DPRINT("Zero fill #%x %08x%08x:%x Buffer %x %wZ\n",
877 RealBcb - CcCacheSections,
878 FileOffset->u.HighPart,
879 FileOffset->u.LowPart,
880 Length,
881 *Buffer,
882 &FileObject->FileName);
883
884 DPRINT1("RtlZeroMemory(%x,%x)\n", *Buffer, Length);
885 RtlZeroMemory(*Buffer, Length);
886 }
887 }
888
889 return Result;
890 }
891
892 /*
893
894 CcpUnpinData is the internal function that generally handles unpinning data.
895 It may be a little confusing, because of the way reference counts are handled.
896
897 A reference count of 2 or greater means that the stripe is still fully pinned
898 and can't be removed. If the owner had taken an exclusive reference, then
899 give one up. Note that it's an error to take more than one exclusive reference
900 or to take a non-exclusive reference after an exclusive reference, so detecting
901 or handling that case is not considered.
902
903 ReleaseBit is unset if we want to detect when a cache stripe would become
904 evictable without actually giving up our reference. We might want to do that
905 if we were going to flush before formally releasing the cache stripe, although
906 that facility is not used meaningfully at this time.
907
908 A reference count of exactly 1 means that the stripe could potentially be
909 reused, but could also be evicted for another mapping. In general, most
910 stripes should be in that state most of the time.
911
912 A reference count of zero means that the Bcb is completely unused. That's the
913 start state and the state of a Bcb formerly owned by a file that is
914 uninitialized.
915
916 */
917
918 BOOLEAN
919 NTAPI
920 CcpUnpinData(IN PNOCC_BCB RealBcb, BOOLEAN ReleaseBit)
921 {
922 if (RealBcb->RefCount <= 2)
923 {
924 RealBcb->Exclusive = FALSE;
925 if (RealBcb->ExclusiveWaiter)
926 {
927 DPRINT("Triggering exclusive waiter\n");
928 KeSetEvent(&RealBcb->ExclusiveWait, IO_NO_INCREMENT, FALSE);
929 return TRUE;
930 }
931 }
932 if (RealBcb->RefCount == 2 && !ReleaseBit)
933 return FALSE;
934 if (RealBcb->RefCount > 1)
935 {
936 DPRINT("Removing one reference #%x\n", RealBcb - CcCacheSections);
937 RealBcb->RefCount--;
938 KeSetEvent(&CcDeleteEvent, IO_DISK_INCREMENT, FALSE);
939 }
940 if (RealBcb->RefCount == 1)
941 {
942 DPRINT("Clearing allocation bit #%x\n", RealBcb - CcCacheSections);
943
944 RtlClearBit(CcCacheBitmap, RealBcb - CcCacheSections);
945
946 #ifdef PIN_WRITE_ONLY
947 PVOID BaseAddress = RealBcb->BaseAddress;
948 SIZE_T NumberOfBytes = RealBcb->Length;
949 ULONG OldProtect;
950
951 MiProtectVirtualMemory(NULL,
952 &BaseAddress,
953 &NumberOfBytes,
954 PAGE_READONLY,
955 &OldProtect);
956 #endif
957 }
958
959 return TRUE;
960 }
961
962 VOID
963 NTAPI
964 CcUnpinData(IN PVOID Bcb)
965 {
966 PNOCC_BCB RealBcb = (PNOCC_BCB)Bcb;
967 ULONG Selected = RealBcb - CcCacheSections;
968 BOOLEAN Released;
969
970 ASSERT(RealBcb >= CcCacheSections &&
971 RealBcb - CcCacheSections < CACHE_NUM_SECTIONS);
972
973 DPRINT("CcUnpinData Bcb #%x (RefCount %d)\n", Selected, RealBcb->RefCount);
974
975 CcpLock();
976 Released = CcpUnpinData(RealBcb, FALSE);
977 CcpUnlock();
978
979 if (!Released) {
980 CcpLock();
981 CcpUnpinData(RealBcb, TRUE);
982 CcpUnlock();
983 }
984 }
985
986 VOID
987 NTAPI
988 CcSetBcbOwnerPointer(IN PVOID Bcb,
989 IN PVOID OwnerPointer)
990 {
991 PNOCC_BCB RealBcb = (PNOCC_BCB)Bcb;
992 CcpLock();
993 CcpReferenceCache(RealBcb - CcCacheSections);
994 RealBcb->OwnerPointer = OwnerPointer;
995 CcpUnlock();
996 }
997
998 VOID
999 NTAPI
1000 CcUnpinDataForThread(IN PVOID Bcb,
1001 IN ERESOURCE_THREAD ResourceThreadId)
1002 {
1003 CcUnpinData(Bcb);
1004 }
1005
1006 /* EOF */