[STORPORT] Fix x64 build
[reactos.git] / ntoskrnl / cache / pinsup.c
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS Kernel
4 * FILE: ntoskrnl/cache/pinsup.c
5 * PURPOSE: Logging and configuration routines
6 * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
7 * Art Yerkes
8 */
9
10 /* INCLUDES *******************************************************************/
11
12 #include <ntoskrnl.h>
13 #include "newcc.h"
14 #include "section/newmm.h"
15 #define NDEBUG
16 #include <debug.h>
17
18 /* The following is a test mode that only works with modified filesystems.
19 * it maps the cache sections read only until they're pinned writable, and then
20 * turns them readonly again when they're unpinned.
21 * This helped me determine that a certain bug was not a memory overwrite. */
22
23 //#define PIN_WRITE_ONLY
24
25 /*
26
27 Pinsup implements the core of NewCC.
28
29 A couple of things about this code:
30
31 I wrote this code over the course of about 2 years, often referring to Rajeev
32 Nagar's Filesystem Internals, book, the msdn pages on the Cc interface, and
33 a few NT filesystems that are open sourced. I went to fairly great lengths to
34 achieve a couple of goals.
35
36 1) To make a strictly layered facility that relies entirely on Mm to provide
37 maps. There were many ways in which data segments in the legacy Mm were unable
38 to provide what I needed; page maps were only 4 gig, and all offsets were in
39 ULONG, so no mapping at an offset greater than 4 gig was possible. Worse than
40 that, due to a convoluted set of dependencies, it would have been impossible to
41 support any two mappings farther apart than 4 gig, even if the above was
42 corrected. Along with that, the cache system's ownership of some pages was
43 integral to the operation of legacy Mm. All of the above problems, along with
44 an ambiguity about when the size of a file for mapping purposes is acquired,
45 and its inability to allow a file to be resized when any mappings were active
46 led me to rewrite data sections (and all other kinds of sections in the
47 original version), and use that layer to implement the Cc API without regard
48 to any internal, undocumented parts.
49
50 2) To write the simplest possible code that implements the Cc interface as
51 documented. Again this is without regard to any information that might be
52 gained through reverse engineering the real Cc. All conclusions about workings
53 of Cc here are mine, any failures are mine, any differences to the documented
54 interface were introduced by me due to misreading, misunderstanding or mis
55 remembering while implementing the code. I also implemented some obvious, but
56 not actually specified behaviors of Cc, for example that each cache stripe is
57 represented by a distinct BCB that the user can make decisions about as an
58 opaque pointer.
59
60 3) To make real filesystems work properly.
61
62 So about how it works:
63
64 CcCacheSections is the collection of cache sections that are currently mapped.
65 The cache ranges which are allocated and contain pages is larger, due to the
66 addition of sections containing rmaps and page references, but this array
67 determines the actual mapped pages on behalf of all mapped files for Cc's use.
68 All BCB pointers yielded to a driver are a pointer to one of these cache stripe
69 structures. The data structure is specified as opaque and so it contains
70 information convenient to NEWCC's implementation here. Free entries are
71 summarized in CcpBitmapBuffer, for which bits are set when the entry may be
72 safely evicted and redirected for use by another client. Note that the
73 reference count for an evictable cache section will generally be 1, since
74 we'll keep a reference to wait for any subsequent mapping of the same stripe.
75 We use CcCacheClockHand as a hint to start checking free bits at a point that
76 walks around the cache stripe list, so that we might evict a different stripe
77 every time even if all are awaiting reuse. This is a way to avoid thrashing.
78
79 CcpBitmapBuffer is the RTL_BITMAP that allows us to quickly decide what buffer
80 to allocate from the mapped buffer set.
81
82 CcDeleteEvent is an event used to wait for a cache stripe reference count to
83 go to 1, thus making the stripe eligible for eviction. It's used by CcpMapData
84 to wait for a free map when we can't fail.
85
86 All in all, use of Mm by Cc makes this code into a simple manager that wields
87 sections on behalf of filesystems. As such, its code is fairly high level and
88 no architecture specific changes should be necessary.
89
90 */
91
92 /* GLOBALS ********************************************************************/
93
94 #define TAG_MAP_SEC TAG('C', 'c', 'S', 'x')
95 #define TAG_MAP_READ TAG('M', 'c', 'p', 'y')
96 #define TAG_MAP_BCB TAG('B', 'c', 'b', ' ')
97
98 NOCC_BCB CcCacheSections[CACHE_NUM_SECTIONS];
99 CHAR CcpBitmapBuffer[sizeof(RTL_BITMAP) + ROUND_UP((CACHE_NUM_SECTIONS), 32) / 8];
100 PRTL_BITMAP CcCacheBitmap = (PRTL_BITMAP)&CcpBitmapBuffer;
101 FAST_MUTEX CcMutex;
102 KEVENT CcDeleteEvent;
103 KEVENT CcFinalizeEvent;
104 ULONG CcCacheClockHand;
105 LONG CcOutstandingDeletes;
106
107 /* FUNCTIONS ******************************************************************/
108
109 PETHREAD LastThread;
110
111 VOID
112 _CcpLock(const char *file,
113 int line)
114 {
115 //DPRINT("<<<---<<< CC In Mutex(%s:%d %x)!\n", file, line, PsGetCurrentThread());
116 ExAcquireFastMutex(&CcMutex);
117 }
118
119 VOID
120 _CcpUnlock(const char *file,
121 int line)
122 {
123 ExReleaseFastMutex(&CcMutex);
124 //DPRINT(">>>--->>> CC Exit Mutex!\n", file, line);
125 }
126
127 PDEVICE_OBJECT
128 NTAPI
129 MmGetDeviceObjectForFile(IN PFILE_OBJECT FileObject);
130
131 /*
132
133 Allocate an almost ordinary section object for use by the cache system.
134 The special internal SEC_CACHE flag is used to indicate that the section
135 should not count when determining whether the file can be resized.
136
137 */
138
139 NTSTATUS
140 CcpAllocateSection(PFILE_OBJECT FileObject,
141 ULONG Length,
142 ULONG Protect,
143 PROS_SECTION_OBJECT *Result)
144 {
145 NTSTATUS Status;
146 LARGE_INTEGER MaxSize;
147
148 MaxSize.QuadPart = Length;
149
150 DPRINT("Making Section for File %x\n", FileObject);
151 DPRINT("File name %wZ\n", &FileObject->FileName);
152
153 Status = MmCreateSection((PVOID*)Result,
154 STANDARD_RIGHTS_REQUIRED,
155 NULL,
156 &MaxSize,
157 Protect,
158 SEC_RESERVE | SEC_CACHE,
159 NULL,
160 FileObject);
161
162 return Status;
163 }
164
165 typedef struct _WORK_QUEUE_WITH_CONTEXT
166 {
167 WORK_QUEUE_ITEM WorkItem;
168 PVOID ToUnmap;
169 LARGE_INTEGER FileOffset;
170 LARGE_INTEGER MapSize;
171 PROS_SECTION_OBJECT ToDeref;
172 PACQUIRE_FOR_LAZY_WRITE AcquireForLazyWrite;
173 PRELEASE_FROM_LAZY_WRITE ReleaseFromLazyWrite;
174 PVOID LazyContext;
175 BOOLEAN Dirty;
176 } WORK_QUEUE_WITH_CONTEXT, *PWORK_QUEUE_WITH_CONTEXT;
177
178 /*
179
180 Unmap a cache stripe. Note that cache stripes aren't unmapped when their
181 last reference disappears. We enter this code only if cache for the file
182 is uninitialized in the last file object, or a cache stripe is evicted.
183
184 */
185
186 VOID
187 CcpUnmapCache(PVOID Context)
188 {
189 PWORK_QUEUE_WITH_CONTEXT WorkItem = (PWORK_QUEUE_WITH_CONTEXT)Context;
190 DPRINT("Unmapping (finally) %x\n", WorkItem->ToUnmap);
191 MmUnmapCacheViewInSystemSpace(WorkItem->ToUnmap);
192 ObDereferenceObject(WorkItem->ToDeref);
193 ExFreePool(WorkItem);
194 DPRINT("Done\n");
195 }
196
197 /*
198
199 Somewhat deceptively named function which removes the last reference to a
200 cache stripe and completely removes it using CcUnmapCache. This may be
201 done either inline (if the Immediate BOOLEAN is set), or using a work item
202 at a later time. Whether this is called to unmap immeidately is mainly
203 determined by whether the caller is calling from a place in filesystem code
204 where a deadlock may occur if immediate flushing is required.
205
206 It's always safe to reuse the Bcb at CcCacheSections[Start] after calling
207 this.
208
209 */
210
211 /* Must have acquired the mutex */
212 VOID
213 CcpDereferenceCache(ULONG Start,
214 BOOLEAN Immediate)
215 {
216 PVOID ToUnmap;
217 PNOCC_BCB Bcb;
218 BOOLEAN Dirty;
219 LARGE_INTEGER MappedSize;
220 LARGE_INTEGER BaseOffset;
221 PWORK_QUEUE_WITH_CONTEXT WorkItem;
222
223 DPRINT("CcpDereferenceCache(#%x)\n", Start);
224
225 Bcb = &CcCacheSections[Start];
226
227 Dirty = Bcb->Dirty;
228 ToUnmap = Bcb->BaseAddress;
229 BaseOffset = Bcb->FileOffset;
230 MappedSize = Bcb->Map->FileSizes.ValidDataLength;
231
232 DPRINT("Dereference #%x (count %d)\n", Start, Bcb->RefCount);
233 ASSERT(Bcb->SectionObject);
234 ASSERT(Bcb->RefCount == 1);
235
236 DPRINT("Firing work item for %x\n", Bcb->BaseAddress);
237
238 if (Dirty) {
239 CcpUnlock();
240 Bcb->RefCount++;
241 MiFlushMappedSection(ToUnmap, &BaseOffset, &MappedSize, Dirty);
242 Bcb->RefCount--;
243 CcpLock();
244 }
245
246 if (Immediate)
247 {
248 PROS_SECTION_OBJECT ToDeref = Bcb->SectionObject;
249 Bcb->Map = NULL;
250 Bcb->SectionObject = NULL;
251 Bcb->BaseAddress = NULL;
252 Bcb->FileOffset.QuadPart = 0;
253 Bcb->Length = 0;
254 Bcb->RefCount = 0;
255 Bcb->Dirty = FALSE;
256 RemoveEntryList(&Bcb->ThisFileList);
257
258 CcpUnlock();
259 MmUnmapCacheViewInSystemSpace(ToUnmap);
260 ObDereferenceObject(ToDeref);
261 CcpLock();
262 }
263 else
264 {
265 WorkItem = ExAllocatePool(NonPagedPool, sizeof(*WorkItem));
266 if (!WorkItem) KeBugCheck(0);
267 WorkItem->ToUnmap = Bcb->BaseAddress;
268 WorkItem->FileOffset = Bcb->FileOffset;
269 WorkItem->Dirty = Bcb->Dirty;
270 WorkItem->MapSize = MappedSize;
271 WorkItem->ToDeref = Bcb->SectionObject;
272 WorkItem->AcquireForLazyWrite = Bcb->Map->Callbacks.AcquireForLazyWrite;
273 WorkItem->ReleaseFromLazyWrite = Bcb->Map->Callbacks.ReleaseFromLazyWrite;
274 WorkItem->LazyContext = Bcb->Map->LazyContext;
275
276 ExInitializeWorkItem(&WorkItem->WorkItem,
277 (PWORKER_THREAD_ROUTINE)CcpUnmapCache,
278 WorkItem);
279
280 Bcb->Map = NULL;
281 Bcb->SectionObject = NULL;
282 Bcb->BaseAddress = NULL;
283 Bcb->FileOffset.QuadPart = 0;
284 Bcb->Length = 0;
285 Bcb->RefCount = 0;
286 Bcb->Dirty = FALSE;
287 RemoveEntryList(&Bcb->ThisFileList);
288
289 CcpUnlock();
290 ExQueueWorkItem(&WorkItem->WorkItem, DelayedWorkQueue);
291 CcpLock();
292 }
293 DPRINT("Done\n");
294 }
295
296 /*
297
298 CcpAllocateCacheSections is called by CcpMapData to obtain a cache stripe,
299 possibly evicting an old stripe by calling CcpDereferenceCache in order to
300 obtain an empty Bcb.
301
302 This function was named plural due to a question I had at the beginning of
303 this endeavor about whether a map may span a 256k stripe boundary. It can't
304 so this function can only return the index of one Bcb. Returns INVALID_CACHE
305 on failure.
306
307 */
308 /* Needs mutex */
309 ULONG
310 CcpAllocateCacheSections(PFILE_OBJECT FileObject,
311 PROS_SECTION_OBJECT SectionObject)
312 {
313 ULONG i = INVALID_CACHE;
314 PNOCC_CACHE_MAP Map;
315 PNOCC_BCB Bcb;
316
317 DPRINT("AllocateCacheSections: FileObject %x\n", FileObject);
318
319 if (!FileObject->SectionObjectPointer)
320 return INVALID_CACHE;
321
322 Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap;
323
324 if (!Map)
325 return INVALID_CACHE;
326
327 DPRINT("Allocating Cache Section\n");
328
329 i = RtlFindClearBitsAndSet(CcCacheBitmap, 1, CcCacheClockHand);
330 CcCacheClockHand = (i + 1) % CACHE_NUM_SECTIONS;
331
332 if (i != INVALID_CACHE)
333 {
334 DPRINT("Setting up Bcb #%x\n", i);
335
336 Bcb = &CcCacheSections[i];
337
338 ASSERT(Bcb->RefCount < 2);
339
340 if (Bcb->RefCount > 0)
341 {
342 CcpDereferenceCache(i, FALSE);
343 }
344
345 ASSERT(!Bcb->RefCount);
346 Bcb->RefCount = 1;
347
348 DPRINT("Bcb #%x RefCount %d\n", Bcb - CcCacheSections, Bcb->RefCount);
349
350 if (!RtlTestBit(CcCacheBitmap, i))
351 {
352 DPRINT1("Somebody stoeled BCB #%x\n", i);
353 }
354 ASSERT(RtlTestBit(CcCacheBitmap, i));
355
356 DPRINT("Allocated #%x\n", i);
357 ASSERT(CcCacheSections[i].RefCount);
358 }
359 else
360 {
361 DPRINT1("Failed to allocate cache segment\n");
362 }
363 return i;
364 }
365
366 /* Must have acquired the mutex */
367 VOID
368 CcpReferenceCache(ULONG Start)
369 {
370 PNOCC_BCB Bcb;
371 Bcb = &CcCacheSections[Start];
372 ASSERT(Bcb->SectionObject);
373 Bcb->RefCount++;
374 RtlSetBit(CcCacheBitmap, Start);
375
376 }
377
378 VOID
379 CcpMarkForExclusive(ULONG Start)
380 {
381 PNOCC_BCB Bcb;
382 Bcb = &CcCacheSections[Start];
383 Bcb->ExclusiveWaiter++;
384 }
385
386 /*
387
388 Cache stripes have an idea of exclusive access, which would be hard to support
389 properly in the previous code. In our case, it's fairly easy, since we have
390 an event that indicates that the previous exclusive waiter has returned in each
391 Bcb.
392
393 */
394 /* Must not have the mutex */
395 VOID
396 CcpReferenceCacheExclusive(ULONG Start)
397 {
398 PNOCC_BCB Bcb = &CcCacheSections[Start];
399
400 KeWaitForSingleObject(&Bcb->ExclusiveWait,
401 Executive,
402 KernelMode,
403 FALSE,
404 NULL);
405
406 CcpLock();
407 ASSERT(Bcb->ExclusiveWaiter);
408 ASSERT(Bcb->SectionObject);
409 Bcb->Exclusive = TRUE;
410 Bcb->ExclusiveWaiter--;
411 RtlSetBit(CcCacheBitmap, Start);
412 CcpUnlock();
413 }
414
415 /*
416
417 Find a map that encompasses the target range. This function does not check
418 whether the desired range is partly outside the stripe. This could be
419 implemented with a generic table, but we generally aren't carring around a lot
420 of segments at once for a particular file.
421
422 When this returns a map for a given file address, then that address is by
423 definition already mapped and can be operated on.
424
425 Returns a valid index or INVALID_CACHE.
426
427 */
428 /* Must have the mutex */
429 ULONG
430 CcpFindMatchingMap(PLIST_ENTRY Head,
431 PLARGE_INTEGER FileOffset,
432 ULONG Length)
433 {
434 PLIST_ENTRY Entry;
435 //DPRINT("Find Matching Map: (%x) %x:%x\n", FileOffset->LowPart, Length);
436 for (Entry = Head->Flink; Entry != Head; Entry = Entry->Flink)
437 {
438 //DPRINT("Link @%x\n", Entry);
439 PNOCC_BCB Bcb = CONTAINING_RECORD(Entry, NOCC_BCB, ThisFileList);
440 //DPRINT("Selected BCB %x #%x\n", Bcb, Bcb - CcCacheSections);
441 //DPRINT("This File: %x:%x\n", Bcb->FileOffset.LowPart, Bcb->Length);
442 if (FileOffset->QuadPart >= Bcb->FileOffset.QuadPart &&
443 FileOffset->QuadPart < Bcb->FileOffset.QuadPart + CACHE_STRIPE)
444 {
445 //DPRINT("Found match at #%x\n", Bcb - CcCacheSections);
446 return Bcb - CcCacheSections;
447 }
448 }
449
450 //DPRINT("This region isn't mapped\n");
451
452 return INVALID_CACHE;
453 }
454
455 /*
456
457 Internal function that's used by all pinning functions.
458 It causes a mapped region to exist and prefaults the pages in it if possible,
459 possibly evicting another stripe in order to get our stripe.
460
461 */
462
463 BOOLEAN
464 NTAPI
465 CcpMapData(IN PFILE_OBJECT FileObject,
466 IN PLARGE_INTEGER FileOffset,
467 IN ULONG Length,
468 IN ULONG Flags,
469 OUT PVOID *BcbResult,
470 OUT PVOID *Buffer)
471 {
472 BOOLEAN Success = FALSE, FaultIn = FALSE;
473 /* Note: windows 2000 drivers treat this as a bool */
474 //BOOLEAN Wait = (Flags & MAP_WAIT) || (Flags == TRUE);
475 LARGE_INTEGER Target, EndInterval;
476 ULONG BcbHead, SectionSize, ViewSize;
477 PNOCC_BCB Bcb = NULL;
478 PROS_SECTION_OBJECT SectionObject = NULL;
479 NTSTATUS Status;
480 PNOCC_CACHE_MAP Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap;
481 ViewSize = CACHE_STRIPE;
482
483 if (!Map)
484 {
485 DPRINT1("File object was not mapped\n");
486 return FALSE;
487 }
488
489 DPRINT("CcMapData(F->%x, %I64x:%d)\n",
490 FileObject,
491 FileOffset->QuadPart,
492 Length);
493
494 ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL);
495
496 Target.HighPart = FileOffset->HighPart;
497 Target.LowPart = CACHE_ROUND_DOWN(FileOffset->LowPart);
498
499 CcpLock();
500
501 /* Find out if any range is a superset of what we want */
502 /* Find an accomodating section */
503 BcbHead = CcpFindMatchingMap(&Map->AssociatedBcb, FileOffset, Length);
504
505 if (BcbHead != INVALID_CACHE)
506 {
507 Bcb = &CcCacheSections[BcbHead];
508 Success = TRUE;
509 *BcbResult = Bcb;
510 *Buffer = ((PCHAR)Bcb->BaseAddress) + (int)(FileOffset->QuadPart - Bcb->FileOffset.QuadPart);
511
512 DPRINT("Bcb #%x Buffer maps (%I64x) At %x Length %x (Getting %p:%x) %wZ\n",
513 Bcb - CcCacheSections,
514 Bcb->FileOffset.QuadPart,
515 Bcb->BaseAddress,
516 Bcb->Length,
517 *Buffer,
518 Length,
519 &FileObject->FileName);
520
521 DPRINT("w1n\n");
522 goto cleanup;
523 }
524
525 DPRINT("File size %I64x\n",
526 Map->FileSizes.ValidDataLength.QuadPart);
527
528 /* Not all files have length, in fact filesystems often use stream file
529 objects for various internal purposes and are loose about the file
530 length, since the filesystem promises itself to write the right number
531 of bytes to the internal stream. In these cases, we just allow the file
532 to have the full stripe worth of space. */
533 if (Map->FileSizes.ValidDataLength.QuadPart)
534 {
535 SectionSize = min(CACHE_STRIPE,
536 Map->FileSizes.ValidDataLength.QuadPart - Target.QuadPart);
537 }
538 else
539 {
540 SectionSize = CACHE_STRIPE;
541 }
542
543 DPRINT("Allocating a cache stripe at %x:%d\n",
544 Target.LowPart, SectionSize);
545
546 //ASSERT(SectionSize <= CACHE_STRIPE);
547
548 CcpUnlock();
549 /* CcpAllocateSection doesn't need the lock, so we'll give other action
550 a chance in here. */
551 Status = CcpAllocateSection(FileObject,
552 SectionSize,
553 #ifdef PIN_WRITE_ONLY
554 PAGE_READONLY,
555 #else
556 PAGE_READWRITE,
557 #endif
558 &SectionObject);
559 CcpLock();
560
561 if (!NT_SUCCESS(Status))
562 {
563 *BcbResult = NULL;
564 *Buffer = NULL;
565 DPRINT1("End %08x\n", Status);
566 goto cleanup;
567 }
568
569 retry:
570 /* Returns a reference */
571 DPRINT("Allocating cache sections: %wZ\n", &FileObject->FileName);
572 BcbHead = CcpAllocateCacheSections(FileObject, SectionObject);
573 /* XXX todo: we should handle the immediate fail case here, but don't */
574 if (BcbHead == INVALID_CACHE)
575 {
576 ULONG i;
577 DbgPrint("Cache Map:");
578 for (i = 0; i < CACHE_NUM_SECTIONS; i++)
579 {
580 if (!(i % 64)) DbgPrint("\n");
581 DbgPrint("%c",
582 CcCacheSections[i].RefCount + (RtlTestBit(CcCacheBitmap, i) ? '@' : '`'));
583 }
584 DbgPrint("\n");
585
586 KeWaitForSingleObject(&CcDeleteEvent,
587 Executive,
588 KernelMode,
589 FALSE,
590 NULL);
591
592 goto retry;
593 }
594
595 DPRINT("BcbHead #%x (final)\n", BcbHead);
596
597 if (BcbHead == INVALID_CACHE)
598 {
599 *BcbResult = NULL;
600 *Buffer = NULL;
601 DPRINT1("End\n");
602 goto cleanup;
603 }
604
605 DPRINT("Selected BCB #%x\n", BcbHead);
606 ViewSize = CACHE_STRIPE;
607
608 Bcb = &CcCacheSections[BcbHead];
609 /* MmMapCacheViewInSystemSpaceAtOffset is one of three methods of Mm
610 that are specific to NewCC. In this case, it's implementation
611 exactly mirrors MmMapViewInSystemSpace, but allows an offset to
612 be specified. */
613 Status = MmMapCacheViewInSystemSpaceAtOffset(SectionObject->Segment,
614 &Bcb->BaseAddress,
615 &Target,
616 &ViewSize);
617
618 /* Summary: Failure. Dereference our section and tell the user we failed */
619 if (!NT_SUCCESS(Status))
620 {
621 *BcbResult = NULL;
622 *Buffer = NULL;
623 ObDereferenceObject(SectionObject);
624 RemoveEntryList(&Bcb->ThisFileList);
625 RtlZeroMemory(Bcb, sizeof(*Bcb));
626 RtlClearBit(CcCacheBitmap, BcbHead);
627 DPRINT1("Failed to map\n");
628 goto cleanup;
629 }
630
631 /* Summary: Success. Put together a valid Bcb and link it with the others
632 * in the NOCC_CACHE_MAP.
633 */
634 Success = TRUE;
635
636 Bcb->Length = MIN(Map->FileSizes.ValidDataLength.QuadPart - Target.QuadPart,
637 CACHE_STRIPE);
638
639 Bcb->SectionObject = SectionObject;
640 Bcb->Map = Map;
641 Bcb->FileOffset = Target;
642 InsertTailList(&Map->AssociatedBcb, &Bcb->ThisFileList);
643
644 *BcbResult = &CcCacheSections[BcbHead];
645 *Buffer = ((PCHAR)Bcb->BaseAddress) + (int)(FileOffset->QuadPart - Bcb->FileOffset.QuadPart);
646 FaultIn = TRUE;
647
648 DPRINT("Bcb #%x Buffer maps (%I64x) At %x Length %x (Getting %p:%lx) %wZ\n",
649 Bcb - CcCacheSections,
650 Bcb->FileOffset.QuadPart,
651 Bcb->BaseAddress,
652 Bcb->Length,
653 *Buffer,
654 Length,
655 &FileObject->FileName);
656
657 EndInterval.QuadPart = Bcb->FileOffset.QuadPart + Bcb->Length - 1;
658 ASSERT((EndInterval.QuadPart & ~(CACHE_STRIPE - 1)) ==
659 (Bcb->FileOffset.QuadPart & ~(CACHE_STRIPE - 1)));
660
661 cleanup:
662 CcpUnlock();
663 if (Success)
664 {
665 if (FaultIn)
666 {
667 /* Fault in the pages. This forces reads to happen now. */
668 ULONG i;
669 PCHAR FaultIn = Bcb->BaseAddress;
670
671 DPRINT("Faulting in pages at this point: file %wZ %I64x:%x\n",
672 &FileObject->FileName,
673 Bcb->FileOffset.QuadPart,
674 Bcb->Length);
675
676 for (i = 0; i < Bcb->Length; i += PAGE_SIZE)
677 {
678 FaultIn[i] ^= 0;
679 }
680 }
681 ASSERT(Bcb >= CcCacheSections &&
682 Bcb < (CcCacheSections + CACHE_NUM_SECTIONS));
683 }
684 else
685 {
686 ASSERT(FALSE);
687 }
688
689 return Success;
690 }
691
692 BOOLEAN
693 NTAPI
694 CcMapData(IN PFILE_OBJECT FileObject,
695 IN PLARGE_INTEGER FileOffset,
696 IN ULONG Length,
697 IN ULONG Flags,
698 OUT PVOID *BcbResult,
699 OUT PVOID *Buffer)
700 {
701 BOOLEAN Result;
702
703 Result = CcpMapData(FileObject,
704 FileOffset,
705 Length,
706 Flags,
707 BcbResult,
708 Buffer);
709
710 if (Result)
711 {
712 PNOCC_BCB Bcb = (PNOCC_BCB)*BcbResult;
713
714 ASSERT(Bcb >= CcCacheSections &&
715 Bcb < CcCacheSections + CACHE_NUM_SECTIONS);
716
717 ASSERT(Bcb->BaseAddress);
718 CcpLock();
719 CcpReferenceCache(Bcb - CcCacheSections);
720 CcpUnlock();
721 }
722
723 return Result;
724 }
725
726 /* Used by functions that repin data, CcpPinMappedData does not alter the map,
727 but finds the appropriate stripe and update the accounting. */
728 BOOLEAN
729 NTAPI
730 CcpPinMappedData(IN PNOCC_CACHE_MAP Map,
731 IN PLARGE_INTEGER FileOffset,
732 IN ULONG Length,
733 IN ULONG Flags,
734 IN OUT PVOID *Bcb)
735 {
736 BOOLEAN Exclusive = Flags & PIN_EXCLUSIVE;
737 ULONG BcbHead;
738 PNOCC_BCB TheBcb;
739
740 CcpLock();
741
742 ASSERT(Map->AssociatedBcb.Flink == &Map->AssociatedBcb || (CONTAINING_RECORD(Map->AssociatedBcb.Flink, NOCC_BCB, ThisFileList) >= CcCacheSections && CONTAINING_RECORD(Map->AssociatedBcb.Flink, NOCC_BCB, ThisFileList) < CcCacheSections + CACHE_NUM_SECTIONS));
743 BcbHead = CcpFindMatchingMap(&Map->AssociatedBcb, FileOffset, Length);
744 if (BcbHead == INVALID_CACHE)
745 {
746 CcpUnlock();
747 return FALSE;
748 }
749
750 TheBcb = &CcCacheSections[BcbHead];
751
752 if (Exclusive)
753 {
754 DPRINT("Requesting #%x Exclusive\n", BcbHead);
755 CcpMarkForExclusive(BcbHead);
756 }
757 else
758 {
759 DPRINT("Reference #%x\n", BcbHead);
760 CcpReferenceCache(BcbHead);
761 }
762
763 if (Exclusive)
764 CcpReferenceCacheExclusive(BcbHead);
765
766 CcpUnlock();
767
768 *Bcb = TheBcb;
769 return TRUE;
770 }
771
772 BOOLEAN
773 NTAPI
774 CcPinMappedData(IN PFILE_OBJECT FileObject,
775 IN PLARGE_INTEGER FileOffset,
776 IN ULONG Length,
777 IN ULONG Flags,
778 IN OUT PVOID *Bcb)
779 {
780 PVOID Buffer;
781 PNOCC_CACHE_MAP Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap;
782
783 if (!Map)
784 {
785 DPRINT1("Not cached\n");
786 return FALSE;
787 }
788
789 if (CcpMapData(FileObject, FileOffset, Length, Flags, Bcb, &Buffer))
790 {
791 return CcpPinMappedData(Map, FileOffset, Length, Flags, Bcb);
792 }
793 else
794 {
795 DPRINT1("could not map\n");
796 return FALSE;
797 }
798 }
799
800 BOOLEAN
801 NTAPI
802 CcPinRead(IN PFILE_OBJECT FileObject,
803 IN PLARGE_INTEGER FileOffset,
804 IN ULONG Length,
805 IN ULONG Flags,
806 OUT PVOID *Bcb,
807 OUT PVOID *Buffer)
808 {
809 PNOCC_BCB RealBcb;
810 BOOLEAN Result;
811
812 Result = CcPinMappedData(FileObject, FileOffset, Length, Flags, Bcb);
813
814 if (Result)
815 {
816 CcpLock();
817 RealBcb = *Bcb;
818 *Buffer = ((PCHAR)RealBcb->BaseAddress) + (int)(FileOffset->QuadPart - RealBcb->FileOffset.QuadPart);
819 CcpUnlock();
820 }
821
822 return Result;
823 }
824
825 BOOLEAN
826 NTAPI
827 CcPreparePinWrite(IN PFILE_OBJECT FileObject,
828 IN PLARGE_INTEGER FileOffset,
829 IN ULONG Length,
830 IN BOOLEAN Zero,
831 IN ULONG Flags,
832 OUT PVOID *Bcb,
833 OUT PVOID *Buffer)
834 {
835 BOOLEAN Result;
836 PNOCC_BCB RealBcb;
837 #ifdef PIN_WRITE_ONLY
838 PVOID BaseAddress;
839 SIZE_T NumberOfBytes;
840 ULONG OldProtect;
841 #endif
842
843 DPRINT("CcPreparePinWrite(%x:%x)\n", Buffer, Length);
844
845 Result = CcPinRead(FileObject, FileOffset, Length, Flags, Bcb, Buffer);
846
847 if (Result)
848 {
849 CcpLock();
850 RealBcb = *Bcb;
851
852 #ifdef PIN_WRITE_ONLY
853 BaseAddress = RealBcb->BaseAddress;
854 NumberOfBytes = RealBcb->Length;
855
856 MiProtectVirtualMemory(NULL,
857 &BaseAddress,
858 &NumberOfBytes,
859 PAGE_READWRITE,
860 &OldProtect);
861 #endif
862
863 CcpUnlock();
864 RealBcb->Dirty = TRUE;
865
866 if (Zero)
867 {
868 DPRINT("Zero fill #%x %I64x:%x Buffer %x %wZ\n",
869 RealBcb - CcCacheSections,
870 FileOffset->QuadPart,
871 Length,
872 *Buffer,
873 &FileObject->FileName);
874
875 DPRINT1("RtlZeroMemory(%p, %lx)\n", *Buffer, Length);
876 RtlZeroMemory(*Buffer, Length);
877 }
878 }
879
880 return Result;
881 }
882
883 /*
884
885 CcpUnpinData is the internal function that generally handles unpinning data.
886 It may be a little confusing, because of the way reference counts are handled.
887
888 A reference count of 2 or greater means that the stripe is still fully pinned
889 and can't be removed. If the owner had taken an exclusive reference, then
890 give one up. Note that it's an error to take more than one exclusive reference
891 or to take a non-exclusive reference after an exclusive reference, so detecting
892 or handling that case is not considered.
893
894 ReleaseBit is unset if we want to detect when a cache stripe would become
895 evictable without actually giving up our reference. We might want to do that
896 if we were going to flush before formally releasing the cache stripe, although
897 that facility is not used meaningfully at this time.
898
899 A reference count of exactly 1 means that the stripe could potentially be
900 reused, but could also be evicted for another mapping. In general, most
901 stripes should be in that state most of the time.
902
903 A reference count of zero means that the Bcb is completely unused. That's the
904 start state and the state of a Bcb formerly owned by a file that is
905 uninitialized.
906
907 */
908
909 BOOLEAN
910 NTAPI
911 CcpUnpinData(IN PNOCC_BCB RealBcb, BOOLEAN ReleaseBit)
912 {
913 if (RealBcb->RefCount <= 2)
914 {
915 RealBcb->Exclusive = FALSE;
916 if (RealBcb->ExclusiveWaiter)
917 {
918 DPRINT("Triggering exclusive waiter\n");
919 KeSetEvent(&RealBcb->ExclusiveWait, IO_NO_INCREMENT, FALSE);
920 return TRUE;
921 }
922 }
923 if (RealBcb->RefCount == 2 && !ReleaseBit)
924 return FALSE;
925 if (RealBcb->RefCount > 1)
926 {
927 DPRINT("Removing one reference #%x\n", RealBcb - CcCacheSections);
928 RealBcb->RefCount--;
929 KeSetEvent(&CcDeleteEvent, IO_DISK_INCREMENT, FALSE);
930 }
931 if (RealBcb->RefCount == 1)
932 {
933 DPRINT("Clearing allocation bit #%x\n", RealBcb - CcCacheSections);
934
935 RtlClearBit(CcCacheBitmap, RealBcb - CcCacheSections);
936
937 #ifdef PIN_WRITE_ONLY
938 PVOID BaseAddress = RealBcb->BaseAddress;
939 SIZE_T NumberOfBytes = RealBcb->Length;
940 ULONG OldProtect;
941
942 MiProtectVirtualMemory(NULL,
943 &BaseAddress,
944 &NumberOfBytes,
945 PAGE_READONLY,
946 &OldProtect);
947 #endif
948 }
949
950 return TRUE;
951 }
952
953 VOID
954 NTAPI
955 CcUnpinData(IN PVOID Bcb)
956 {
957 PNOCC_BCB RealBcb = (PNOCC_BCB)Bcb;
958 ULONG Selected = RealBcb - CcCacheSections;
959 BOOLEAN Released;
960
961 ASSERT(RealBcb >= CcCacheSections &&
962 RealBcb - CcCacheSections < CACHE_NUM_SECTIONS);
963
964 DPRINT("CcUnpinData Bcb #%x (RefCount %d)\n", Selected, RealBcb->RefCount);
965
966 CcpLock();
967 Released = CcpUnpinData(RealBcb, FALSE);
968 CcpUnlock();
969
970 if (!Released) {
971 CcpLock();
972 CcpUnpinData(RealBcb, TRUE);
973 CcpUnlock();
974 }
975 }
976
977 VOID
978 NTAPI
979 CcSetBcbOwnerPointer(IN PVOID Bcb,
980 IN PVOID OwnerPointer)
981 {
982 PNOCC_BCB RealBcb = (PNOCC_BCB)Bcb;
983 CcpLock();
984 CcpReferenceCache(RealBcb - CcCacheSections);
985 RealBcb->OwnerPointer = OwnerPointer;
986 CcpUnlock();
987 }
988
989 VOID
990 NTAPI
991 CcUnpinDataForThread(IN PVOID Bcb,
992 IN ERESOURCE_THREAD ResourceThreadId)
993 {
994 CcUnpinData(Bcb);
995 }
996
997 /* EOF */