ntoskrnl/cache/pinsup.c

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS Kernel
   4  * FILE:            ntoskrnl/cache/pinsup.c
   5  * PURPOSE:         Logging and configuration routines
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  *                  Art Yerkes
   8  */
   9
  10 /* INCLUDES *******************************************************************/
  11
  12 #include <ntoskrnl.h>
  13 #include "newcc.h"
  14 #include "section/newmm.h"
  15 #define NDEBUG
  16 #include <debug.h>
  17
  18 /* The following is a test mode that only works with modified filesystems.
  19  * it maps the cache sections read only until they're pinned writable, and then
  20  * turns them readonly again when they're unpinned.
  21  * This helped me determine that a certain bug was not a memory overwrite. */
  22
  23 //#define PIN_WRITE_ONLY
  24
  25 /*
  26
  27 Pinsup implements the core of NewCC.
  28
  29 A couple of things about this code:
  30
  31 I wrote this code over the course of about 2 years, often referring to Rajeev
  32 Nagar's Filesystem Internals, book, the msdn pages on the Cc interface, and
  33 a few NT filesystems that are open sourced.  I went to fairly great lengths to
  34 achieve a couple of goals.
  35
  36 1) To make a strictly layered facility that relies entirely on Mm to provide
  37 maps.  There were many ways in which data segments in the legacy Mm were unable
  38 to provide what I needed; page maps were only 4 gig, and all offsets were in
  39 ULONG, so no mapping at an offset greater than 4 gig was possible.  Worse than
  40 that, due to a convoluted set of dependencies, it would have been impossible to
  41 support any two mappings farther apart than 4 gig, even if the above was
  42 corrected.  Along with that, the cache system's ownership of some pages was
  43 integral to the operation of legacy Mm.  All of the above problems, along with
  44 an ambiguity about when the size of a file for mapping purposes is acquired,
  45 and its inability to allow a file to be resized when any mappings were active
  46 led me to rewrite data sections (and all other kinds of sections in the
  47 original version), and use that layer to implement the Cc API without regard
  48 to any internal, undocumented parts.
  49
  50 2) To write the simplest possible code that implements the Cc interface as
  51 documented.  Again this is without regard to any information that might be
  52 gained through reverse engineering the real Cc.  All conclusions about workings
  53 of Cc here are mine, any failures are mine, any differences to the documented
  54 interface were introduced by me due to misreading, misunderstanding or mis
  55 remembering while implementing the code.  I also implemented some obvious, but
  56 not actually specified behaviors of Cc, for example that each cache stripe is
  57 represented by a distinct BCB that the user can make decisions about as an
  58 opaque pointer.
  59
  60 3) To make real filesystems work properly.
  61
  62 So about how it works:
  63
  64 CcCacheSections is the collection of cache sections that are currently mapped.
  65 The cache ranges which are allocated and contain pages is larger, due to the
  66 addition of sections containing rmaps and page references, but this array
  67 determines the actual mapped pages on behalf of all mapped files for Cc's use.
  68 All BCB pointers yielded to a driver are a pointer to one of these cache stripe
  69 structures.  The data structure is specified as opaque and so it contains
  70 information convenient to NEWCC's implementation here.  Free entries are
  71 summarized in CcpBitmapBuffer, for which bits are set when the entry may be
  72 safely evicted and redirected for use by another client.  Note that the
  73 reference count for an evictable cache section will generally be 1, since
  74 we'll keep a reference to wait for any subsequent mapping of the same stripe.
  75 We use CcCacheClockHand as a hint to start checking free bits at a point that
  76 walks around the cache stripe list, so that we might evict a different stripe
  77 every time even if all are awaiting reuse.  This is a way to avoid thrashing.
  78
  79 CcpBitmapBuffer is the RTL_BITMAP that allows us to quickly decide what buffer
  80 to allocate from the mapped buffer set.
  81
  82 CcDeleteEvent is an event used to wait for a cache stripe reference count to
  83 go to 1, thus making the stripe eligible for eviction.  It's used by CcpMapData
  84 to wait for a free map when we can't fail.
  85
  86 All in all, use of Mm by Cc makes this code into a simple manager that wields
  87 sections on behalf of filesystems.  As such, its code is fairly high level and
  88 no architecture specific changes should be necessary.
  89
  90 */
  91
  92 /* GLOBALS ********************************************************************/
  93
  94 #define TAG_MAP_SEC    TAG('C', 'c', 'S', 'x')
  95 #define TAG_MAP_READ   TAG('M', 'c', 'p', 'y')
  96 #define TAG_MAP_BCB    TAG('B', 'c', 'b', ' ')
  97
  98 NOCC_BCB CcCacheSections[CACHE_NUM_SECTIONS];
  99 CHAR CcpBitmapBuffer[sizeof(RTL_BITMAP) + ROUND_UP((CACHE_NUM_SECTIONS), 32) / 8];
 100 PRTL_BITMAP CcCacheBitmap = (PRTL_BITMAP)&CcpBitmapBuffer;
 101 FAST_MUTEX CcMutex;
 102 KEVENT CcDeleteEvent;
 103 KEVENT CcFinalizeEvent;
 104 ULONG CcCacheClockHand;
 105 LONG CcOutstandingDeletes;
 106
 107 /* FUNCTIONS ******************************************************************/
 108
 109 PETHREAD LastThread;
 110
 111 VOID
 112 _CcpLock(const char *file,
 113          int line)
 114 {
 115     //DPRINT("<<<---<<< CC In Mutex(%s:%d %x)!\n", file, line, PsGetCurrentThread());
 116     ExAcquireFastMutex(&CcMutex);
 117 }
 118
 119 VOID
 120 _CcpUnlock(const char *file,
 121            int line)
 122 {
 123     ExReleaseFastMutex(&CcMutex);
 124     //DPRINT(">>>--->>> CC Exit Mutex!\n", file, line);
 125 }
 126
 127 PDEVICE_OBJECT
 128 NTAPI
 129 MmGetDeviceObjectForFile(IN PFILE_OBJECT FileObject);
 130
 131 /*
 132
 133 Allocate an almost ordinary section object for use by the cache system.
 134 The special internal SEC_CACHE flag is used to indicate that the section
 135 should not count when determining whether the file can be resized.
 136
 137 */
 138
 139 NTSTATUS
 140 CcpAllocateSection(PFILE_OBJECT FileObject,
 141                    ULONG Length,
 142                    ULONG Protect,
 143                    PROS_SECTION_OBJECT *Result)
 144 {
 145     NTSTATUS Status;
 146     LARGE_INTEGER MaxSize;
 147
 148     MaxSize.QuadPart = Length;
 149
 150     DPRINT("Making Section for File %x\n", FileObject);
 151     DPRINT("File name %wZ\n", &FileObject->FileName);
 152
 153     Status = MmCreateSection((PVOID*)Result,
 154                              STANDARD_RIGHTS_REQUIRED,
 155                              NULL,
 156                              &MaxSize,
 157                              Protect,
 158                              SEC_RESERVE | SEC_CACHE,
 159                              NULL,
 160                              FileObject);
 161
 162     return Status;
 163 }
 164
 165 typedef struct _WORK_QUEUE_WITH_CONTEXT
 166 {
 167     WORK_QUEUE_ITEM WorkItem;
 168     PVOID ToUnmap;
 169     LARGE_INTEGER FileOffset;
 170     LARGE_INTEGER MapSize;
 171     PROS_SECTION_OBJECT ToDeref;
 172     PACQUIRE_FOR_LAZY_WRITE AcquireForLazyWrite;
 173     PRELEASE_FROM_LAZY_WRITE ReleaseFromLazyWrite;
 174     PVOID LazyContext;
 175     BOOLEAN Dirty;
 176 } WORK_QUEUE_WITH_CONTEXT, *PWORK_QUEUE_WITH_CONTEXT;
 177
 178 /*
 179
 180 Unmap a cache stripe.  Note that cache stripes aren't unmapped when their
 181 last reference disappears.  We enter this code only if cache for the file
 182 is uninitialized in the last file object, or a cache stripe is evicted.
 183
 184 */
 185
 186 VOID
 187 CcpUnmapCache(PVOID Context)
 188 {
 189     PWORK_QUEUE_WITH_CONTEXT WorkItem = (PWORK_QUEUE_WITH_CONTEXT)Context;
 190     DPRINT("Unmapping (finally) %x\n", WorkItem->ToUnmap);
 191     MmUnmapCacheViewInSystemSpace(WorkItem->ToUnmap);
 192     ObDereferenceObject(WorkItem->ToDeref);
 193     ExFreePool(WorkItem);
 194     DPRINT("Done\n");
 195 }
 196
 197 /*
 198
 199 Somewhat deceptively named function which removes the last reference to a
 200 cache stripe and completely removes it using CcUnmapCache.  This may be
 201 done either inline (if the Immediate BOOLEAN is set), or using a work item
 202 at a later time.  Whether this is called to unmap immeidately is mainly
 203 determined by whether the caller is calling from a place in filesystem code
 204 where a deadlock may occur if immediate flushing is required.
 205
 206 It's always safe to reuse the Bcb at CcCacheSections[Start] after calling
 207 this.
 208
 209  */
 210
 211 /* Must have acquired the mutex */
 212 VOID
 213 CcpDereferenceCache(ULONG Start,
 214                     BOOLEAN Immediate)
 215 {
 216     PVOID ToUnmap;
 217     PNOCC_BCB Bcb;
 218     BOOLEAN Dirty;
 219     LARGE_INTEGER MappedSize;
 220     LARGE_INTEGER BaseOffset;
 221     PWORK_QUEUE_WITH_CONTEXT WorkItem;
 222
 223     DPRINT("CcpDereferenceCache(#%x)\n", Start);
 224
 225     Bcb = &CcCacheSections[Start];
 226
 227     Dirty = Bcb->Dirty;
 228     ToUnmap = Bcb->BaseAddress;
 229     BaseOffset = Bcb->FileOffset;
 230     MappedSize = Bcb->Map->FileSizes.ValidDataLength;
 231
 232     DPRINT("Dereference #%x (count %d)\n", Start, Bcb->RefCount);
 233     ASSERT(Bcb->SectionObject);
 234     ASSERT(Bcb->RefCount == 1);
 235
 236     DPRINT("Firing work item for %x\n", Bcb->BaseAddress);
 237
 238     if (Dirty) {
 239         CcpUnlock();
 240         Bcb->RefCount++;
 241         MiFlushMappedSection(ToUnmap, &BaseOffset, &MappedSize, Dirty);
 242         Bcb->RefCount--;
 243         CcpLock();
 244     }
 245
 246     if (Immediate)
 247     {
 248         PROS_SECTION_OBJECT ToDeref = Bcb->SectionObject;
 249         Bcb->Map = NULL;
 250         Bcb->SectionObject = NULL;
 251         Bcb->BaseAddress = NULL;
 252         Bcb->FileOffset.QuadPart = 0;
 253         Bcb->Length = 0;
 254         Bcb->RefCount = 0;
 255         Bcb->Dirty = FALSE;
 256         RemoveEntryList(&Bcb->ThisFileList);
 257
 258         CcpUnlock();
 259         MmUnmapCacheViewInSystemSpace(ToUnmap);
 260         ObDereferenceObject(ToDeref);
 261         CcpLock();
 262     }
 263     else
 264     {
 265         WorkItem = ExAllocatePool(NonPagedPool, sizeof(*WorkItem));
 266         if (!WorkItem) KeBugCheck(0);
 267         WorkItem->ToUnmap = Bcb->BaseAddress;
 268         WorkItem->FileOffset = Bcb->FileOffset;
 269         WorkItem->Dirty = Bcb->Dirty;
 270         WorkItem->MapSize = MappedSize;
 271         WorkItem->ToDeref = Bcb->SectionObject;
 272         WorkItem->AcquireForLazyWrite = Bcb->Map->Callbacks.AcquireForLazyWrite;
 273         WorkItem->ReleaseFromLazyWrite = Bcb->Map->Callbacks.ReleaseFromLazyWrite;
 274         WorkItem->LazyContext = Bcb->Map->LazyContext;
 275
 276         ExInitializeWorkItem(((PWORK_QUEUE_ITEM)WorkItem),
 277                              (PWORKER_THREAD_ROUTINE)CcpUnmapCache,
 278                              WorkItem);
 279
 280         Bcb->Map = NULL;
 281         Bcb->SectionObject = NULL;
 282         Bcb->BaseAddress = NULL;
 283         Bcb->FileOffset.QuadPart = 0;
 284         Bcb->Length = 0;
 285         Bcb->RefCount = 0;
 286         Bcb->Dirty = FALSE;
 287         RemoveEntryList(&Bcb->ThisFileList);
 288
 289         CcpUnlock();
 290         ExQueueWorkItem((PWORK_QUEUE_ITEM)WorkItem, DelayedWorkQueue);
 291         CcpLock();
 292     }
 293     DPRINT("Done\n");
 294 }
 295
 296 /*
 297
 298 CcpAllocateCacheSections is called by CcpMapData to obtain a cache stripe,
 299 possibly evicting an old stripe by calling CcpDereferenceCache in order to
 300 obtain an empty Bcb.
 301
 302 This function was named plural due to a question I had at the beginning of
 303 this endeavor about whether a map may span a 256k stripe boundary.  It can't
 304 so this function can only return the index of one Bcb.  Returns INVALID_CACHE
 305 on failure.
 306
 307  */
 308 /* Needs mutex */
 309 ULONG
 310 CcpAllocateCacheSections(PFILE_OBJECT FileObject,
 311                          PROS_SECTION_OBJECT SectionObject)
 312 {
 313     ULONG i = INVALID_CACHE;
 314     PNOCC_CACHE_MAP Map;
 315     PNOCC_BCB Bcb;
 316
 317     DPRINT("AllocateCacheSections: FileObject %x\n", FileObject);
 318
 319     if (!FileObject->SectionObjectPointer)
 320         return INVALID_CACHE;
 321
 322     Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap;
 323
 324     if (!Map)
 325         return INVALID_CACHE;
 326
 327     DPRINT("Allocating Cache Section\n");
 328
 329     i = RtlFindClearBitsAndSet(CcCacheBitmap, 1, CcCacheClockHand);
 330     CcCacheClockHand = (i + 1) % CACHE_NUM_SECTIONS;
 331
 332     if (i != INVALID_CACHE)
 333     {
 334         DPRINT("Setting up Bcb #%x\n", i);
 335
 336         Bcb = &CcCacheSections[i];
 337
 338         ASSERT(Bcb->RefCount < 2);
 339
 340         if (Bcb->RefCount > 0)
 341         {
 342             CcpDereferenceCache(i, FALSE);
 343         }
 344
 345         ASSERT(!Bcb->RefCount);
 346         Bcb->RefCount = 1;
 347
 348         DPRINT("Bcb #%x RefCount %d\n", Bcb - CcCacheSections, Bcb->RefCount);
 349
 350         if (!RtlTestBit(CcCacheBitmap, i))
 351         {
 352             DPRINT1("Somebody stoeled BCB #%x\n", i);
 353         }
 354         ASSERT(RtlTestBit(CcCacheBitmap, i));
 355
 356         DPRINT("Allocated #%x\n", i);
 357         ASSERT(CcCacheSections[i].RefCount);
 358     }
 359     else
 360     {
 361         DPRINT1("Failed to allocate cache segment\n");
 362     }
 363     return i;
 364 }
 365
 366 /* Must have acquired the mutex */
 367 VOID
 368 CcpReferenceCache(ULONG Start)
 369 {
 370     PNOCC_BCB Bcb;
 371     Bcb = &CcCacheSections[Start];
 372     ASSERT(Bcb->SectionObject);
 373     Bcb->RefCount++;
 374     RtlSetBit(CcCacheBitmap, Start);
 375
 376 }
 377
 378 VOID
 379 CcpMarkForExclusive(ULONG Start)
 380 {
 381     PNOCC_BCB Bcb;
 382     Bcb = &CcCacheSections[Start];
 383     Bcb->ExclusiveWaiter++;
 384 }
 385
 386 /*
 387
 388 Cache stripes have an idea of exclusive access, which would be hard to support
 389 properly in the previous code.  In our case, it's fairly easy, since we have
 390 an event that indicates that the previous exclusive waiter has returned in each
 391 Bcb.
 392
 393 */
 394 /* Must not have the mutex */
 395 VOID
 396 CcpReferenceCacheExclusive(ULONG Start)
 397 {
 398     PNOCC_BCB Bcb = &CcCacheSections[Start];
 399
 400     KeWaitForSingleObject(&Bcb->ExclusiveWait,
 401                           Executive,
 402                           KernelMode,
 403                           FALSE,
 404                           NULL);
 405
 406     CcpLock();
 407     ASSERT(Bcb->ExclusiveWaiter);
 408     ASSERT(Bcb->SectionObject);
 409     Bcb->Exclusive = TRUE;
 410     Bcb->ExclusiveWaiter--;
 411     RtlSetBit(CcCacheBitmap, Start);
 412     CcpUnlock();
 413 }
 414
 415 /*
 416
 417 Find a map that encompasses the target range.  This function does not check
 418 whether the desired range is partly outside the stripe.  This could be
 419 implemented with a generic table, but we generally aren't carring around a lot
 420 of segments at once for a particular file.
 421
 422 When this returns a map for a given file address, then that address is by
 423 definition already mapped and can be operated on.
 424
 425 Returns a valid index or INVALID_CACHE.
 426
 427 */
 428 /* Must have the mutex */
 429 ULONG
 430 CcpFindMatchingMap(PLIST_ENTRY Head,
 431                    PLARGE_INTEGER FileOffset,
 432                    ULONG Length)
 433 {
 434     PLIST_ENTRY Entry;
 435     //DPRINT("Find Matching Map: (%x) %x:%x\n", FileOffset->LowPart, Length);
 436     for (Entry = Head->Flink; Entry != Head; Entry = Entry->Flink)
 437     {
 438         //DPRINT("Link @%x\n", Entry);
 439         PNOCC_BCB Bcb = CONTAINING_RECORD(Entry, NOCC_BCB, ThisFileList);
 440         //DPRINT("Selected BCB %x #%x\n", Bcb, Bcb - CcCacheSections);
 441         //DPRINT("This File: %x:%x\n", Bcb->FileOffset.LowPart, Bcb->Length);
 442         if (FileOffset->QuadPart >= Bcb->FileOffset.QuadPart &&
 443             FileOffset->QuadPart < Bcb->FileOffset.QuadPart + CACHE_STRIPE)
 444         {
 445             //DPRINT("Found match at #%x\n", Bcb - CcCacheSections);
 446             return Bcb - CcCacheSections;
 447         }
 448     }
 449
 450     //DPRINT("This region isn't mapped\n");
 451
 452     return INVALID_CACHE;
 453 }
 454
 455 /*
 456
 457 Internal function that's used by all pinning functions.
 458 It causes a mapped region to exist and prefaults the pages in it if possible,
 459 possibly evicting another stripe in order to get our stripe.
 460
 461 */
 462
 463 BOOLEAN
 464 NTAPI
 465 CcpMapData(IN PFILE_OBJECT FileObject,
 466            IN PLARGE_INTEGER FileOffset,
 467            IN ULONG Length,
 468            IN ULONG Flags,
 469            OUT PVOID *BcbResult,
 470            OUT PVOID *Buffer)
 471 {
 472     BOOLEAN Success = FALSE, FaultIn = FALSE;
 473     /* Note: windows 2000 drivers treat this as a bool */
 474     //BOOLEAN Wait = (Flags & MAP_WAIT) || (Flags == TRUE);
 475     LARGE_INTEGER Target, EndInterval;
 476     ULONG BcbHead, SectionSize, ViewSize;
 477     PNOCC_BCB Bcb = NULL;
 478     PROS_SECTION_OBJECT SectionObject = NULL;
 479     NTSTATUS Status;
 480     PNOCC_CACHE_MAP Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap;
 481     ViewSize = CACHE_STRIPE;
 482
 483     if (!Map)
 484     {
 485         DPRINT1("File object was not mapped\n");
 486         return FALSE;
 487     }
 488
 489     DPRINT("CcMapData(F->%x,%08x%08x:%d)\n",
 490            FileObject,
 491            FileOffset->HighPart,
 492            FileOffset->LowPart,
 493            Length);
 494
 495     ASSERT(KeGetCurrentIrql() < DISPATCH_LEVEL);
 496
 497     Target.HighPart = FileOffset->HighPart;
 498     Target.LowPart = CACHE_ROUND_DOWN(FileOffset->LowPart);
 499
 500     CcpLock();
 501
 502     /* Find out if any range is a superset of what we want */
 503     /* Find an accomodating section */
 504     BcbHead = CcpFindMatchingMap(&Map->AssociatedBcb, FileOffset, Length);
 505
 506     if (BcbHead != INVALID_CACHE)
 507     {
 508         Bcb = &CcCacheSections[BcbHead];
 509         Success = TRUE;
 510         *BcbResult = Bcb;
 511         *Buffer = ((PCHAR)Bcb->BaseAddress) + (int)(FileOffset->QuadPart - Bcb->FileOffset.QuadPart);
 512
 513         DPRINT("Bcb #%x Buffer maps (%08x%08x) At %x Length %x (Getting %x:%x) %wZ\n",
 514                Bcb - CcCacheSections,
 515                Bcb->FileOffset.HighPart,
 516                Bcb->FileOffset.LowPart,
 517                Bcb->BaseAddress,
 518                Bcb->Length,
 519                *Buffer,
 520                Length,
 521                &FileObject->FileName);
 522
 523         DPRINT("w1n\n");
 524         goto cleanup;
 525     }
 526
 527     DPRINT("File size %08x%08x\n",
 528            Map->FileSizes.ValidDataLength.HighPart,
 529            Map->FileSizes.ValidDataLength.LowPart);
 530
 531     /* Not all files have length, in fact filesystems often use stream file
 532        objects for various internal purposes and are loose about the file
 533        length, since the filesystem promises itself to write the right number
 534        of bytes to the internal stream.  In these cases, we just allow the file
 535        to have the full stripe worth of space. */
 536     if (Map->FileSizes.ValidDataLength.QuadPart)
 537     {
 538         SectionSize = min(CACHE_STRIPE,
 539                           Map->FileSizes.ValidDataLength.QuadPart - Target.QuadPart);
 540     }
 541     else
 542     {
 543         SectionSize = CACHE_STRIPE;
 544     }
 545
 546     DPRINT("Allocating a cache stripe at %x:%d\n",
 547            Target.LowPart, SectionSize);
 548
 549     //ASSERT(SectionSize <= CACHE_STRIPE);
 550
 551     CcpUnlock();
 552     /* CcpAllocateSection doesn't need the lock, so we'll give other action
 553        a chance in here. */
 554     Status = CcpAllocateSection(FileObject,
 555                                 SectionSize,
 556 #ifdef PIN_WRITE_ONLY
 557                                 PAGE_READONLY,
 558 #else
 559                                 PAGE_READWRITE,
 560 #endif
 561                                 &SectionObject);
 562     CcpLock();
 563
 564     if (!NT_SUCCESS(Status))
 565     {
 566         *BcbResult = NULL;
 567         *Buffer = NULL;
 568         DPRINT1("End %08x\n", Status);
 569         goto cleanup;
 570     }
 571
 572 retry:
 573     /* Returns a reference */
 574     DPRINT("Allocating cache sections: %wZ\n", &FileObject->FileName);
 575     BcbHead = CcpAllocateCacheSections(FileObject, SectionObject);
 576     /* XXX todo: we should handle the immediate fail case here, but don't */
 577     if (BcbHead == INVALID_CACHE)
 578     {
 579         ULONG i;
 580         DbgPrint("Cache Map:");
 581         for (i = 0; i < CACHE_NUM_SECTIONS; i++)
 582         {
 583             if (!(i % 64)) DbgPrint("\n");
 584             DbgPrint("%c",
 585                      CcCacheSections[i].RefCount + (RtlTestBit(CcCacheBitmap, i) ? '@' : '`'));
 586         }
 587         DbgPrint("\n");
 588
 589         KeWaitForSingleObject(&CcDeleteEvent,
 590                               Executive,
 591                               KernelMode,
 592                               FALSE,
 593                               NULL);
 594
 595         goto retry;
 596     }
 597
 598     DPRINT("BcbHead #%x (final)\n", BcbHead);
 599
 600     if (BcbHead == INVALID_CACHE)
 601     {
 602         *BcbResult = NULL;
 603         *Buffer = NULL;
 604         DPRINT1("End\n");
 605         goto cleanup;
 606     }
 607
 608     DPRINT("Selected BCB #%x\n", BcbHead);
 609     ViewSize = CACHE_STRIPE;
 610
 611     Bcb = &CcCacheSections[BcbHead];
 612     /* MmMapCacheViewInSystemSpaceAtOffset is one of three methods of Mm
 613        that are specific to NewCC.  In this case, it's implementation
 614        exactly mirrors MmMapViewInSystemSpace, but allows an offset to
 615        be specified. */
 616     Status = MmMapCacheViewInSystemSpaceAtOffset(SectionObject->Segment,
 617                                                  &Bcb->BaseAddress,
 618                                                  &Target,
 619                                                  &ViewSize);
 620
 621     /* Summary: Failure.  Dereference our section and tell the user we failed */
 622     if (!NT_SUCCESS(Status))
 623     {
 624         *BcbResult = NULL;
 625         *Buffer = NULL;
 626         ObDereferenceObject(SectionObject);
 627         RemoveEntryList(&Bcb->ThisFileList);
 628         RtlZeroMemory(Bcb, sizeof(*Bcb));
 629         RtlClearBit(CcCacheBitmap, BcbHead);
 630         DPRINT1("Failed to map\n");
 631         goto cleanup;
 632     }
 633
 634     /* Summary: Success.  Put together a valid Bcb and link it with the others
 635      * in the NOCC_CACHE_MAP.
 636      */
 637     Success = TRUE;
 638     //DPRINT("w1n\n");
 639
 640     Bcb->Length = MIN(Map->FileSizes.ValidDataLength.QuadPart - Target.QuadPart,
 641                       CACHE_STRIPE);
 642
 643     Bcb->SectionObject = SectionObject;
 644     Bcb->Map = Map;
 645     Bcb->FileOffset = Target;
 646     InsertTailList(&Map->AssociatedBcb, &Bcb->ThisFileList);
 647
 648     *BcbResult = &CcCacheSections[BcbHead];
 649     *Buffer = ((PCHAR)Bcb->BaseAddress) + (int)(FileOffset->QuadPart - Bcb->FileOffset.QuadPart);
 650     FaultIn = TRUE;
 651
 652     DPRINT("Bcb #%x Buffer maps (%08x%08x) At %x Length %x (Getting %x:%x) %wZ\n",
 653            Bcb - CcCacheSections,
 654            Bcb->FileOffset.HighPart,
 655            Bcb->FileOffset.LowPart,
 656            Bcb->BaseAddress,
 657            Bcb->Length,
 658            *Buffer,
 659            Length,
 660            &FileObject->FileName);
 661
 662     EndInterval.QuadPart = Bcb->FileOffset.QuadPart + Bcb->Length - 1;
 663     ASSERT((EndInterval.QuadPart & ~(CACHE_STRIPE - 1)) ==
 664            (Bcb->FileOffset.QuadPart & ~(CACHE_STRIPE - 1)));
 665
 666     //DPRINT("TERM!\n");
 667
 668 cleanup:
 669     CcpUnlock();
 670     if (Success)
 671     {
 672         if (FaultIn)
 673         {
 674             /* Fault in the pages.  This forces reads to happen now. */
 675             ULONG i;
 676             PCHAR FaultIn = Bcb->BaseAddress;
 677
 678             DPRINT("Faulting in pages at this point: file %wZ %08x%08x:%x\n",
 679                    &FileObject->FileName,
 680                    Bcb->FileOffset.HighPart,
 681                    Bcb->FileOffset.LowPart,
 682                    Bcb->Length);
 683
 684             for (i = 0; i < Bcb->Length; i += PAGE_SIZE)
 685             {
 686                 FaultIn[i] ^= 0;
 687             }
 688         }
 689         ASSERT(Bcb >= CcCacheSections &&
 690                Bcb < (CcCacheSections + CACHE_NUM_SECTIONS));
 691     }
 692     else
 693     {
 694         ASSERT(FALSE);
 695     }
 696
 697     return Success;
 698 }
 699
 700 BOOLEAN
 701 NTAPI
 702 CcMapData(IN PFILE_OBJECT FileObject,
 703           IN PLARGE_INTEGER FileOffset,
 704           IN ULONG Length,
 705           IN ULONG Flags,
 706           OUT PVOID *BcbResult,
 707           OUT PVOID *Buffer)
 708 {
 709     BOOLEAN Result;
 710
 711     Result = CcpMapData(FileObject,
 712                         FileOffset,
 713                         Length,
 714                         Flags,
 715                         BcbResult,
 716                         Buffer);
 717
 718     if (Result)
 719     {
 720         PNOCC_BCB Bcb = (PNOCC_BCB)*BcbResult;
 721
 722         ASSERT(Bcb >= CcCacheSections &&
 723                Bcb < CcCacheSections + CACHE_NUM_SECTIONS);
 724
 725         ASSERT(Bcb->BaseAddress);
 726         CcpLock();
 727         CcpReferenceCache(Bcb - CcCacheSections);
 728         CcpUnlock();
 729     }
 730
 731     return Result;
 732 }
 733
 734 /* Used by functions that repin data, CcpPinMappedData does not alter the map,
 735    but finds the appropriate stripe and update the accounting. */
 736 BOOLEAN
 737 NTAPI
 738 CcpPinMappedData(IN PNOCC_CACHE_MAP Map,
 739                  IN PLARGE_INTEGER FileOffset,
 740                  IN ULONG Length,
 741                  IN ULONG Flags,
 742                  IN OUT PVOID *Bcb)
 743 {
 744     BOOLEAN Exclusive = Flags & PIN_EXCLUSIVE;
 745     ULONG BcbHead;
 746     PNOCC_BCB TheBcb;
 747
 748     CcpLock();
 749
 750     ASSERT(Map->AssociatedBcb.Flink == &Map->AssociatedBcb || (CONTAINING_RECORD(Map->AssociatedBcb.Flink, NOCC_BCB, ThisFileList) >= CcCacheSections && CONTAINING_RECORD(Map->AssociatedBcb.Flink, NOCC_BCB, ThisFileList) < CcCacheSections + CACHE_NUM_SECTIONS));
 751     BcbHead = CcpFindMatchingMap(&Map->AssociatedBcb, FileOffset, Length);
 752     if (BcbHead == INVALID_CACHE)
 753     {
 754         CcpUnlock();
 755         return FALSE;
 756     }
 757
 758     TheBcb = &CcCacheSections[BcbHead];
 759
 760     if (Exclusive)
 761     {
 762         DPRINT("Requesting #%x Exclusive\n", BcbHead);
 763         CcpMarkForExclusive(BcbHead);
 764     }
 765     else
 766     {
 767         DPRINT("Reference #%x\n", BcbHead);
 768         CcpReferenceCache(BcbHead);
 769     }
 770
 771     if (Exclusive)
 772         CcpReferenceCacheExclusive(BcbHead);
 773
 774     CcpUnlock();
 775
 776     *Bcb = TheBcb;
 777     return TRUE;
 778 }
 779
 780 BOOLEAN
 781 NTAPI
 782 CcPinMappedData(IN PFILE_OBJECT FileObject,
 783                 IN PLARGE_INTEGER FileOffset,
 784                 IN ULONG Length,
 785                 IN ULONG Flags,
 786                 IN OUT PVOID *Bcb)
 787 {
 788     PVOID Buffer;
 789     PNOCC_CACHE_MAP Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap;
 790
 791     if (!Map)
 792     {
 793         DPRINT1("Not cached\n");
 794         return FALSE;
 795     }
 796
 797     if (CcpMapData(FileObject, FileOffset, Length, Flags, Bcb, &Buffer))
 798     {
 799         return CcpPinMappedData(Map, FileOffset, Length, Flags, Bcb);
 800     }
 801     else
 802     {
 803         DPRINT1("could not map\n");
 804         return FALSE;
 805     }
 806 }
 807
 808 BOOLEAN
 809 NTAPI
 810 CcPinRead(IN PFILE_OBJECT FileObject,
 811           IN PLARGE_INTEGER FileOffset,
 812           IN ULONG Length,
 813           IN ULONG Flags,
 814           OUT PVOID *Bcb,
 815           OUT PVOID *Buffer)
 816 {
 817     PNOCC_BCB RealBcb;
 818     BOOLEAN Result;
 819
 820     Result = CcPinMappedData(FileObject, FileOffset, Length, Flags, Bcb);
 821
 822     if (Result)
 823     {
 824         CcpLock();
 825         RealBcb = *Bcb;
 826         *Buffer = ((PCHAR)RealBcb->BaseAddress) + (int)(FileOffset->QuadPart - RealBcb->FileOffset.QuadPart);
 827         CcpUnlock();
 828     }
 829
 830     return Result;
 831 }
 832
 833 BOOLEAN
 834 NTAPI
 835 CcPreparePinWrite(IN PFILE_OBJECT FileObject,
 836                   IN PLARGE_INTEGER FileOffset,
 837                   IN ULONG Length,
 838                   IN BOOLEAN Zero,
 839                   IN ULONG Flags,
 840                   OUT PVOID *Bcb,
 841                   OUT PVOID *Buffer)
 842 {
 843     BOOLEAN Result;
 844     PNOCC_BCB RealBcb;
 845 #ifdef PIN_WRITE_ONLY
 846     PVOID BaseAddress;
 847     SIZE_T NumberOfBytes;
 848     ULONG OldProtect;
 849 #endif
 850
 851     DPRINT("CcPreparePinWrite(%x:%x)\n", Buffer, Length);
 852
 853     Result = CcPinRead(FileObject, FileOffset, Length, Flags, Bcb, Buffer);
 854
 855     if (Result)
 856     {
 857         CcpLock();
 858         RealBcb = *Bcb;
 859
 860 #ifdef PIN_WRITE_ONLY
 861         BaseAddress = RealBcb->BaseAddress;
 862         NumberOfBytes = RealBcb->Length;
 863
 864         MiProtectVirtualMemory(NULL,
 865                                &BaseAddress,
 866                                &NumberOfBytes,
 867                                PAGE_READWRITE,
 868                                &OldProtect);
 869 #endif
 870
 871         CcpUnlock();
 872         RealBcb->Dirty = TRUE;
 873
 874         if (Zero)
 875         {
 876             DPRINT("Zero fill #%x %08x%08x:%x Buffer %x %wZ\n",
 877                    RealBcb - CcCacheSections,
 878                    FileOffset->u.HighPart,
 879                    FileOffset->u.LowPart,
 880                    Length,
 881                    *Buffer,
 882                    &FileObject->FileName);
 883
 884             DPRINT1("RtlZeroMemory(%x,%x)\n", *Buffer, Length);
 885             RtlZeroMemory(*Buffer, Length);
 886         }
 887     }
 888
 889     return Result;
 890 }
 891
 892 /*
 893
 894 CcpUnpinData is the internal function that generally handles unpinning data.
 895 It may be a little confusing, because of the way reference counts are handled.
 896
 897 A reference count of 2 or greater means that the stripe is still fully pinned
 898 and can't be removed.  If the owner had taken an exclusive reference, then
 899 give one up.  Note that it's an error to take more than one exclusive reference
 900 or to take a non-exclusive reference after an exclusive reference, so detecting
 901 or handling that case is not considered.
 902
 903 ReleaseBit is unset if we want to detect when a cache stripe would become
 904 evictable without actually giving up our reference.  We might want to do that
 905 if we were going to flush before formally releasing the cache stripe, although
 906 that facility is not used meaningfully at this time.
 907
 908 A reference count of exactly 1 means that the stripe could potentially be
 909 reused, but could also be evicted for another mapping.  In general, most
 910 stripes should be in that state most of the time.
 911
 912 A reference count of zero means that the Bcb is completely unused.  That's the
 913 start state and the state of a Bcb formerly owned by a file that is
 914 uninitialized.
 915
 916 */
 917
 918 BOOLEAN
 919 NTAPI
 920 CcpUnpinData(IN PNOCC_BCB RealBcb, BOOLEAN ReleaseBit)
 921 {
 922     if (RealBcb->RefCount <= 2)
 923     {
 924         RealBcb->Exclusive = FALSE;
 925         if (RealBcb->ExclusiveWaiter)
 926         {
 927             DPRINT("Triggering exclusive waiter\n");
 928             KeSetEvent(&RealBcb->ExclusiveWait, IO_NO_INCREMENT, FALSE);
 929             return TRUE;
 930         }
 931     }
 932     if (RealBcb->RefCount == 2 && !ReleaseBit)
 933         return FALSE;
 934     if (RealBcb->RefCount > 1)
 935     {
 936         DPRINT("Removing one reference #%x\n", RealBcb - CcCacheSections);
 937         RealBcb->RefCount--;
 938         KeSetEvent(&CcDeleteEvent, IO_DISK_INCREMENT, FALSE);
 939     }
 940     if (RealBcb->RefCount == 1)
 941     {
 942         DPRINT("Clearing allocation bit #%x\n", RealBcb - CcCacheSections);
 943
 944         RtlClearBit(CcCacheBitmap, RealBcb - CcCacheSections);
 945
 946 #ifdef PIN_WRITE_ONLY
 947         PVOID BaseAddress = RealBcb->BaseAddress;
 948         SIZE_T NumberOfBytes = RealBcb->Length;
 949         ULONG OldProtect;
 950
 951         MiProtectVirtualMemory(NULL,
 952                                &BaseAddress,
 953                                &NumberOfBytes,
 954                                PAGE_READONLY,
 955                                &OldProtect);
 956 #endif
 957     }
 958
 959     return TRUE;
 960 }
 961
 962 VOID
 963 NTAPI
 964 CcUnpinData(IN PVOID Bcb)
 965 {
 966     PNOCC_BCB RealBcb = (PNOCC_BCB)Bcb;
 967     ULONG Selected = RealBcb - CcCacheSections;
 968     BOOLEAN Released;
 969
 970     ASSERT(RealBcb >= CcCacheSections &&
 971            RealBcb - CcCacheSections < CACHE_NUM_SECTIONS);
 972
 973     DPRINT("CcUnpinData Bcb #%x (RefCount %d)\n", Selected, RealBcb->RefCount);
 974
 975     CcpLock();
 976     Released = CcpUnpinData(RealBcb, FALSE);
 977     CcpUnlock();
 978
 979     if (!Released) {
 980         CcpLock();
 981         CcpUnpinData(RealBcb, TRUE);
 982         CcpUnlock();
 983     }
 984 }
 985
 986 VOID
 987 NTAPI
 988 CcSetBcbOwnerPointer(IN PVOID Bcb,
 989                      IN PVOID OwnerPointer)
 990 {
 991     PNOCC_BCB RealBcb = (PNOCC_BCB)Bcb;
 992     CcpLock();
 993     CcpReferenceCache(RealBcb - CcCacheSections);
 994     RealBcb->OwnerPointer = OwnerPointer;
 995     CcpUnlock();
 996 }
 997
 998 VOID
 999 NTAPI
1000 CcUnpinDataForThread(IN PVOID Bcb,
1001                      IN ERESOURCE_THREAD ResourceThreadId)
1002 {
1003     CcUnpinData(Bcb);
1004 }
1005
1006 /* EOF */