[NTOSKRNL] Don't keep the spin lock hold too long when we lost the BCB race
[reactos.git] / ntoskrnl / cc / pin.c
index 53d51ed..b3906a9 100644 (file)
@@ -31,35 +31,62 @@ ULONG CcPinReadNoWait = 0;
 
 /* FUNCTIONS *****************************************************************/
 
+static
+PINTERNAL_BCB
+NTAPI
+CcpFindBcb(
+    IN PROS_SHARED_CACHE_MAP SharedCacheMap,
+    IN PLARGE_INTEGER FileOffset,
+    IN ULONG Length,
+    IN BOOLEAN Pinned)
+{
+    PINTERNAL_BCB Bcb;
+    BOOLEAN Found = FALSE;
+    PLIST_ENTRY NextEntry;
+
+    for (NextEntry = SharedCacheMap->BcbList.Flink;
+         NextEntry != &SharedCacheMap->BcbList;
+         NextEntry = NextEntry->Flink)
+    {
+        Bcb = CONTAINING_RECORD(NextEntry, INTERNAL_BCB, BcbEntry);
+
+        if (Bcb->PFCB.MappedFileOffset.QuadPart <= FileOffset->QuadPart &&
+            (Bcb->PFCB.MappedFileOffset.QuadPart + Bcb->PFCB.MappedLength) >=
+            (FileOffset->QuadPart + Length))
+        {
+            if ((Pinned && Bcb->PinCount > 0) || (!Pinned && Bcb->PinCount == 0))
+            {
+                Found = TRUE;
+                break;
+            }
+        }
+    }
+
+    return (Found ? Bcb : NULL);
+}
+
 static
 BOOLEAN
 NTAPI
 CcpMapData(
-    IN PFILE_OBJECT FileObject,
+    IN PROS_SHARED_CACHE_MAP SharedCacheMap,
     IN PLARGE_INTEGER FileOffset,
     IN ULONG Length,
     IN ULONG Flags,
+    IN BOOLEAN ToPin,
     OUT PVOID *pBcb,
     OUT PVOID *pBuffer)
 {
     LONGLONG ReadOffset;
     BOOLEAN Valid;
-    PROS_SHARED_CACHE_MAP SharedCacheMap;
     PROS_VACB Vacb;
     NTSTATUS Status;
-    PINTERNAL_BCB iBcb;
+    PINTERNAL_BCB iBcb, DupBcb;
     LONGLONG ROffset;
     KIRQL OldIrql;
 
     ReadOffset = FileOffset->QuadPart;
 
-    ASSERT(FileObject);
-    ASSERT(FileObject->SectionObjectPointer);
-    ASSERT(FileObject->SectionObjectPointer->SharedCacheMap);
-
-    SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap;
-    ASSERT(SharedCacheMap);
-
     DPRINT("SectionSize %I64x, FileSize %I64x\n",
            SharedCacheMap->SectionSize.QuadPart,
            SharedCacheMap->FileSize.QuadPart);
@@ -67,7 +94,7 @@ CcpMapData(
     if (ReadOffset % VACB_MAPPING_GRANULARITY + Length > VACB_MAPPING_GRANULARITY)
     {
         CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n",
-            FileObject, FileOffset, Length, Flags);
+            SharedCacheMap->FileObject, FileOffset, Length, Flags);
         return FALSE;
     }
 
@@ -92,7 +119,7 @@ CcpMapData(
     if (!NT_SUCCESS(Status))
     {
         CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n",
-            FileObject, FileOffset, Length, Flags);
+            SharedCacheMap->FileObject, FileOffset, Length, Flags);
         ExRaiseStatus(Status);
         return FALSE;
     }
@@ -103,7 +130,7 @@ CcpMapData(
         {
             CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE);
             CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n",
-                FileObject, FileOffset, Length, Flags);
+                SharedCacheMap->FileObject, FileOffset, Length, Flags);
             return FALSE;
         }
 
@@ -112,7 +139,7 @@ CcpMapData(
         {
             CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE);
             CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n",
-                FileObject, FileOffset, Length, Flags);
+                SharedCacheMap->FileObject, FileOffset, Length, Flags);
             ExRaiseStatus(Status);
             return FALSE;
         }
@@ -124,7 +151,7 @@ CcpMapData(
     {
         CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE);
         CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n",
-            FileObject, FileOffset, Length, Flags);
+            SharedCacheMap->FileObject, FileOffset, Length, Flags);
         ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES);
         return FALSE;
     }
@@ -141,9 +168,40 @@ CcpMapData(
     ExInitializeResourceLite(&iBcb->Lock);
     *pBcb = (PVOID)iBcb;
 
-    KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql);
-    InsertTailList(&SharedCacheMap->BcbList, &iBcb->BcbEntry);
-    KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql);
+    /* Only insert if we're not to pin data */
+    if (!ToPin)
+    {
+        KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql);
+
+        /* Check if we raced with another BCB creation */
+        DupBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, FALSE);
+        /* Yes, and we've lost */
+        if (DupBcb != NULL)
+        {
+            /* We'll return that BCB */
+            ++DupBcb->RefCount;
+
+            /* Delete the loser */
+            CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE);
+            ExDeleteResourceLite(&iBcb->Lock);
+            ExFreeToNPagedLookasideList(&iBcbLookasideList, iBcb);
+
+            /* Return the winner - no need to update buffer address, it's
+             * relative to the VACB, which is unchanged.
+             */
+            *pBcb = DupBcb;
+        }
+        /* Nope, insert ourselves */
+        else
+        {
+            InsertTailList(&SharedCacheMap->BcbList, &iBcb->BcbEntry);
+        }
+        KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql);
+    }
+    else
+    {
+        InitializeListHead(&iBcb->BcbEntry);
+    }
 
     return TRUE;
 }
@@ -162,11 +220,21 @@ CcMapData (
     OUT PVOID *pBuffer)
 {
     BOOLEAN Ret;
+    KIRQL OldIrql;
+    PINTERNAL_BCB iBcb;
+    PROS_SHARED_CACHE_MAP SharedCacheMap;
 
     DPRINT("CcMapData(FileObject 0x%p, FileOffset %I64x, Length %lu, Flags 0x%lx,"
            " pBcb 0x%p, pBuffer 0x%p)\n", FileObject, FileOffset->QuadPart,
            Length, Flags, pBcb, pBuffer);
 
+    ASSERT(FileObject);
+    ASSERT(FileObject->SectionObjectPointer);
+    ASSERT(FileObject->SectionObjectPointer->SharedCacheMap);
+
+    SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap;
+    ASSERT(SharedCacheMap);
+
     if (Flags & MAP_WAIT)
     {
         ++CcMapDataWait;
@@ -176,7 +244,21 @@ CcMapData (
         ++CcMapDataNoWait;
     }
 
-    Ret = CcpMapData(FileObject, FileOffset, Length, Flags, pBcb, pBuffer);
+    KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql);
+    iBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, FALSE);
+    KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql);
+
+    if (iBcb == NULL)
+    {
+        Ret = CcpMapData(SharedCacheMap, FileOffset, Length, Flags, FALSE, pBcb, pBuffer);
+    }
+    else
+    {
+        ++iBcb->RefCount;
+        *pBcb = iBcb;
+        *pBuffer = (PUCHAR)iBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY;
+        Ret = TRUE;
+    }
 
     CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> %d Bcb=%p\n",
         FileObject, FileOffset, Length, Flags, Ret, *pBcb);
@@ -249,9 +331,21 @@ CcPinRead (
     OUT        PVOID * Bcb,
     OUT        PVOID * Buffer)
 {
+    KIRQL OldIrql;
+    BOOLEAN Result;
+    PINTERNAL_BCB iBcb;
+    PROS_SHARED_CACHE_MAP SharedCacheMap;
+
     CCTRACE(CC_API_DEBUG, "FileOffset=%p FileOffset=%p Length=%lu Flags=0x%lx\n",
         FileObject, FileOffset, Length, Flags);
 
+    ASSERT(FileObject);
+    ASSERT(FileObject->SectionObjectPointer);
+    ASSERT(FileObject->SectionObjectPointer->SharedCacheMap);
+
+    SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap;
+    ASSERT(SharedCacheMap);
+
     if (Flags & PIN_WAIT)
     {
         ++CcPinReadWait;
@@ -261,17 +355,92 @@ CcPinRead (
         ++CcPinReadNoWait;
     }
 
-    /* Map first */
-    if (!CcpMapData(FileObject, FileOffset, Length, Flags, Bcb, Buffer))
+    KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql);
+    iBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, TRUE);
+    KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql);
+
+    if (iBcb == NULL)
     {
-        return FALSE;
-    }
+        /* We failed to find an already existing BCB */
+        if (BooleanFlagOn(Flags, PIN_IF_BCB))
+        {
+            return FALSE;
+        }
+
+        /* Map first */
+        if (!CcpMapData(SharedCacheMap, FileOffset, Length, Flags, TRUE, Bcb, Buffer))
+        {
+            return FALSE;
+        }
+
+        /* Pin then */
+        if (!CcPinMappedData(FileObject, FileOffset, Length, Flags, Bcb))
+        {
+            CcUnpinData(*Bcb);
+            return FALSE;
+        }
+
+        /* Did we race for the BCB creation? */
+        KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql);
+        iBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, TRUE);
+        if (iBcb != NULL)
+        {
+            KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql);
+
+            /* Free our now unused BCB */
+            CcUnpinData(*Bcb);
 
-    /* Pin then */
-    if (!CcPinMappedData(FileObject, FileOffset, Length, Flags, Bcb))
+            /* Lock the found BCB and return it */
+            if (BooleanFlagOn(Flags, PIN_EXCLUSIVE))
+            {
+                Result = ExAcquireResourceExclusiveLite(&iBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT));
+            }
+            else
+            {
+                Result = ExAcquireSharedStarveExclusive(&iBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT));
+            }
+
+            if (!Result)
+            {
+                return FALSE;
+            }
+
+            ++iBcb->PinCount;
+            ++iBcb->RefCount;
+
+            *Bcb = iBcb;
+        }
+        else
+        {
+            iBcb = *Bcb;
+
+            /* Insert ourselves in the linked list */
+            InsertTailList(&SharedCacheMap->BcbList, &iBcb->BcbEntry);
+            KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql);
+        }
+    }
+    /* We found a BCB, lock it and return it */
+    else
     {
-        CcUnpinData(*Bcb);
-        return FALSE;
+        if (BooleanFlagOn(Flags, PIN_EXCLUSIVE))
+        {
+            Result = ExAcquireResourceExclusiveLite(&iBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT));
+        }
+        else
+        {
+            Result = ExAcquireSharedStarveExclusive(&iBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT));
+        }
+
+        if (!Result)
+        {
+            return FALSE;
+        }
+
+        ++iBcb->PinCount;
+        ++iBcb->RefCount;
+
+        *Bcb = iBcb;
+        *Buffer = (PUCHAR)iBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY;
     }
 
     return TRUE;
@@ -371,7 +540,10 @@ CcUnpinDataForThread (
                          FALSE);
 
         KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql);
-        RemoveEntryList(&iBcb->BcbEntry);
+        if (!IsListEmpty(&iBcb->BcbEntry))
+        {
+            RemoveEntryList(&iBcb->BcbEntry);
+        }
         KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql);
 
         ExDeleteResourceLite(&iBcb->Lock);
@@ -445,7 +617,10 @@ CcUnpinRepinnedBcb (
                          FALSE);
 
         KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql);
-        RemoveEntryList(&iBcb->BcbEntry);
+        if (!IsListEmpty(&iBcb->BcbEntry))
+        {
+            RemoveEntryList(&iBcb->BcbEntry);
+        }
         KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql);
 
         ExDeleteResourceLite(&iBcb->Lock);