1 /* Copyright (c) Mark Harmstone 2016-17
3 * This file is part of WinBtrfs.
5 * WinBtrfs is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU Lesser General Public Licence as published by
7 * the Free Software Foundation, either version 3 of the Licence, or
8 * (at your option) any later version.
10 * WinBtrfs is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU Lesser General Public Licence for more details.
15 * You should have received a copy of the GNU Lesser General Public Licence
16 * along with WinBtrfs. If not, see <http://www.gnu.org/licenses/>. */
18 #include "btrfs_drv.h"
28 _Function_class_(IO_COMPLETION_ROUTINE
)
30 static NTSTATUS NTAPI
write_data_completion(PDEVICE_OBJECT DeviceObject
, PIRP Irp
, PVOID conptr
);
32 static NTSTATUS
write_data_completion(PDEVICE_OBJECT DeviceObject
, PIRP Irp
, PVOID conptr
);
35 static void remove_fcb_extent(fcb
* fcb
, extent
* ext
, LIST_ENTRY
* rollback
);
37 extern tPsUpdateDiskCounters fPsUpdateDiskCounters
;
38 extern tCcCopyWriteEx fCcCopyWriteEx
;
39 extern tFsRtlUpdateDiskCounters fFsRtlUpdateDiskCounters
;
42 BOOL
find_data_address_in_chunk(device_extension
* Vcb
, chunk
* c
, UINT64 length
, UINT64
* address
) {
46 TRACE("(%p, %llx, %llx, %p)\n", Vcb
, c
->offset
, length
, address
);
48 if (length
> c
->chunk_item
->size
- c
->used
)
51 if (!c
->cache_loaded
) {
52 NTSTATUS Status
= load_cache_chunk(Vcb
, c
, NULL
);
54 if (!NT_SUCCESS(Status
)) {
55 ERR("load_cache_chunk returned %08x\n", Status
);
60 if (IsListEmpty(&c
->space_size
))
63 le
= c
->space_size
.Flink
;
64 while (le
!= &c
->space_size
) {
65 s
= CONTAINING_RECORD(le
, space
, list_entry_size
);
67 if (s
->size
== length
) {
68 *address
= s
->address
;
70 } else if (s
->size
< length
) {
71 if (le
== c
->space_size
.Flink
)
74 s
= CONTAINING_RECORD(le
->Blink
, space
, list_entry_size
);
76 *address
= s
->address
;
83 s
= CONTAINING_RECORD(c
->space_size
.Blink
, space
, list_entry_size
);
85 if (s
->size
> length
) {
86 *address
= s
->address
;
93 chunk
* get_chunk_from_address(device_extension
* Vcb
, UINT64 address
) {
96 ExAcquireResourceSharedLite(&Vcb
->chunk_lock
, TRUE
);
98 le2
= Vcb
->chunks
.Flink
;
99 while (le2
!= &Vcb
->chunks
) {
100 chunk
* c
= CONTAINING_RECORD(le2
, chunk
, list_entry
);
102 if (address
>= c
->offset
&& address
< c
->offset
+ c
->chunk_item
->size
) {
103 ExReleaseResourceLite(&Vcb
->chunk_lock
);
110 ExReleaseResourceLite(&Vcb
->chunk_lock
);
120 static UINT64
find_new_chunk_address(device_extension
* Vcb
, UINT64 size
) {
126 le
= Vcb
->chunks
.Flink
;
127 while (le
!= &Vcb
->chunks
) {
128 chunk
* c
= CONTAINING_RECORD(le
, chunk
, list_entry
);
130 if (c
->offset
>= lastaddr
+ size
)
133 lastaddr
= c
->offset
+ c
->chunk_item
->size
;
141 static BOOL
find_new_dup_stripes(device_extension
* Vcb
, stripe
* stripes
, UINT64 max_stripe_size
, BOOL full_size
) {
142 UINT64 devusage
= 0xffffffffffffffff;
143 space
*devdh1
= NULL
, *devdh2
= NULL
;
147 le
= Vcb
->devices
.Flink
;
149 while (le
!= &Vcb
->devices
) {
150 device
* dev
= CONTAINING_RECORD(le
, device
, list_entry
);
152 if (!dev
->readonly
&& !dev
->reloc
&& dev
->devobj
) {
153 UINT64 usage
= (dev
->devitem
.bytes_used
* 4096) / dev
->devitem
.num_bytes
;
155 // favour devices which have been used the least
156 if (usage
< devusage
) {
157 if (!IsListEmpty(&dev
->space
)) {
159 space
*dh1
= NULL
, *dh2
= NULL
;
161 le2
= dev
->space
.Flink
;
162 while (le2
!= &dev
->space
) {
163 space
* dh
= CONTAINING_RECORD(le2
, space
, list_entry
);
165 if (dh
->size
>= max_stripe_size
&& (!dh1
|| !dh2
|| dh
->size
< dh1
->size
)) {
173 if (dh1
&& (dh2
|| dh1
->size
>= 2 * max_stripe_size
)) {
177 devdh2
= dh2
? dh2
: dh1
;
189 // Can't find hole of at least max_stripe_size; look for the largest one we can find
194 le
= Vcb
->devices
.Flink
;
195 while (le
!= &Vcb
->devices
) {
196 device
* dev
= CONTAINING_RECORD(le
, device
, list_entry
);
198 if (!dev
->readonly
&& !dev
->reloc
) {
199 if (!IsListEmpty(&dev
->space
)) {
201 space
*dh1
= NULL
, *dh2
= NULL
;
203 le2
= dev
->space
.Flink
;
204 while (le2
!= &dev
->space
) {
205 space
* dh
= CONTAINING_RECORD(le2
, space
, list_entry
);
207 if (!dh1
|| !dh2
|| dh
->size
< dh1
->size
) {
219 devsize
= max(dh1
->size
/ 2, min(dh1
->size
, dh2
->size
));
221 devsize
= dh1
->size
/ 2;
223 if (devsize
> size
) {
227 if (dh2
&& min(dh1
->size
, dh2
->size
) > dh1
->size
/ 2)
245 stripes
[0].device
= stripes
[1].device
= dev2
;
246 stripes
[0].dh
= devdh1
;
247 stripes
[1].dh
= devdh2
;
252 static BOOL
find_new_stripe(device_extension
* Vcb
, stripe
* stripes
, UINT16 i
, UINT64 max_stripe_size
, BOOL allow_missing
, BOOL full_size
) {
253 UINT64 k
, devusage
= 0xffffffffffffffff;
258 le
= Vcb
->devices
.Flink
;
259 while (le
!= &Vcb
->devices
) {
260 device
* dev
= CONTAINING_RECORD(le
, device
, list_entry
);
264 if (dev
->readonly
|| dev
->reloc
|| (!dev
->devobj
&& !allow_missing
)) {
269 // skip this device if it already has a stripe
271 for (k
= 0; k
< i
; k
++) {
272 if (stripes
[k
].device
== dev
) {
280 usage
= (dev
->devitem
.bytes_used
* 4096) / dev
->devitem
.num_bytes
;
282 // favour devices which have been used the least
283 if (usage
< devusage
) {
284 if (!IsListEmpty(&dev
->space
)) {
287 le2
= dev
->space
.Flink
;
288 while (le2
!= &dev
->space
) {
289 space
* dh
= CONTAINING_RECORD(le2
, space
, list_entry
);
291 if ((dev2
!= dev
&& dh
->size
>= max_stripe_size
) ||
292 (dev2
== dev
&& dh
->size
>= max_stripe_size
&& dh
->size
< devdh
->size
)
309 // Can't find hole of at least max_stripe_size; look for the largest one we can find
314 le
= Vcb
->devices
.Flink
;
315 while (le
!= &Vcb
->devices
) {
316 device
* dev
= CONTAINING_RECORD(le
, device
, list_entry
);
319 if (dev
->readonly
|| dev
->reloc
|| (!dev
->devobj
&& !allow_missing
)) {
324 // skip this device if it already has a stripe
326 for (k
= 0; k
< i
; k
++) {
327 if (stripes
[k
].device
== dev
) {
335 if (!IsListEmpty(&dev
->space
)) {
338 le2
= dev
->space
.Flink
;
339 while (le2
!= &dev
->space
) {
340 space
* dh
= CONTAINING_RECORD(le2
, space
, list_entry
);
342 if (!devdh
|| devdh
->size
< dh
->size
) {
359 stripes
[i
].dh
= devdh
;
360 stripes
[i
].device
= dev2
;
365 NTSTATUS
alloc_chunk(device_extension
* Vcb
, UINT64 flags
, chunk
** pc
, BOOL full_size
) {
367 UINT64 max_stripe_size
, max_chunk_size
, stripe_size
, stripe_length
, factor
;
368 UINT64 total_size
= 0, logaddr
;
369 UINT16 i
, type
, num_stripes
, sub_stripes
, max_stripes
, min_stripes
, allowed_missing
;
370 stripe
* stripes
= NULL
;
372 CHUNK_ITEM_STRIPE
* cis
;
377 le
= Vcb
->devices
.Flink
;
378 while (le
!= &Vcb
->devices
) {
379 device
* dev
= CONTAINING_RECORD(le
, device
, list_entry
);
380 total_size
+= dev
->devitem
.num_bytes
;
385 TRACE("total_size = %llx\n", total_size
);
387 // We purposely check for DATA first - mixed blocks have the same size
389 if (flags
& BLOCK_FLAG_DATA
) {
390 max_stripe_size
= 0x40000000; // 1 GB
391 max_chunk_size
= 10 * max_stripe_size
;
392 } else if (flags
& BLOCK_FLAG_METADATA
) {
393 if (total_size
> 0xC80000000) // 50 GB
394 max_stripe_size
= 0x40000000; // 1 GB
396 max_stripe_size
= 0x10000000; // 256 MB
398 max_chunk_size
= max_stripe_size
;
399 } else if (flags
& BLOCK_FLAG_SYSTEM
) {
400 max_stripe_size
= 0x2000000; // 32 MB
401 max_chunk_size
= 2 * max_stripe_size
;
403 ERR("unknown chunk type\n");
404 return STATUS_INTERNAL_ERROR
;
407 max_chunk_size
= min(max_chunk_size
, total_size
/ 10); // cap at 10%
409 TRACE("would allocate a new chunk of %llx bytes and stripe %llx\n", max_chunk_size
, max_stripe_size
);
411 if (flags
& BLOCK_FLAG_DUPLICATE
) {
415 type
= BLOCK_FLAG_DUPLICATE
;
417 } else if (flags
& BLOCK_FLAG_RAID0
) {
419 max_stripes
= (UINT16
)min(0xffff, Vcb
->superblock
.num_devices
);
421 type
= BLOCK_FLAG_RAID0
;
423 } else if (flags
& BLOCK_FLAG_RAID1
) {
427 type
= BLOCK_FLAG_RAID1
;
429 } else if (flags
& BLOCK_FLAG_RAID10
) {
431 max_stripes
= (UINT16
)min(0xffff, Vcb
->superblock
.num_devices
);
433 type
= BLOCK_FLAG_RAID10
;
435 } else if (flags
& BLOCK_FLAG_RAID5
) {
437 max_stripes
= (UINT16
)min(0xffff, Vcb
->superblock
.num_devices
);
439 type
= BLOCK_FLAG_RAID5
;
441 } else if (flags
& BLOCK_FLAG_RAID6
) {
445 type
= BLOCK_FLAG_RAID6
;
455 stripes
= ExAllocatePoolWithTag(PagedPool
, sizeof(stripe
) * max_stripes
, ALLOC_TAG
);
457 ERR("out of memory\n");
458 Status
= STATUS_INSUFFICIENT_RESOURCES
;
464 if (type
== BLOCK_FLAG_DUPLICATE
) {
465 if (!find_new_dup_stripes(Vcb
, stripes
, max_stripe_size
, full_size
)) {
466 Status
= STATUS_DISK_FULL
;
470 num_stripes
= max_stripes
;
472 for (i
= 0; i
< max_stripes
; i
++) {
473 if (!find_new_stripe(Vcb
, stripes
, i
, max_stripe_size
, FALSE
, full_size
))
480 if (num_stripes
< min_stripes
&& Vcb
->options
.allow_degraded
&& allowed_missing
> 0) {
481 UINT16 added_missing
= 0;
483 for (i
= num_stripes
; i
< max_stripes
; i
++) {
484 if (!find_new_stripe(Vcb
, stripes
, i
, max_stripe_size
, TRUE
, full_size
))
488 if (added_missing
>= allowed_missing
)
493 num_stripes
+= added_missing
;
496 // for RAID10, round down to an even number of stripes
497 if (type
== BLOCK_FLAG_RAID10
&& (num_stripes
% sub_stripes
) != 0) {
498 num_stripes
-= num_stripes
% sub_stripes
;
501 if (num_stripes
< min_stripes
) {
502 WARN("found %u stripes, needed at least %u\n", num_stripes
, min_stripes
);
503 Status
= STATUS_DISK_FULL
;
507 c
= ExAllocatePoolWithTag(NonPagedPool
, sizeof(chunk
), ALLOC_TAG
);
509 ERR("out of memory\n");
510 Status
= STATUS_INSUFFICIENT_RESOURCES
;
514 cisize
= sizeof(CHUNK_ITEM
) + (num_stripes
* sizeof(CHUNK_ITEM_STRIPE
));
515 c
->chunk_item
= ExAllocatePoolWithTag(NonPagedPool
, cisize
, ALLOC_TAG
);
516 if (!c
->chunk_item
) {
517 ERR("out of memory\n");
518 Status
= STATUS_INSUFFICIENT_RESOURCES
;
522 stripe_length
= 0x10000; // FIXME? BTRFS_STRIPE_LEN in kernel
524 if (type
== BLOCK_FLAG_DUPLICATE
&& stripes
[1].dh
== stripes
[0].dh
)
525 stripe_size
= min(stripes
[0].dh
->size
/ 2, max_stripe_size
);
527 stripe_size
= max_stripe_size
;
528 for (i
= 0; i
< num_stripes
; i
++) {
529 if (stripes
[i
].dh
->size
< stripe_size
)
530 stripe_size
= stripes
[i
].dh
->size
;
534 if (type
== 0 || type
== BLOCK_FLAG_DUPLICATE
|| type
== BLOCK_FLAG_RAID1
)
536 else if (type
== BLOCK_FLAG_RAID0
)
537 factor
= num_stripes
;
538 else if (type
== BLOCK_FLAG_RAID10
)
539 factor
= num_stripes
/ sub_stripes
;
540 else if (type
== BLOCK_FLAG_RAID5
)
541 factor
= num_stripes
- 1;
542 else if (type
== BLOCK_FLAG_RAID6
)
543 factor
= num_stripes
- 2;
545 if (stripe_size
* factor
> max_chunk_size
)
546 stripe_size
= max_chunk_size
/ factor
;
548 if (stripe_size
% stripe_length
> 0)
549 stripe_size
-= stripe_size
% stripe_length
;
551 if (stripe_size
== 0) {
552 Status
= STATUS_INTERNAL_ERROR
;
556 c
->chunk_item
->size
= stripe_size
* factor
;
557 c
->chunk_item
->root_id
= Vcb
->extent_root
->id
;
558 c
->chunk_item
->stripe_length
= stripe_length
;
559 c
->chunk_item
->type
= flags
;
560 c
->chunk_item
->opt_io_alignment
= (UINT32
)c
->chunk_item
->stripe_length
;
561 c
->chunk_item
->opt_io_width
= (UINT32
)c
->chunk_item
->stripe_length
;
562 c
->chunk_item
->sector_size
= stripes
[0].device
->devitem
.minimal_io_size
;
563 c
->chunk_item
->num_stripes
= num_stripes
;
564 c
->chunk_item
->sub_stripes
= sub_stripes
;
566 c
->devices
= ExAllocatePoolWithTag(NonPagedPool
, sizeof(device
*) * num_stripes
, ALLOC_TAG
);
568 ERR("out of memory\n");
569 Status
= STATUS_INSUFFICIENT_RESOURCES
;
573 cis
= (CHUNK_ITEM_STRIPE
*)&c
->chunk_item
[1];
574 for (i
= 0; i
< num_stripes
; i
++) {
575 cis
[i
].dev_id
= stripes
[i
].device
->devitem
.dev_id
;
577 if (type
== BLOCK_FLAG_DUPLICATE
&& i
== 1 && stripes
[i
].dh
== stripes
[0].dh
)
578 cis
[i
].offset
= stripes
[0].dh
->address
+ stripe_size
;
580 cis
[i
].offset
= stripes
[i
].dh
->address
;
582 cis
[i
].dev_uuid
= stripes
[i
].device
->devitem
.device_uuid
;
584 c
->devices
[i
] = stripes
[i
].device
;
587 logaddr
= find_new_chunk_address(Vcb
, c
->chunk_item
->size
);
589 Vcb
->superblock
.chunk_root_generation
= Vcb
->superblock
.generation
;
593 c
->used
= c
->oldused
= 0;
594 c
->cache
= c
->old_cache
= NULL
;
597 c
->last_alloc_set
= FALSE
;
599 c
->cache_loaded
= TRUE
;
601 c
->space_changed
= FALSE
;
604 InitializeListHead(&c
->space
);
605 InitializeListHead(&c
->space_size
);
606 InitializeListHead(&c
->deleting
);
607 InitializeListHead(&c
->changed_extents
);
609 InitializeListHead(&c
->range_locks
);
610 ExInitializeResourceLite(&c
->range_locks_lock
);
611 KeInitializeEvent(&c
->range_locks_event
, NotificationEvent
, FALSE
);
613 InitializeListHead(&c
->partial_stripes
);
614 ExInitializeResourceLite(&c
->partial_stripes_lock
);
616 ExInitializeResourceLite(&c
->lock
);
617 ExInitializeResourceLite(&c
->changed_extents_lock
);
619 s
= ExAllocatePoolWithTag(NonPagedPool
, sizeof(space
), ALLOC_TAG
);
621 ERR("out of memory\n");
622 Status
= STATUS_INSUFFICIENT_RESOURCES
;
626 s
->address
= c
->offset
;
627 s
->size
= c
->chunk_item
->size
;
628 InsertTailList(&c
->space
, &s
->list_entry
);
629 InsertTailList(&c
->space_size
, &s
->list_entry_size
);
631 protect_superblocks(c
);
633 for (i
= 0; i
< num_stripes
; i
++) {
634 stripes
[i
].device
->devitem
.bytes_used
+= stripe_size
;
636 space_list_subtract2(&stripes
[i
].device
->space
, NULL
, cis
[i
].offset
, stripe_size
, NULL
, NULL
);
639 Status
= STATUS_SUCCESS
;
641 if (flags
& BLOCK_FLAG_RAID5
|| flags
& BLOCK_FLAG_RAID6
)
642 Vcb
->superblock
.incompat_flags
|= BTRFS_INCOMPAT_FLAGS_RAID56
;
648 if (!NT_SUCCESS(Status
)) {
651 ExFreePool(c
->devices
);
654 ExFreePool(c
->chunk_item
);
659 if (s
) ExFreePool(s
);
663 le
= Vcb
->chunks
.Flink
;
664 while (le
!= &Vcb
->chunks
) {
665 chunk
* c2
= CONTAINING_RECORD(le
, chunk
, list_entry
);
667 if (c2
->offset
> c
->offset
) {
668 InsertHeadList(le
->Blink
, &c
->list_entry
);
677 InsertTailList(&Vcb
->chunks
, &c
->list_entry
);
681 c
->space_changed
= TRUE
;
682 c
->list_entry_balance
.Flink
= NULL
;
690 static NTSTATUS
prepare_raid0_write(_Pre_satisfies_(_Curr_
->chunk_item
->num_stripes
>0) _In_ chunk
* c
, _In_ UINT64 address
, _In_reads_bytes_(length
) void* data
,
691 _In_ UINT32 length
, _In_ write_stripe
* stripes
, _In_ PIRP Irp
, _In_ UINT64 irp_offset
, _In_ write_data_context
* wtc
) {
692 UINT64 startoff
, endoff
;
693 UINT16 startoffstripe
, endoffstripe
, stripenum
;
694 UINT64 pos
, *stripeoff
;
696 BOOL file_write
= Irp
&& Irp
->MdlAddress
&& (Irp
->MdlAddress
->ByteOffset
== 0);
700 stripeoff
= ExAllocatePoolWithTag(PagedPool
, sizeof(UINT64
) * c
->chunk_item
->num_stripes
, ALLOC_TAG
);
702 ERR("out of memory\n");
703 return STATUS_INSUFFICIENT_RESOURCES
;
706 get_raid0_offset(address
- c
->offset
, c
->chunk_item
->stripe_length
, c
->chunk_item
->num_stripes
, &startoff
, &startoffstripe
);
707 get_raid0_offset(address
+ length
- c
->offset
- 1, c
->chunk_item
->stripe_length
, c
->chunk_item
->num_stripes
, &endoff
, &endoffstripe
);
710 master_mdl
= Irp
->MdlAddress
;
712 pfns
= (PFN_NUMBER
*)(Irp
->MdlAddress
+ 1);
713 pfns
= &pfns
[irp_offset
>> PAGE_SHIFT
];
714 } else if (((ULONG_PTR
)data
% PAGE_SIZE
) != 0) {
715 wtc
->scratch
= ExAllocatePoolWithTag(NonPagedPool
, length
, ALLOC_TAG
);
717 ERR("out of memory\n");
718 return STATUS_INSUFFICIENT_RESOURCES
;
721 RtlCopyMemory(wtc
->scratch
, data
, length
);
723 master_mdl
= IoAllocateMdl(wtc
->scratch
, length
, FALSE
, FALSE
, NULL
);
725 ERR("out of memory\n");
726 return STATUS_INSUFFICIENT_RESOURCES
;
729 MmBuildMdlForNonPagedPool(master_mdl
);
731 wtc
->mdl
= master_mdl
;
733 pfns
= (PFN_NUMBER
*)(master_mdl
+ 1);
735 NTSTATUS Status
= STATUS_SUCCESS
;
737 master_mdl
= IoAllocateMdl(data
, length
, FALSE
, FALSE
, NULL
);
739 ERR("out of memory\n");
740 return STATUS_INSUFFICIENT_RESOURCES
;
744 MmProbeAndLockPages(master_mdl
, KernelMode
, IoReadAccess
);
745 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
746 Status
= _SEH2_GetExceptionCode();
749 if (!NT_SUCCESS(Status
)) {
750 ERR("MmProbeAndLockPages threw exception %08x\n", Status
);
751 IoFreeMdl(master_mdl
);
755 wtc
->mdl
= master_mdl
;
757 pfns
= (PFN_NUMBER
*)(master_mdl
+ 1);
760 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
761 if (startoffstripe
> i
)
762 stripes
[i
].start
= startoff
- (startoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
763 else if (startoffstripe
== i
)
764 stripes
[i
].start
= startoff
;
766 stripes
[i
].start
= startoff
- (startoff
% c
->chunk_item
->stripe_length
);
768 if (endoffstripe
> i
)
769 stripes
[i
].end
= endoff
- (endoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
770 else if (endoffstripe
== i
)
771 stripes
[i
].end
= endoff
+ 1;
773 stripes
[i
].end
= endoff
- (endoff
% c
->chunk_item
->stripe_length
);
775 if (stripes
[i
].start
!= stripes
[i
].end
) {
776 stripes
[i
].mdl
= IoAllocateMdl(NULL
, (ULONG
)(stripes
[i
].end
- stripes
[i
].start
), FALSE
, FALSE
, NULL
);
777 if (!stripes
[i
].mdl
) {
778 ERR("IoAllocateMdl failed\n");
779 ExFreePool(stripeoff
);
780 return STATUS_INSUFFICIENT_RESOURCES
;
786 RtlZeroMemory(stripeoff
, sizeof(UINT64
) * c
->chunk_item
->num_stripes
);
788 stripenum
= startoffstripe
;
790 while (pos
< length
) {
791 PFN_NUMBER
* stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripenum
].mdl
+ 1);
794 UINT32 writelen
= (UINT32
)min(stripes
[stripenum
].end
- stripes
[stripenum
].start
,
795 c
->chunk_item
->stripe_length
- (stripes
[stripenum
].start
% c
->chunk_item
->stripe_length
));
797 RtlCopyMemory(stripe_pfns
, pfns
, writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
799 stripeoff
[stripenum
] += writelen
;
801 } else if (length
- pos
< c
->chunk_item
->stripe_length
) {
802 RtlCopyMemory(&stripe_pfns
[stripeoff
[stripenum
] >> PAGE_SHIFT
], &pfns
[pos
>> PAGE_SHIFT
], (ULONG
)((length
- pos
) * sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
805 RtlCopyMemory(&stripe_pfns
[stripeoff
[stripenum
] >> PAGE_SHIFT
], &pfns
[pos
>> PAGE_SHIFT
], (ULONG
)(c
->chunk_item
->stripe_length
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
807 stripeoff
[stripenum
] += c
->chunk_item
->stripe_length
;
808 pos
+= c
->chunk_item
->stripe_length
;
811 stripenum
= (stripenum
+ 1) % c
->chunk_item
->num_stripes
;
814 ExFreePool(stripeoff
);
816 return STATUS_SUCCESS
;
819 static NTSTATUS
prepare_raid10_write(_Pre_satisfies_(_Curr_
->chunk_item
->sub_stripes
>0&&_Curr_
->chunk_item
->num_stripes
>=_Curr_
->chunk_item
->sub_stripes
) _In_ chunk
* c
,
820 _In_ UINT64 address
, _In_reads_bytes_(length
) void* data
, _In_ UINT32 length
, _In_ write_stripe
* stripes
,
821 _In_ PIRP Irp
, _In_ UINT64 irp_offset
, _In_ write_data_context
* wtc
) {
822 UINT64 startoff
, endoff
;
823 UINT16 startoffstripe
, endoffstripe
, stripenum
;
824 UINT64 pos
, *stripeoff
;
826 BOOL file_write
= Irp
&& Irp
->MdlAddress
&& (Irp
->MdlAddress
->ByteOffset
== 0);
830 get_raid0_offset(address
- c
->offset
, c
->chunk_item
->stripe_length
, c
->chunk_item
->num_stripes
/ c
->chunk_item
->sub_stripes
, &startoff
, &startoffstripe
);
831 get_raid0_offset(address
+ length
- c
->offset
- 1, c
->chunk_item
->stripe_length
, c
->chunk_item
->num_stripes
/ c
->chunk_item
->sub_stripes
, &endoff
, &endoffstripe
);
833 stripenum
= startoffstripe
;
834 startoffstripe
*= c
->chunk_item
->sub_stripes
;
835 endoffstripe
*= c
->chunk_item
->sub_stripes
;
838 master_mdl
= Irp
->MdlAddress
;
840 pfns
= (PFN_NUMBER
*)(Irp
->MdlAddress
+ 1);
841 pfns
= &pfns
[irp_offset
>> PAGE_SHIFT
];
842 } else if (((ULONG_PTR
)data
% PAGE_SIZE
) != 0) {
843 wtc
->scratch
= ExAllocatePoolWithTag(NonPagedPool
, length
, ALLOC_TAG
);
845 ERR("out of memory\n");
846 return STATUS_INSUFFICIENT_RESOURCES
;
849 RtlCopyMemory(wtc
->scratch
, data
, length
);
851 master_mdl
= IoAllocateMdl(wtc
->scratch
, length
, FALSE
, FALSE
, NULL
);
853 ERR("out of memory\n");
854 return STATUS_INSUFFICIENT_RESOURCES
;
857 MmBuildMdlForNonPagedPool(master_mdl
);
859 wtc
->mdl
= master_mdl
;
861 pfns
= (PFN_NUMBER
*)(master_mdl
+ 1);
863 NTSTATUS Status
= STATUS_SUCCESS
;
865 master_mdl
= IoAllocateMdl(data
, length
, FALSE
, FALSE
, NULL
);
867 ERR("out of memory\n");
868 return STATUS_INSUFFICIENT_RESOURCES
;
872 MmProbeAndLockPages(master_mdl
, KernelMode
, IoReadAccess
);
873 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
874 Status
= _SEH2_GetExceptionCode();
877 if (!NT_SUCCESS(Status
)) {
878 ERR("MmProbeAndLockPages threw exception %08x\n", Status
);
879 IoFreeMdl(master_mdl
);
883 wtc
->mdl
= master_mdl
;
885 pfns
= (PFN_NUMBER
*)(master_mdl
+ 1);
888 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
+= c
->chunk_item
->sub_stripes
) {
891 if (startoffstripe
> i
)
892 stripes
[i
].start
= startoff
- (startoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
893 else if (startoffstripe
== i
)
894 stripes
[i
].start
= startoff
;
896 stripes
[i
].start
= startoff
- (startoff
% c
->chunk_item
->stripe_length
);
898 if (endoffstripe
> i
)
899 stripes
[i
].end
= endoff
- (endoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
900 else if (endoffstripe
== i
)
901 stripes
[i
].end
= endoff
+ 1;
903 stripes
[i
].end
= endoff
- (endoff
% c
->chunk_item
->stripe_length
);
905 stripes
[i
].mdl
= IoAllocateMdl(NULL
, (ULONG
)(stripes
[i
].end
- stripes
[i
].start
), FALSE
, FALSE
, NULL
);
906 if (!stripes
[i
].mdl
) {
907 ERR("IoAllocateMdl failed\n");
908 return STATUS_INSUFFICIENT_RESOURCES
;
911 for (j
= 1; j
< c
->chunk_item
->sub_stripes
; j
++) {
912 stripes
[i
+j
].start
= stripes
[i
].start
;
913 stripes
[i
+j
].end
= stripes
[i
].end
;
914 stripes
[i
+j
].data
= stripes
[i
].data
;
915 stripes
[i
+j
].mdl
= stripes
[i
].mdl
;
921 stripeoff
= ExAllocatePoolWithTag(PagedPool
, sizeof(UINT64
) * c
->chunk_item
->num_stripes
/ c
->chunk_item
->sub_stripes
, ALLOC_TAG
);
923 ERR("out of memory\n");
924 return STATUS_INSUFFICIENT_RESOURCES
;
927 RtlZeroMemory(stripeoff
, sizeof(UINT64
) * c
->chunk_item
->num_stripes
/ c
->chunk_item
->sub_stripes
);
929 while (pos
< length
) {
930 PFN_NUMBER
* stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripenum
* c
->chunk_item
->sub_stripes
].mdl
+ 1);
933 UINT32 writelen
= (UINT32
)min(stripes
[stripenum
* c
->chunk_item
->sub_stripes
].end
- stripes
[stripenum
* c
->chunk_item
->sub_stripes
].start
,
934 c
->chunk_item
->stripe_length
- (stripes
[stripenum
* c
->chunk_item
->sub_stripes
].start
% c
->chunk_item
->stripe_length
));
936 RtlCopyMemory(stripe_pfns
, pfns
, writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
938 stripeoff
[stripenum
] += writelen
;
940 } else if (length
- pos
< c
->chunk_item
->stripe_length
) {
941 RtlCopyMemory(&stripe_pfns
[stripeoff
[stripenum
] >> PAGE_SHIFT
], &pfns
[pos
>> PAGE_SHIFT
], (ULONG
)((length
- pos
) * sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
944 RtlCopyMemory(&stripe_pfns
[stripeoff
[stripenum
] >> PAGE_SHIFT
], &pfns
[pos
>> PAGE_SHIFT
], (ULONG
)(c
->chunk_item
->stripe_length
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
946 stripeoff
[stripenum
] += c
->chunk_item
->stripe_length
;
947 pos
+= c
->chunk_item
->stripe_length
;
950 stripenum
= (stripenum
+ 1) % (c
->chunk_item
->num_stripes
/ c
->chunk_item
->sub_stripes
);
953 ExFreePool(stripeoff
);
955 return STATUS_SUCCESS
;
958 static NTSTATUS
add_partial_stripe(device_extension
* Vcb
, chunk
*c
, UINT64 address
, UINT32 length
, void* data
) {
963 UINT16 num_data_stripes
;
966 num_data_stripes
= c
->chunk_item
->num_stripes
- (c
->chunk_item
->type
& BLOCK_FLAG_RAID5
? 1 : 2);
967 stripe_addr
= address
- ((address
- c
->offset
) % (num_data_stripes
* c
->chunk_item
->stripe_length
));
969 ExAcquireResourceExclusiveLite(&c
->partial_stripes_lock
, TRUE
);
971 le
= c
->partial_stripes
.Flink
;
972 while (le
!= &c
->partial_stripes
) {
973 ps
= CONTAINING_RECORD(le
, partial_stripe
, list_entry
);
975 if (ps
->address
== stripe_addr
) {
976 // update existing entry
978 RtlCopyMemory(ps
->data
+ address
- stripe_addr
, data
, length
);
979 RtlClearBits(&ps
->bmp
, (ULONG
)((address
- stripe_addr
) / Vcb
->superblock
.sector_size
), length
/ Vcb
->superblock
.sector_size
);
981 // if now filled, flush
982 if (RtlAreBitsClear(&ps
->bmp
, 0, (ULONG
)((num_data_stripes
* c
->chunk_item
->stripe_length
) / Vcb
->superblock
.sector_size
))) {
983 Status
= flush_partial_stripe(Vcb
, c
, ps
);
984 if (!NT_SUCCESS(Status
)) {
985 ERR("flush_partial_stripe returned %08x\n", Status
);
989 RemoveEntryList(&ps
->list_entry
);
992 ExFreePool(ps
->bmparr
);
997 Status
= STATUS_SUCCESS
;
999 } else if (ps
->address
> stripe_addr
)
1007 ps
= ExAllocatePoolWithTag(NonPagedPool
, offsetof(partial_stripe
, data
[0]) + (ULONG
)(num_data_stripes
* c
->chunk_item
->stripe_length
), ALLOC_TAG
);
1009 ERR("out of memory\n");
1010 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1014 bmplen
= (ULONG
)sector_align(((num_data_stripes
* c
->chunk_item
->stripe_length
) / (8 * Vcb
->superblock
.sector_size
) + 1), sizeof(ULONG
));
1016 ps
->address
= stripe_addr
;
1017 ps
->bmparr
= ExAllocatePoolWithTag(NonPagedPool
, bmplen
, ALLOC_TAG
);
1019 ERR("out of memory\n");
1021 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1025 RtlInitializeBitMap(&ps
->bmp
, ps
->bmparr
, (ULONG
)((num_data_stripes
* c
->chunk_item
->stripe_length
) / Vcb
->superblock
.sector_size
));
1026 RtlSetAllBits(&ps
->bmp
);
1028 RtlCopyMemory(ps
->data
+ address
- stripe_addr
, data
, length
);
1029 RtlClearBits(&ps
->bmp
, (ULONG
)((address
- stripe_addr
) / Vcb
->superblock
.sector_size
), length
/ Vcb
->superblock
.sector_size
);
1031 InsertHeadList(le
->Blink
, &ps
->list_entry
);
1033 Status
= STATUS_SUCCESS
;
1036 ExReleaseResourceLite(&c
->partial_stripes_lock
);
1046 static NTSTATUS
prepare_raid5_write(device_extension
* Vcb
, chunk
* c
, UINT64 address
, void* data
, UINT32 length
, write_stripe
* stripes
, PIRP Irp
,
1047 UINT64 irp_offset
, ULONG priority
, write_data_context
* wtc
) {
1048 UINT64 startoff
, endoff
, parity_start
, parity_end
;
1049 UINT16 startoffstripe
, endoffstripe
, parity
, num_data_stripes
= c
->chunk_item
->num_stripes
- 1;
1050 UINT64 pos
, parity_pos
, *stripeoff
= NULL
;
1052 BOOL file_write
= Irp
&& Irp
->MdlAddress
&& (Irp
->MdlAddress
->ByteOffset
== 0);
1055 PFN_NUMBER
*pfns
, *parity_pfns
;
1056 log_stripe
* log_stripes
= NULL
;
1058 if ((address
+ length
- c
->offset
) % (num_data_stripes
* c
->chunk_item
->stripe_length
) > 0) {
1059 UINT64 delta
= (address
+ length
- c
->offset
) % (num_data_stripes
* c
->chunk_item
->stripe_length
);
1061 delta
= min(irp_offset
+ length
, delta
);
1062 Status
= add_partial_stripe(Vcb
, c
, address
+ length
- delta
, (UINT32
)delta
, (UINT8
*)data
+ irp_offset
+ length
- delta
);
1063 if (!NT_SUCCESS(Status
)) {
1064 ERR("add_partial_stripe returned %08x\n", Status
);
1068 length
-= (UINT32
)delta
;
1071 if (length
> 0 && (address
- c
->offset
) % (num_data_stripes
* c
->chunk_item
->stripe_length
) > 0) {
1072 UINT64 delta
= (num_data_stripes
* c
->chunk_item
->stripe_length
) - ((address
- c
->offset
) % (num_data_stripes
* c
->chunk_item
->stripe_length
));
1074 Status
= add_partial_stripe(Vcb
, c
, address
, (UINT32
)delta
, (UINT8
*)data
+ irp_offset
);
1075 if (!NT_SUCCESS(Status
)) {
1076 ERR("add_partial_stripe returned %08x\n", Status
);
1081 length
-= (UINT32
)delta
;
1082 irp_offset
+= delta
;
1086 Status
= STATUS_SUCCESS
;
1090 get_raid0_offset(address
- c
->offset
, c
->chunk_item
->stripe_length
, num_data_stripes
, &startoff
, &startoffstripe
);
1091 get_raid0_offset(address
+ length
- c
->offset
- 1, c
->chunk_item
->stripe_length
, num_data_stripes
, &endoff
, &endoffstripe
);
1094 while (pos
< length
) {
1095 parity
= (((address
- c
->offset
+ pos
) / (num_data_stripes
* c
->chunk_item
->stripe_length
)) + num_data_stripes
) % c
->chunk_item
->num_stripes
;
1098 UINT16 stripe
= (parity
+ startoffstripe
+ 1) % c
->chunk_item
->num_stripes
;
1099 ULONG skip
, writelen
;
1102 while (stripe
!= parity
) {
1103 if (i
== startoffstripe
) {
1104 writelen
= (ULONG
)min(length
, c
->chunk_item
->stripe_length
- (startoff
% c
->chunk_item
->stripe_length
));
1106 stripes
[stripe
].start
= startoff
;
1107 stripes
[stripe
].end
= startoff
+ writelen
;
1114 writelen
= (ULONG
)min(length
- pos
, c
->chunk_item
->stripe_length
);
1116 stripes
[stripe
].start
= startoff
- (startoff
% c
->chunk_item
->stripe_length
);
1117 stripes
[stripe
].end
= stripes
[stripe
].start
+ writelen
;
1126 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1132 for (i
= 0; i
< startoffstripe
; i
++) {
1133 stripe
= (parity
+ i
+ 1) % c
->chunk_item
->num_stripes
;
1135 stripes
[stripe
].start
= stripes
[stripe
].end
= startoff
- (startoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
1138 stripes
[parity
].start
= stripes
[parity
].end
= startoff
- (startoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
1140 if (length
- pos
> c
->chunk_item
->num_stripes
* num_data_stripes
* c
->chunk_item
->stripe_length
) {
1141 skip
= (ULONG
)(((length
- pos
) / (c
->chunk_item
->num_stripes
* num_data_stripes
* c
->chunk_item
->stripe_length
)) - 1);
1143 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1144 stripes
[i
].end
+= skip
* c
->chunk_item
->num_stripes
* c
->chunk_item
->stripe_length
;
1147 pos
+= skip
* num_data_stripes
* c
->chunk_item
->num_stripes
* c
->chunk_item
->stripe_length
;
1149 } else if (length
- pos
>= c
->chunk_item
->stripe_length
* num_data_stripes
) {
1150 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1151 stripes
[i
].end
+= c
->chunk_item
->stripe_length
;
1154 pos
+= c
->chunk_item
->stripe_length
* num_data_stripes
;
1156 UINT16 stripe
= (parity
+ 1) % c
->chunk_item
->num_stripes
;
1159 while (stripe
!= parity
) {
1160 if (endoffstripe
== i
) {
1161 stripes
[stripe
].end
= endoff
+ 1;
1163 } else if (endoffstripe
> i
)
1164 stripes
[stripe
].end
= endoff
- (endoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
1167 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1174 parity_start
= 0xffffffffffffffff;
1177 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1178 if (stripes
[i
].start
!= 0 || stripes
[i
].end
!= 0) {
1179 parity_start
= min(stripes
[i
].start
, parity_start
);
1180 parity_end
= max(stripes
[i
].end
, parity_end
);
1184 if (parity_end
== parity_start
) {
1185 Status
= STATUS_SUCCESS
;
1189 parity
= (((address
- c
->offset
) / (num_data_stripes
* c
->chunk_item
->stripe_length
)) + num_data_stripes
) % c
->chunk_item
->num_stripes
;
1190 stripes
[parity
].start
= parity_start
;
1192 parity
= (((address
- c
->offset
+ length
- 1) / (num_data_stripes
* c
->chunk_item
->stripe_length
)) + num_data_stripes
) % c
->chunk_item
->num_stripes
;
1193 stripes
[parity
].end
= parity_end
;
1195 log_stripes
= ExAllocatePoolWithTag(NonPagedPool
, sizeof(log_stripe
) * num_data_stripes
, ALLOC_TAG
);
1197 ERR("out of memory\n");
1198 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1202 RtlZeroMemory(log_stripes
, sizeof(log_stripe
) * num_data_stripes
);
1204 for (i
= 0; i
< num_data_stripes
; i
++) {
1205 log_stripes
[i
].mdl
= IoAllocateMdl(NULL
, (ULONG
)(parity_end
- parity_start
), FALSE
, FALSE
, NULL
);
1206 if (!log_stripes
[i
].mdl
) {
1207 ERR("out of memory\n");
1208 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1212 log_stripes
[i
].mdl
->MdlFlags
|= MDL_PARTIAL
;
1213 log_stripes
[i
].pfns
= (PFN_NUMBER
*)(log_stripes
[i
].mdl
+ 1);
1216 wtc
->parity1
= ExAllocatePoolWithTag(NonPagedPool
, (ULONG
)(parity_end
- parity_start
), ALLOC_TAG
);
1217 if (!wtc
->parity1
) {
1218 ERR("out of memory\n");
1219 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1223 wtc
->parity1_mdl
= IoAllocateMdl(wtc
->parity1
, (ULONG
)(parity_end
- parity_start
), FALSE
, FALSE
, NULL
);
1224 if (!wtc
->parity1_mdl
) {
1225 ERR("out of memory\n");
1226 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1230 MmBuildMdlForNonPagedPool(wtc
->parity1_mdl
);
1233 master_mdl
= Irp
->MdlAddress
;
1234 else if (((ULONG_PTR
)data
% PAGE_SIZE
) != 0) {
1235 wtc
->scratch
= ExAllocatePoolWithTag(NonPagedPool
, length
, ALLOC_TAG
);
1236 if (!wtc
->scratch
) {
1237 ERR("out of memory\n");
1238 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1242 RtlCopyMemory(wtc
->scratch
, (UINT8
*)data
+ irp_offset
, length
);
1244 master_mdl
= IoAllocateMdl(wtc
->scratch
, length
, FALSE
, FALSE
, NULL
);
1246 ERR("out of memory\n");
1247 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1251 MmBuildMdlForNonPagedPool(master_mdl
);
1253 wtc
->mdl
= master_mdl
;
1255 master_mdl
= IoAllocateMdl((UINT8
*)data
+ irp_offset
, length
, FALSE
, FALSE
, NULL
);
1257 ERR("out of memory\n");
1258 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1262 Status
= STATUS_SUCCESS
;
1265 MmProbeAndLockPages(master_mdl
, KernelMode
, IoReadAccess
);
1266 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
1267 Status
= _SEH2_GetExceptionCode();
1270 if (!NT_SUCCESS(Status
)) {
1271 ERR("MmProbeAndLockPages threw exception %08x\n", Status
);
1272 IoFreeMdl(master_mdl
);
1276 wtc
->mdl
= master_mdl
;
1279 pfns
= (PFN_NUMBER
*)(master_mdl
+ 1);
1280 parity_pfns
= (PFN_NUMBER
*)(wtc
->parity1_mdl
+ 1);
1283 pfns
= &pfns
[irp_offset
>> PAGE_SHIFT
];
1285 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1286 if (stripes
[i
].start
!= stripes
[i
].end
) {
1287 stripes
[i
].mdl
= IoAllocateMdl((UINT8
*)MmGetMdlVirtualAddress(master_mdl
) + irp_offset
, (ULONG
)(stripes
[i
].end
- stripes
[i
].start
), FALSE
, FALSE
, NULL
);
1288 if (!stripes
[i
].mdl
) {
1289 ERR("IoAllocateMdl failed\n");
1290 return STATUS_INSUFFICIENT_RESOURCES
;
1295 stripeoff
= ExAllocatePoolWithTag(PagedPool
, sizeof(UINT64
) * c
->chunk_item
->num_stripes
, ALLOC_TAG
);
1297 ERR("out of memory\n");
1298 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1302 RtlZeroMemory(stripeoff
, sizeof(UINT64
) * c
->chunk_item
->num_stripes
);
1307 while (pos
< length
) {
1308 PFN_NUMBER
* stripe_pfns
;
1310 parity
= (((address
- c
->offset
+ pos
) / (num_data_stripes
* c
->chunk_item
->stripe_length
)) + num_data_stripes
) % c
->chunk_item
->num_stripes
;
1313 UINT16 stripe
= (parity
+ startoffstripe
+ 1) % c
->chunk_item
->num_stripes
;
1314 UINT32 writelen
= (UINT32
)min(length
- pos
, min(stripes
[stripe
].end
- stripes
[stripe
].start
,
1315 c
->chunk_item
->stripe_length
- (stripes
[stripe
].start
% c
->chunk_item
->stripe_length
)));
1316 UINT32 maxwritelen
= writelen
;
1318 stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripe
].mdl
+ 1);
1320 RtlCopyMemory(stripe_pfns
, pfns
, writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1322 RtlCopyMemory(log_stripes
[startoffstripe
].pfns
, pfns
, writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1323 log_stripes
[startoffstripe
].pfns
+= writelen
>> PAGE_SHIFT
;
1325 stripeoff
[stripe
] = writelen
;
1328 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1329 i
= startoffstripe
+ 1;
1331 while (stripe
!= parity
) {
1332 stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripe
].mdl
+ 1);
1333 writelen
= (UINT32
)min(length
- pos
, min(stripes
[stripe
].end
- stripes
[stripe
].start
, c
->chunk_item
->stripe_length
));
1338 if (writelen
> maxwritelen
)
1339 maxwritelen
= writelen
;
1341 RtlCopyMemory(stripe_pfns
, &pfns
[pos
>> PAGE_SHIFT
], writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1343 RtlCopyMemory(log_stripes
[i
].pfns
, &pfns
[pos
>> PAGE_SHIFT
], writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1344 log_stripes
[i
].pfns
+= writelen
>> PAGE_SHIFT
;
1346 stripeoff
[stripe
] = writelen
;
1349 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1353 stripe_pfns
= (PFN_NUMBER
*)(stripes
[parity
].mdl
+ 1);
1355 RtlCopyMemory(stripe_pfns
, parity_pfns
, maxwritelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1356 stripeoff
[parity
] = maxwritelen
;
1357 parity_pos
= maxwritelen
;
1358 } else if (length
- pos
>= c
->chunk_item
->stripe_length
* num_data_stripes
) {
1359 UINT16 stripe
= (parity
+ 1) % c
->chunk_item
->num_stripes
;
1362 while (stripe
!= parity
) {
1363 stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripe
].mdl
+ 1);
1365 RtlCopyMemory(&stripe_pfns
[stripeoff
[stripe
] >> PAGE_SHIFT
], &pfns
[pos
>> PAGE_SHIFT
], (ULONG
)(c
->chunk_item
->stripe_length
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
1367 RtlCopyMemory(log_stripes
[i
].pfns
, &pfns
[pos
>> PAGE_SHIFT
], (ULONG
)(c
->chunk_item
->stripe_length
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
1368 log_stripes
[i
].pfns
+= c
->chunk_item
->stripe_length
>> PAGE_SHIFT
;
1370 stripeoff
[stripe
] += c
->chunk_item
->stripe_length
;
1371 pos
+= c
->chunk_item
->stripe_length
;
1373 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1377 stripe_pfns
= (PFN_NUMBER
*)(stripes
[parity
].mdl
+ 1);
1379 RtlCopyMemory(&stripe_pfns
[stripeoff
[parity
] >> PAGE_SHIFT
], &parity_pfns
[parity_pos
>> PAGE_SHIFT
], (ULONG
)(c
->chunk_item
->stripe_length
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
1380 stripeoff
[parity
] += c
->chunk_item
->stripe_length
;
1381 parity_pos
+= c
->chunk_item
->stripe_length
;
1383 UINT16 stripe
= (parity
+ 1) % c
->chunk_item
->num_stripes
;
1384 UINT32 writelen
, maxwritelen
= 0;
1387 while (pos
< length
) {
1388 stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripe
].mdl
+ 1);
1389 writelen
= (UINT32
)min(length
- pos
, min(stripes
[stripe
].end
- stripes
[stripe
].start
, c
->chunk_item
->stripe_length
));
1394 if (writelen
> maxwritelen
)
1395 maxwritelen
= writelen
;
1397 RtlCopyMemory(&stripe_pfns
[stripeoff
[stripe
] >> PAGE_SHIFT
], &pfns
[pos
>> PAGE_SHIFT
], writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1399 RtlCopyMemory(log_stripes
[i
].pfns
, &pfns
[pos
>> PAGE_SHIFT
], writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1400 log_stripes
[i
].pfns
+= writelen
>> PAGE_SHIFT
;
1402 stripeoff
[stripe
] += writelen
;
1405 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1409 stripe_pfns
= (PFN_NUMBER
*)(stripes
[parity
].mdl
+ 1);
1411 RtlCopyMemory(&stripe_pfns
[stripeoff
[parity
] >> PAGE_SHIFT
], &parity_pfns
[parity_pos
>> PAGE_SHIFT
], maxwritelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1415 for (i
= 0; i
< num_data_stripes
; i
++) {
1416 UINT8
* ss
= MmGetSystemAddressForMdlSafe(log_stripes
[i
].mdl
, priority
);
1419 RtlCopyMemory(wtc
->parity1
, ss
, (UINT32
)(parity_end
- parity_start
));
1421 do_xor(wtc
->parity1
, ss
, (UINT32
)(parity_end
- parity_start
));
1424 Status
= STATUS_SUCCESS
;
1428 for (i
= 0; i
< num_data_stripes
; i
++) {
1429 if (log_stripes
[i
].mdl
)
1430 IoFreeMdl(log_stripes
[i
].mdl
);
1433 ExFreePool(log_stripes
);
1437 ExFreePool(stripeoff
);
1442 static NTSTATUS
prepare_raid6_write(device_extension
* Vcb
, chunk
* c
, UINT64 address
, void* data
, UINT32 length
, write_stripe
* stripes
, PIRP Irp
,
1443 UINT64 irp_offset
, ULONG priority
, write_data_context
* wtc
) {
1444 UINT64 startoff
, endoff
, parity_start
, parity_end
;
1445 UINT16 startoffstripe
, endoffstripe
, parity1
, num_data_stripes
= c
->chunk_item
->num_stripes
- 2;
1446 UINT64 pos
, parity_pos
, *stripeoff
= NULL
;
1448 BOOL file_write
= Irp
&& Irp
->MdlAddress
&& (Irp
->MdlAddress
->ByteOffset
== 0);
1451 PFN_NUMBER
*pfns
, *parity1_pfns
, *parity2_pfns
;
1452 log_stripe
* log_stripes
= NULL
;
1454 if ((address
+ length
- c
->offset
) % (num_data_stripes
* c
->chunk_item
->stripe_length
) > 0) {
1455 UINT64 delta
= (address
+ length
- c
->offset
) % (num_data_stripes
* c
->chunk_item
->stripe_length
);
1457 delta
= min(irp_offset
+ length
, delta
);
1458 Status
= add_partial_stripe(Vcb
, c
, address
+ length
- delta
, (UINT32
)delta
, (UINT8
*)data
+ irp_offset
+ length
- delta
);
1459 if (!NT_SUCCESS(Status
)) {
1460 ERR("add_partial_stripe returned %08x\n", Status
);
1464 length
-= (UINT32
)delta
;
1467 if (length
> 0 && (address
- c
->offset
) % (num_data_stripes
* c
->chunk_item
->stripe_length
) > 0) {
1468 UINT64 delta
= (num_data_stripes
* c
->chunk_item
->stripe_length
) - ((address
- c
->offset
) % (num_data_stripes
* c
->chunk_item
->stripe_length
));
1470 Status
= add_partial_stripe(Vcb
, c
, address
, (UINT32
)delta
, (UINT8
*)data
+ irp_offset
);
1471 if (!NT_SUCCESS(Status
)) {
1472 ERR("add_partial_stripe returned %08x\n", Status
);
1477 length
-= (UINT32
)delta
;
1478 irp_offset
+= delta
;
1482 Status
= STATUS_SUCCESS
;
1486 get_raid0_offset(address
- c
->offset
, c
->chunk_item
->stripe_length
, num_data_stripes
, &startoff
, &startoffstripe
);
1487 get_raid0_offset(address
+ length
- c
->offset
- 1, c
->chunk_item
->stripe_length
, num_data_stripes
, &endoff
, &endoffstripe
);
1490 while (pos
< length
) {
1491 parity1
= (((address
- c
->offset
+ pos
) / (num_data_stripes
* c
->chunk_item
->stripe_length
)) + num_data_stripes
) % c
->chunk_item
->num_stripes
;
1494 UINT16 stripe
= (parity1
+ startoffstripe
+ 2) % c
->chunk_item
->num_stripes
;
1495 UINT16 parity2
= (parity1
+ 1) % c
->chunk_item
->num_stripes
;
1496 ULONG skip
, writelen
;
1499 while (stripe
!= parity1
) {
1500 if (i
== startoffstripe
) {
1501 writelen
= (ULONG
)min(length
, c
->chunk_item
->stripe_length
- (startoff
% c
->chunk_item
->stripe_length
));
1503 stripes
[stripe
].start
= startoff
;
1504 stripes
[stripe
].end
= startoff
+ writelen
;
1511 writelen
= (ULONG
)min(length
- pos
, c
->chunk_item
->stripe_length
);
1513 stripes
[stripe
].start
= startoff
- (startoff
% c
->chunk_item
->stripe_length
);
1514 stripes
[stripe
].end
= stripes
[stripe
].start
+ writelen
;
1523 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1529 for (i
= 0; i
< startoffstripe
; i
++) {
1530 stripe
= (parity1
+ i
+ 2) % c
->chunk_item
->num_stripes
;
1532 stripes
[stripe
].start
= stripes
[stripe
].end
= startoff
- (startoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
1535 stripes
[parity1
].start
= stripes
[parity1
].end
= stripes
[parity2
].start
= stripes
[parity2
].end
=
1536 startoff
- (startoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
1538 if (length
- pos
> c
->chunk_item
->num_stripes
* num_data_stripes
* c
->chunk_item
->stripe_length
) {
1539 skip
= (ULONG
)(((length
- pos
) / (c
->chunk_item
->num_stripes
* num_data_stripes
* c
->chunk_item
->stripe_length
)) - 1);
1541 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1542 stripes
[i
].end
+= skip
* c
->chunk_item
->num_stripes
* c
->chunk_item
->stripe_length
;
1545 pos
+= skip
* num_data_stripes
* c
->chunk_item
->num_stripes
* c
->chunk_item
->stripe_length
;
1547 } else if (length
- pos
>= c
->chunk_item
->stripe_length
* num_data_stripes
) {
1548 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1549 stripes
[i
].end
+= c
->chunk_item
->stripe_length
;
1552 pos
+= c
->chunk_item
->stripe_length
* num_data_stripes
;
1554 UINT16 stripe
= (parity1
+ 2) % c
->chunk_item
->num_stripes
;
1557 while (stripe
!= parity1
) {
1558 if (endoffstripe
== i
) {
1559 stripes
[stripe
].end
= endoff
+ 1;
1561 } else if (endoffstripe
> i
)
1562 stripes
[stripe
].end
= endoff
- (endoff
% c
->chunk_item
->stripe_length
) + c
->chunk_item
->stripe_length
;
1565 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1572 parity_start
= 0xffffffffffffffff;
1575 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1576 if (stripes
[i
].start
!= 0 || stripes
[i
].end
!= 0) {
1577 parity_start
= min(stripes
[i
].start
, parity_start
);
1578 parity_end
= max(stripes
[i
].end
, parity_end
);
1582 if (parity_end
== parity_start
) {
1583 Status
= STATUS_SUCCESS
;
1587 parity1
= (((address
- c
->offset
) / (num_data_stripes
* c
->chunk_item
->stripe_length
)) + num_data_stripes
) % c
->chunk_item
->num_stripes
;
1588 stripes
[parity1
].start
= stripes
[(parity1
+ 1) % c
->chunk_item
->num_stripes
].start
= parity_start
;
1590 parity1
= (((address
- c
->offset
+ length
- 1) / (num_data_stripes
* c
->chunk_item
->stripe_length
)) + num_data_stripes
) % c
->chunk_item
->num_stripes
;
1591 stripes
[parity1
].end
= stripes
[(parity1
+ 1) % c
->chunk_item
->num_stripes
].end
= parity_end
;
1593 log_stripes
= ExAllocatePoolWithTag(NonPagedPool
, sizeof(log_stripe
) * num_data_stripes
, ALLOC_TAG
);
1595 ERR("out of memory\n");
1596 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1600 RtlZeroMemory(log_stripes
, sizeof(log_stripe
) * num_data_stripes
);
1602 for (i
= 0; i
< num_data_stripes
; i
++) {
1603 log_stripes
[i
].mdl
= IoAllocateMdl(NULL
, (ULONG
)(parity_end
- parity_start
), FALSE
, FALSE
, NULL
);
1604 if (!log_stripes
[i
].mdl
) {
1605 ERR("out of memory\n");
1606 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1610 log_stripes
[i
].mdl
->MdlFlags
|= MDL_PARTIAL
;
1611 log_stripes
[i
].pfns
= (PFN_NUMBER
*)(log_stripes
[i
].mdl
+ 1);
1614 wtc
->parity1
= ExAllocatePoolWithTag(NonPagedPool
, (ULONG
)(parity_end
- parity_start
), ALLOC_TAG
);
1615 if (!wtc
->parity1
) {
1616 ERR("out of memory\n");
1617 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1621 wtc
->parity2
= ExAllocatePoolWithTag(NonPagedPool
, (ULONG
)(parity_end
- parity_start
), ALLOC_TAG
);
1622 if (!wtc
->parity2
) {
1623 ERR("out of memory\n");
1624 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1628 wtc
->parity1_mdl
= IoAllocateMdl(wtc
->parity1
, (ULONG
)(parity_end
- parity_start
), FALSE
, FALSE
, NULL
);
1629 if (!wtc
->parity1_mdl
) {
1630 ERR("out of memory\n");
1631 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1635 MmBuildMdlForNonPagedPool(wtc
->parity1_mdl
);
1637 wtc
->parity2_mdl
= IoAllocateMdl(wtc
->parity2
, (ULONG
)(parity_end
- parity_start
), FALSE
, FALSE
, NULL
);
1638 if (!wtc
->parity2_mdl
) {
1639 ERR("out of memory\n");
1640 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1644 MmBuildMdlForNonPagedPool(wtc
->parity2_mdl
);
1647 master_mdl
= Irp
->MdlAddress
;
1648 else if (((ULONG_PTR
)data
% PAGE_SIZE
) != 0) {
1649 wtc
->scratch
= ExAllocatePoolWithTag(NonPagedPool
, length
, ALLOC_TAG
);
1650 if (!wtc
->scratch
) {
1651 ERR("out of memory\n");
1652 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1656 RtlCopyMemory(wtc
->scratch
, (UINT8
*)data
+ irp_offset
, length
);
1658 master_mdl
= IoAllocateMdl(wtc
->scratch
, length
, FALSE
, FALSE
, NULL
);
1660 ERR("out of memory\n");
1661 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1665 MmBuildMdlForNonPagedPool(master_mdl
);
1667 wtc
->mdl
= master_mdl
;
1669 master_mdl
= IoAllocateMdl((UINT8
*)data
+ irp_offset
, length
, FALSE
, FALSE
, NULL
);
1671 ERR("out of memory\n");
1672 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1676 Status
= STATUS_SUCCESS
;
1679 MmProbeAndLockPages(master_mdl
, KernelMode
, IoReadAccess
);
1680 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
1681 Status
= _SEH2_GetExceptionCode();
1684 if (!NT_SUCCESS(Status
)) {
1685 ERR("MmProbeAndLockPages threw exception %08x\n", Status
);
1686 IoFreeMdl(master_mdl
);
1690 wtc
->mdl
= master_mdl
;
1693 pfns
= (PFN_NUMBER
*)(master_mdl
+ 1);
1694 parity1_pfns
= (PFN_NUMBER
*)(wtc
->parity1_mdl
+ 1);
1695 parity2_pfns
= (PFN_NUMBER
*)(wtc
->parity2_mdl
+ 1);
1698 pfns
= &pfns
[irp_offset
>> PAGE_SHIFT
];
1700 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1701 if (stripes
[i
].start
!= stripes
[i
].end
) {
1702 stripes
[i
].mdl
= IoAllocateMdl((UINT8
*)MmGetMdlVirtualAddress(master_mdl
) + irp_offset
, (ULONG
)(stripes
[i
].end
- stripes
[i
].start
), FALSE
, FALSE
, NULL
);
1703 if (!stripes
[i
].mdl
) {
1704 ERR("IoAllocateMdl failed\n");
1705 return STATUS_INSUFFICIENT_RESOURCES
;
1710 stripeoff
= ExAllocatePoolWithTag(PagedPool
, sizeof(UINT64
) * c
->chunk_item
->num_stripes
, ALLOC_TAG
);
1712 ERR("out of memory\n");
1713 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1717 RtlZeroMemory(stripeoff
, sizeof(UINT64
) * c
->chunk_item
->num_stripes
);
1722 while (pos
< length
) {
1723 PFN_NUMBER
* stripe_pfns
;
1725 parity1
= (((address
- c
->offset
+ pos
) / (num_data_stripes
* c
->chunk_item
->stripe_length
)) + num_data_stripes
) % c
->chunk_item
->num_stripes
;
1728 UINT16 stripe
= (parity1
+ startoffstripe
+ 2) % c
->chunk_item
->num_stripes
, parity2
;
1729 UINT32 writelen
= (UINT32
)min(length
- pos
, min(stripes
[stripe
].end
- stripes
[stripe
].start
,
1730 c
->chunk_item
->stripe_length
- (stripes
[stripe
].start
% c
->chunk_item
->stripe_length
)));
1731 UINT32 maxwritelen
= writelen
;
1733 stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripe
].mdl
+ 1);
1735 RtlCopyMemory(stripe_pfns
, pfns
, writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1737 RtlCopyMemory(log_stripes
[startoffstripe
].pfns
, pfns
, writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1738 log_stripes
[startoffstripe
].pfns
+= writelen
>> PAGE_SHIFT
;
1740 stripeoff
[stripe
] = writelen
;
1743 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1744 i
= startoffstripe
+ 1;
1746 while (stripe
!= parity1
) {
1747 stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripe
].mdl
+ 1);
1748 writelen
= (UINT32
)min(length
- pos
, min(stripes
[stripe
].end
- stripes
[stripe
].start
, c
->chunk_item
->stripe_length
));
1753 if (writelen
> maxwritelen
)
1754 maxwritelen
= writelen
;
1756 RtlCopyMemory(stripe_pfns
, &pfns
[pos
>> PAGE_SHIFT
], writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1758 RtlCopyMemory(log_stripes
[i
].pfns
, &pfns
[pos
>> PAGE_SHIFT
], writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1759 log_stripes
[i
].pfns
+= writelen
>> PAGE_SHIFT
;
1761 stripeoff
[stripe
] = writelen
;
1764 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1768 stripe_pfns
= (PFN_NUMBER
*)(stripes
[parity1
].mdl
+ 1);
1769 RtlCopyMemory(stripe_pfns
, parity1_pfns
, maxwritelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1770 stripeoff
[parity1
] = maxwritelen
;
1772 parity2
= (parity1
+ 1) % c
->chunk_item
->num_stripes
;
1774 stripe_pfns
= (PFN_NUMBER
*)(stripes
[parity2
].mdl
+ 1);
1775 RtlCopyMemory(stripe_pfns
, parity2_pfns
, maxwritelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1776 stripeoff
[parity2
] = maxwritelen
;
1778 parity_pos
= maxwritelen
;
1779 } else if (length
- pos
>= c
->chunk_item
->stripe_length
* num_data_stripes
) {
1780 UINT16 stripe
= (parity1
+ 2) % c
->chunk_item
->num_stripes
, parity2
;
1783 while (stripe
!= parity1
) {
1784 stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripe
].mdl
+ 1);
1786 RtlCopyMemory(&stripe_pfns
[stripeoff
[stripe
] >> PAGE_SHIFT
], &pfns
[pos
>> PAGE_SHIFT
], (ULONG
)(c
->chunk_item
->stripe_length
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
1788 RtlCopyMemory(log_stripes
[i
].pfns
, &pfns
[pos
>> PAGE_SHIFT
], (ULONG
)(c
->chunk_item
->stripe_length
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
1789 log_stripes
[i
].pfns
+= c
->chunk_item
->stripe_length
>> PAGE_SHIFT
;
1791 stripeoff
[stripe
] += c
->chunk_item
->stripe_length
;
1792 pos
+= c
->chunk_item
->stripe_length
;
1794 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1798 stripe_pfns
= (PFN_NUMBER
*)(stripes
[parity1
].mdl
+ 1);
1799 RtlCopyMemory(&stripe_pfns
[stripeoff
[parity1
] >> PAGE_SHIFT
], &parity1_pfns
[parity_pos
>> PAGE_SHIFT
], (ULONG
)(c
->chunk_item
->stripe_length
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
1800 stripeoff
[parity1
] += c
->chunk_item
->stripe_length
;
1802 parity2
= (parity1
+ 1) % c
->chunk_item
->num_stripes
;
1804 stripe_pfns
= (PFN_NUMBER
*)(stripes
[parity2
].mdl
+ 1);
1805 RtlCopyMemory(&stripe_pfns
[stripeoff
[parity2
] >> PAGE_SHIFT
], &parity2_pfns
[parity_pos
>> PAGE_SHIFT
], (ULONG
)(c
->chunk_item
->stripe_length
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
));
1806 stripeoff
[parity2
] += c
->chunk_item
->stripe_length
;
1808 parity_pos
+= c
->chunk_item
->stripe_length
;
1810 UINT16 stripe
= (parity1
+ 2) % c
->chunk_item
->num_stripes
, parity2
;
1811 UINT32 writelen
, maxwritelen
= 0;
1814 while (pos
< length
) {
1815 stripe_pfns
= (PFN_NUMBER
*)(stripes
[stripe
].mdl
+ 1);
1816 writelen
= (UINT32
)min(length
- pos
, min(stripes
[stripe
].end
- stripes
[stripe
].start
, c
->chunk_item
->stripe_length
));
1821 if (writelen
> maxwritelen
)
1822 maxwritelen
= writelen
;
1824 RtlCopyMemory(&stripe_pfns
[stripeoff
[stripe
] >> PAGE_SHIFT
], &pfns
[pos
>> PAGE_SHIFT
], writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1826 RtlCopyMemory(log_stripes
[i
].pfns
, &pfns
[pos
>> PAGE_SHIFT
], writelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1827 log_stripes
[i
].pfns
+= writelen
>> PAGE_SHIFT
;
1829 stripeoff
[stripe
] += writelen
;
1832 stripe
= (stripe
+ 1) % c
->chunk_item
->num_stripes
;
1836 stripe_pfns
= (PFN_NUMBER
*)(stripes
[parity1
].mdl
+ 1);
1837 RtlCopyMemory(&stripe_pfns
[stripeoff
[parity1
] >> PAGE_SHIFT
], &parity1_pfns
[parity_pos
>> PAGE_SHIFT
], maxwritelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1839 parity2
= (parity1
+ 1) % c
->chunk_item
->num_stripes
;
1841 stripe_pfns
= (PFN_NUMBER
*)(stripes
[parity2
].mdl
+ 1);
1842 RtlCopyMemory(&stripe_pfns
[stripeoff
[parity2
] >> PAGE_SHIFT
], &parity2_pfns
[parity_pos
>> PAGE_SHIFT
], maxwritelen
* sizeof(PFN_NUMBER
) >> PAGE_SHIFT
);
1846 for (i
= 0; i
< num_data_stripes
; i
++) {
1847 UINT8
* ss
= MmGetSystemAddressForMdlSafe(log_stripes
[c
->chunk_item
->num_stripes
- 3 - i
].mdl
, priority
);
1850 RtlCopyMemory(wtc
->parity1
, ss
, (ULONG
)(parity_end
- parity_start
));
1851 RtlCopyMemory(wtc
->parity2
, ss
, (ULONG
)(parity_end
- parity_start
));
1853 do_xor(wtc
->parity1
, ss
, (UINT32
)(parity_end
- parity_start
));
1855 galois_double(wtc
->parity2
, (UINT32
)(parity_end
- parity_start
));
1856 do_xor(wtc
->parity2
, ss
, (UINT32
)(parity_end
- parity_start
));
1860 Status
= STATUS_SUCCESS
;
1864 for (i
= 0; i
< num_data_stripes
; i
++) {
1865 if (log_stripes
[i
].mdl
)
1866 IoFreeMdl(log_stripes
[i
].mdl
);
1869 ExFreePool(log_stripes
);
1873 ExFreePool(stripeoff
);
1878 NTSTATUS
write_data(_In_ device_extension
* Vcb
, _In_ UINT64 address
, _In_reads_bytes_(length
) void* data
, _In_ UINT32 length
, _In_ write_data_context
* wtc
,
1879 _In_opt_ PIRP Irp
, _In_opt_ chunk
* c
, _In_ BOOL file_write
, _In_ UINT64 irp_offset
, _In_ ULONG priority
) {
1882 CHUNK_ITEM_STRIPE
* cis
;
1883 write_data_stripe
* stripe
;
1884 write_stripe
* stripes
= NULL
;
1885 UINT64 total_writing
= 0;
1886 ULONG allowed_missing
, missing
;
1888 TRACE("(%p, %llx, %p, %x)\n", Vcb
, address
, data
, length
);
1891 c
= get_chunk_from_address(Vcb
, address
);
1893 ERR("could not get chunk for address %llx\n", address
);
1894 return STATUS_INTERNAL_ERROR
;
1898 stripes
= ExAllocatePoolWithTag(PagedPool
, sizeof(write_stripe
) * c
->chunk_item
->num_stripes
, ALLOC_TAG
);
1900 ERR("out of memory\n");
1901 return STATUS_INSUFFICIENT_RESOURCES
;
1904 RtlZeroMemory(stripes
, sizeof(write_stripe
) * c
->chunk_item
->num_stripes
);
1906 cis
= (CHUNK_ITEM_STRIPE
*)&c
->chunk_item
[1];
1908 if (c
->chunk_item
->type
& BLOCK_FLAG_RAID0
) {
1909 Status
= prepare_raid0_write(c
, address
, data
, length
, stripes
, file_write
? Irp
: NULL
, irp_offset
, wtc
);
1910 if (!NT_SUCCESS(Status
)) {
1911 ERR("prepare_raid0_write returned %08x\n", Status
);
1912 goto prepare_failed
;
1915 allowed_missing
= 0;
1916 } else if (c
->chunk_item
->type
& BLOCK_FLAG_RAID10
) {
1917 Status
= prepare_raid10_write(c
, address
, data
, length
, stripes
, file_write
? Irp
: NULL
, irp_offset
, wtc
);
1918 if (!NT_SUCCESS(Status
)) {
1919 ERR("prepare_raid10_write returned %08x\n", Status
);
1920 goto prepare_failed
;
1923 allowed_missing
= 1;
1924 } else if (c
->chunk_item
->type
& BLOCK_FLAG_RAID5
) {
1925 Status
= prepare_raid5_write(Vcb
, c
, address
, data
, length
, stripes
, file_write
? Irp
: NULL
, irp_offset
, priority
, wtc
);
1926 if (!NT_SUCCESS(Status
)) {
1927 ERR("prepare_raid5_write returned %08x\n", Status
);
1928 goto prepare_failed
;
1931 allowed_missing
= 1;
1932 } else if (c
->chunk_item
->type
& BLOCK_FLAG_RAID6
) {
1933 Status
= prepare_raid6_write(Vcb
, c
, address
, data
, length
, stripes
, file_write
? Irp
: NULL
, irp_offset
, priority
, wtc
);
1934 if (!NT_SUCCESS(Status
)) {
1935 ERR("prepare_raid6_write returned %08x\n", Status
);
1936 goto prepare_failed
;
1939 allowed_missing
= 2;
1940 } else { // write same data to every location - SINGLE, DUP, RAID1
1941 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1942 stripes
[i
].start
= address
- c
->offset
;
1943 stripes
[i
].end
= stripes
[i
].start
+ length
;
1944 stripes
[i
].data
= data
;
1945 stripes
[i
].irp_offset
= irp_offset
;
1947 if (c
->devices
[i
]->devobj
) {
1950 ULONG writelen
= (ULONG
)(stripes
[i
].end
- stripes
[i
].start
);
1952 va
= (UINT8
*)MmGetMdlVirtualAddress(Irp
->MdlAddress
) + stripes
[i
].irp_offset
;
1954 stripes
[i
].mdl
= IoAllocateMdl(va
, writelen
, FALSE
, FALSE
, NULL
);
1955 if (!stripes
[i
].mdl
) {
1956 ERR("IoAllocateMdl failed\n");
1957 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1958 goto prepare_failed
;
1961 IoBuildPartialMdl(Irp
->MdlAddress
, stripes
[i
].mdl
, va
, writelen
);
1963 stripes
[i
].mdl
= IoAllocateMdl(stripes
[i
].data
, (ULONG
)(stripes
[i
].end
- stripes
[i
].start
), FALSE
, FALSE
, NULL
);
1964 if (!stripes
[i
].mdl
) {
1965 ERR("IoAllocateMdl failed\n");
1966 Status
= STATUS_INSUFFICIENT_RESOURCES
;
1967 goto prepare_failed
;
1970 Status
= STATUS_SUCCESS
;
1973 MmProbeAndLockPages(stripes
[i
].mdl
, KernelMode
, IoReadAccess
);
1974 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
1975 Status
= _SEH2_GetExceptionCode();
1978 if (!NT_SUCCESS(Status
)) {
1979 ERR("MmProbeAndLockPages threw exception %08x\n", Status
);
1980 IoFreeMdl(stripes
[i
].mdl
);
1981 stripes
[i
].mdl
= NULL
;
1982 goto prepare_failed
;
1988 allowed_missing
= c
->chunk_item
->num_stripes
- 1;
1992 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
1993 if (!c
->devices
[i
]->devobj
)
1997 if (missing
> allowed_missing
) {
1998 ERR("cannot write as %u missing devices (maximum %u)\n", missing
, allowed_missing
);
1999 Status
= STATUS_DEVICE_NOT_READY
;
2000 goto prepare_failed
;
2003 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
2004 PIO_STACK_LOCATION IrpSp
;
2006 stripe
= ExAllocatePoolWithTag(NonPagedPool
, sizeof(write_data_stripe
), ALLOC_TAG
);
2008 ERR("out of memory\n");
2009 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2013 if (stripes
[i
].start
== stripes
[i
].end
|| !c
->devices
[i
]->devobj
) {
2014 stripe
->status
= WriteDataStatus_Ignore
;
2016 stripe
->buf
= stripes
[i
].data
;
2019 stripe
->context
= (struct _write_data_context
*)wtc
;
2020 stripe
->buf
= stripes
[i
].data
;
2021 stripe
->device
= c
->devices
[i
];
2022 RtlZeroMemory(&stripe
->iosb
, sizeof(IO_STATUS_BLOCK
));
2023 stripe
->status
= WriteDataStatus_Pending
;
2024 stripe
->mdl
= stripes
[i
].mdl
;
2027 stripe
->Irp
= IoAllocateIrp(stripe
->device
->devobj
->StackSize
, FALSE
);
2030 ERR("IoAllocateIrp failed\n");
2031 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2035 stripe
->Irp
= IoMakeAssociatedIrp(Irp
, stripe
->device
->devobj
->StackSize
);
2038 ERR("IoMakeAssociatedIrp failed\n");
2039 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2044 IrpSp
= IoGetNextIrpStackLocation(stripe
->Irp
);
2045 IrpSp
->MajorFunction
= IRP_MJ_WRITE
;
2047 if (stripe
->device
->devobj
->Flags
& DO_BUFFERED_IO
) {
2048 stripe
->Irp
->AssociatedIrp
.SystemBuffer
= MmGetSystemAddressForMdlSafe(stripes
[i
].mdl
, priority
);
2050 stripe
->Irp
->Flags
= IRP_BUFFERED_IO
;
2051 } else if (stripe
->device
->devobj
->Flags
& DO_DIRECT_IO
)
2052 stripe
->Irp
->MdlAddress
= stripe
->mdl
;
2054 stripe
->Irp
->UserBuffer
= MmGetSystemAddressForMdlSafe(stripes
[i
].mdl
, priority
);
2056 #ifdef DEBUG_PARANOID
2057 if (stripes
[i
].end
< stripes
[i
].start
) {
2058 ERR("trying to write stripe with negative length (%llx < %llx)\n", stripes
[i
].end
, stripes
[i
].start
);
2063 IrpSp
->Parameters
.Write
.Length
= (ULONG
)(stripes
[i
].end
- stripes
[i
].start
);
2064 IrpSp
->Parameters
.Write
.ByteOffset
.QuadPart
= stripes
[i
].start
+ cis
[i
].offset
;
2066 total_writing
+= IrpSp
->Parameters
.Write
.Length
;
2068 stripe
->Irp
->UserIosb
= &stripe
->iosb
;
2069 wtc
->stripes_left
++;
2071 IoSetCompletionRoutine(stripe
->Irp
, write_data_completion
, stripe
, TRUE
, TRUE
, TRUE
);
2074 InsertTailList(&wtc
->stripes
, &stripe
->list_entry
);
2078 fFsRtlUpdateDiskCounters(0, total_writing
);
2080 Status
= STATUS_SUCCESS
;
2084 if (stripes
) ExFreePool(stripes
);
2086 if (!NT_SUCCESS(Status
))
2087 free_write_data_stripes(wtc
);
2092 for (i
= 0; i
< c
->chunk_item
->num_stripes
; i
++) {
2093 if (stripes
[i
].mdl
&& (i
== 0 || stripes
[i
].mdl
!= stripes
[i
-1].mdl
)) {
2094 if (stripes
[i
].mdl
->MdlFlags
& MDL_PAGES_LOCKED
)
2095 MmUnlockPages(stripes
[i
].mdl
);
2097 IoFreeMdl(stripes
[i
].mdl
);
2101 if (wtc
->parity1_mdl
) {
2102 if (wtc
->parity1_mdl
->MdlFlags
& MDL_PAGES_LOCKED
)
2103 MmUnlockPages(wtc
->parity1_mdl
);
2105 IoFreeMdl(wtc
->parity1_mdl
);
2106 wtc
->parity1_mdl
= NULL
;
2109 if (wtc
->parity2_mdl
) {
2110 if (wtc
->parity2_mdl
->MdlFlags
& MDL_PAGES_LOCKED
)
2111 MmUnlockPages(wtc
->parity2_mdl
);
2113 IoFreeMdl(wtc
->parity2_mdl
);
2114 wtc
->parity2_mdl
= NULL
;
2118 if (wtc
->mdl
->MdlFlags
& MDL_PAGES_LOCKED
)
2119 MmUnlockPages(wtc
->mdl
);
2121 IoFreeMdl(wtc
->mdl
);
2126 ExFreePool(wtc
->parity1
);
2127 wtc
->parity1
= NULL
;
2131 ExFreePool(wtc
->parity2
);
2132 wtc
->parity2
= NULL
;
2136 ExFreePool(wtc
->scratch
);
2137 wtc
->scratch
= NULL
;
2140 ExFreePool(stripes
);
2144 void get_raid56_lock_range(chunk
* c
, UINT64 address
, UINT64 length
, UINT64
* lockaddr
, UINT64
* locklen
) {
2145 UINT64 startoff
, endoff
;
2146 UINT16 startoffstripe
, endoffstripe
, datastripes
;
2148 datastripes
= c
->chunk_item
->num_stripes
- (c
->chunk_item
->type
& BLOCK_FLAG_RAID5
? 1 : 2);
2150 get_raid0_offset(address
- c
->offset
, c
->chunk_item
->stripe_length
, datastripes
, &startoff
, &startoffstripe
);
2151 get_raid0_offset(address
+ length
- c
->offset
- 1, c
->chunk_item
->stripe_length
, datastripes
, &endoff
, &endoffstripe
);
2153 startoff
-= startoff
% c
->chunk_item
->stripe_length
;
2154 endoff
= sector_align(endoff
, c
->chunk_item
->stripe_length
);
2156 *lockaddr
= c
->offset
+ (startoff
* datastripes
);
2157 *locklen
= (endoff
- startoff
) * datastripes
;
2160 NTSTATUS
write_data_complete(device_extension
* Vcb
, UINT64 address
, void* data
, UINT32 length
, PIRP Irp
, chunk
* c
, BOOL file_write
, UINT64 irp_offset
, ULONG priority
) {
2161 write_data_context wtc
;
2163 UINT64 lockaddr
, locklen
;
2165 KeInitializeEvent(&wtc
.Event
, NotificationEvent
, FALSE
);
2166 InitializeListHead(&wtc
.stripes
);
2167 wtc
.stripes_left
= 0;
2168 wtc
.parity1
= wtc
.parity2
= wtc
.scratch
= NULL
;
2169 wtc
.mdl
= wtc
.parity1_mdl
= wtc
.parity2_mdl
= NULL
;
2172 c
= get_chunk_from_address(Vcb
, address
);
2174 ERR("could not get chunk for address %llx\n", address
);
2175 return STATUS_INTERNAL_ERROR
;
2179 if (c
->chunk_item
->type
& BLOCK_FLAG_RAID5
|| c
->chunk_item
->type
& BLOCK_FLAG_RAID6
) {
2180 get_raid56_lock_range(c
, address
, length
, &lockaddr
, &locklen
);
2181 chunk_lock_range(Vcb
, c
, lockaddr
, locklen
);
2185 Status
= write_data(Vcb
, address
, data
, length
, &wtc
, Irp
, c
, file_write
, irp_offset
, priority
);
2186 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
2187 Status
= _SEH2_GetExceptionCode();
2190 if (!NT_SUCCESS(Status
)) {
2191 ERR("write_data returned %08x\n", Status
);
2193 if (c
->chunk_item
->type
& BLOCK_FLAG_RAID5
|| c
->chunk_item
->type
& BLOCK_FLAG_RAID6
)
2194 chunk_unlock_range(Vcb
, c
, lockaddr
, locklen
);
2196 free_write_data_stripes(&wtc
);
2200 if (wtc
.stripes
.Flink
!= &wtc
.stripes
) {
2201 // launch writes and wait
2202 LIST_ENTRY
* le
= wtc
.stripes
.Flink
;
2203 BOOL no_wait
= TRUE
;
2205 while (le
!= &wtc
.stripes
) {
2206 write_data_stripe
* stripe
= CONTAINING_RECORD(le
, write_data_stripe
, list_entry
);
2208 if (stripe
->status
!= WriteDataStatus_Ignore
) {
2209 IoCallDriver(stripe
->device
->devobj
, stripe
->Irp
);
2217 KeWaitForSingleObject(&wtc
.Event
, Executive
, KernelMode
, FALSE
, NULL
);
2219 le
= wtc
.stripes
.Flink
;
2220 while (le
!= &wtc
.stripes
) {
2221 write_data_stripe
* stripe
= CONTAINING_RECORD(le
, write_data_stripe
, list_entry
);
2223 if (stripe
->status
!= WriteDataStatus_Ignore
&& !NT_SUCCESS(stripe
->iosb
.Status
)) {
2224 Status
= stripe
->iosb
.Status
;
2226 log_device_error(Vcb
, stripe
->device
, BTRFS_DEV_STAT_WRITE_ERRORS
);
2233 free_write_data_stripes(&wtc
);
2236 if (c
->chunk_item
->type
& BLOCK_FLAG_RAID5
|| c
->chunk_item
->type
& BLOCK_FLAG_RAID6
)
2237 chunk_unlock_range(Vcb
, c
, lockaddr
, locklen
);
2242 _Function_class_(IO_COMPLETION_ROUTINE
)
2244 static NTSTATUS NTAPI
write_data_completion(PDEVICE_OBJECT DeviceObject
, PIRP Irp
, PVOID conptr
) {
2246 static NTSTATUS
write_data_completion(PDEVICE_OBJECT DeviceObject
, PIRP Irp
, PVOID conptr
) {
2248 write_data_stripe
* stripe
= conptr
;
2249 write_data_context
* context
= (write_data_context
*)stripe
->context
;
2252 UNUSED(DeviceObject
);
2254 // FIXME - we need a lock here
2256 if (stripe
->status
== WriteDataStatus_Cancelling
) {
2257 stripe
->status
= WriteDataStatus_Cancelled
;
2261 stripe
->iosb
= Irp
->IoStatus
;
2263 if (NT_SUCCESS(Irp
->IoStatus
.Status
)) {
2264 stripe
->status
= WriteDataStatus_Success
;
2266 le
= context
->stripes
.Flink
;
2268 stripe
->status
= WriteDataStatus_Error
;
2270 while (le
!= &context
->stripes
) {
2271 write_data_stripe
* s2
= CONTAINING_RECORD(le
, write_data_stripe
, list_entry
);
2273 if (s2
->status
== WriteDataStatus_Pending
) {
2274 s2
->status
= WriteDataStatus_Cancelling
;
2275 IoCancelIrp(s2
->Irp
);
2283 if (InterlockedDecrement(&context
->stripes_left
) == 0)
2284 KeSetEvent(&context
->Event
, 0, FALSE
);
2286 return STATUS_MORE_PROCESSING_REQUIRED
;
2289 void free_write_data_stripes(write_data_context
* wtc
) {
2291 PMDL last_mdl
= NULL
;
2293 if (wtc
->parity1_mdl
) {
2294 if (wtc
->parity1_mdl
->MdlFlags
& MDL_PAGES_LOCKED
)
2295 MmUnlockPages(wtc
->parity1_mdl
);
2297 IoFreeMdl(wtc
->parity1_mdl
);
2300 if (wtc
->parity2_mdl
) {
2301 if (wtc
->parity2_mdl
->MdlFlags
& MDL_PAGES_LOCKED
)
2302 MmUnlockPages(wtc
->parity2_mdl
);
2304 IoFreeMdl(wtc
->parity2_mdl
);
2308 if (wtc
->mdl
->MdlFlags
& MDL_PAGES_LOCKED
)
2309 MmUnlockPages(wtc
->mdl
);
2311 IoFreeMdl(wtc
->mdl
);
2315 ExFreePool(wtc
->parity1
);
2318 ExFreePool(wtc
->parity2
);
2321 ExFreePool(wtc
->scratch
);
2323 le
= wtc
->stripes
.Flink
;
2324 while (le
!= &wtc
->stripes
) {
2325 write_data_stripe
* stripe
= CONTAINING_RECORD(le
, write_data_stripe
, list_entry
);
2327 if (stripe
->mdl
&& stripe
->mdl
!= last_mdl
) {
2328 if (stripe
->mdl
->MdlFlags
& MDL_PAGES_LOCKED
)
2329 MmUnlockPages(stripe
->mdl
);
2331 IoFreeMdl(stripe
->mdl
);
2334 last_mdl
= stripe
->mdl
;
2339 while (!IsListEmpty(&wtc
->stripes
)) {
2340 write_data_stripe
* stripe
= CONTAINING_RECORD(RemoveHeadList(&wtc
->stripes
), write_data_stripe
, list_entry
);
2346 void add_extent(_In_ fcb
* fcb
, _In_ LIST_ENTRY
* prevextle
, _In_ __drv_aliasesMem extent
* newext
) {
2347 LIST_ENTRY
* le
= prevextle
->Flink
;
2349 while (le
!= &fcb
->extents
) {
2350 extent
* ext
= CONTAINING_RECORD(le
, extent
, list_entry
);
2352 if (ext
->offset
>= newext
->offset
) {
2353 InsertHeadList(ext
->list_entry
.Blink
, &newext
->list_entry
);
2360 InsertTailList(&fcb
->extents
, &newext
->list_entry
);
2363 NTSTATUS
excise_extents(device_extension
* Vcb
, fcb
* fcb
, UINT64 start_data
, UINT64 end_data
, PIRP Irp
, LIST_ENTRY
* rollback
) {
2367 le
= fcb
->extents
.Flink
;
2369 while (le
!= &fcb
->extents
) {
2370 LIST_ENTRY
* le2
= le
->Flink
;
2371 extent
* ext
= CONTAINING_RECORD(le
, extent
, list_entry
);
2372 EXTENT_DATA
* ed
= &ext
->extent_data
;
2373 EXTENT_DATA2
* ed2
= NULL
;
2377 if (ed
->type
!= EXTENT_TYPE_INLINE
)
2378 ed2
= (EXTENT_DATA2
*)ed
->data
;
2380 len
= ed
->type
== EXTENT_TYPE_INLINE
? ed
->decoded_size
: ed2
->num_bytes
;
2382 if (ext
->offset
< end_data
&& ext
->offset
+ len
> start_data
) {
2383 if (ed
->type
== EXTENT_TYPE_INLINE
) {
2384 if (start_data
<= ext
->offset
&& end_data
>= ext
->offset
+ len
) { // remove all
2385 remove_fcb_extent(fcb
, ext
, rollback
);
2387 fcb
->inode_item
.st_blocks
-= len
;
2388 fcb
->inode_item_changed
= TRUE
;
2390 ERR("trying to split inline extent\n");
2391 #ifdef DEBUG_PARANOID
2394 return STATUS_INTERNAL_ERROR
;
2396 } else if (ed
->type
!= EXTENT_TYPE_INLINE
) {
2397 if (start_data
<= ext
->offset
&& end_data
>= ext
->offset
+ len
) { // remove all
2398 if (ed2
->size
!= 0) {
2401 fcb
->inode_item
.st_blocks
-= len
;
2402 fcb
->inode_item_changed
= TRUE
;
2404 c
= get_chunk_from_address(Vcb
, ed2
->address
);
2407 ERR("get_chunk_from_address(%llx) failed\n", ed2
->address
);
2409 Status
= update_changed_extent_ref(Vcb
, c
, ed2
->address
, ed2
->size
, fcb
->subvol
->id
, fcb
->inode
, ext
->offset
- ed2
->offset
, -1,
2410 fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
, FALSE
, Irp
);
2411 if (!NT_SUCCESS(Status
)) {
2412 ERR("update_changed_extent_ref returned %08x\n", Status
);
2418 remove_fcb_extent(fcb
, ext
, rollback
);
2419 } else if (start_data
<= ext
->offset
&& end_data
< ext
->offset
+ len
) { // remove beginning
2423 if (ed2
->size
!= 0) {
2424 fcb
->inode_item
.st_blocks
-= end_data
- ext
->offset
;
2425 fcb
->inode_item_changed
= TRUE
;
2428 newext
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + sizeof(EXTENT_DATA
) - 1 + sizeof(EXTENT_DATA2
), ALLOC_TAG
);
2430 ERR("out of memory\n");
2431 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2435 ned2
= (EXTENT_DATA2
*)newext
->extent_data
.data
;
2437 newext
->extent_data
.generation
= Vcb
->superblock
.generation
;
2438 newext
->extent_data
.decoded_size
= ed
->decoded_size
;
2439 newext
->extent_data
.compression
= ed
->compression
;
2440 newext
->extent_data
.encryption
= ed
->encryption
;
2441 newext
->extent_data
.encoding
= ed
->encoding
;
2442 newext
->extent_data
.type
= ed
->type
;
2443 ned2
->address
= ed2
->address
;
2444 ned2
->size
= ed2
->size
;
2445 ned2
->offset
= ed2
->offset
+ (end_data
- ext
->offset
);
2446 ned2
->num_bytes
= ed2
->num_bytes
- (end_data
- ext
->offset
);
2448 newext
->offset
= end_data
;
2449 newext
->datalen
= sizeof(EXTENT_DATA
) - 1 + sizeof(EXTENT_DATA2
);
2450 newext
->unique
= ext
->unique
;
2451 newext
->ignore
= FALSE
;
2452 newext
->inserted
= TRUE
;
2455 if (ed
->compression
== BTRFS_COMPRESSION_NONE
) {
2456 newext
->csum
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(ned2
->num_bytes
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
), ALLOC_TAG
);
2457 if (!newext
->csum
) {
2458 ERR("out of memory\n");
2459 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2464 RtlCopyMemory(newext
->csum
, &ext
->csum
[(end_data
- ext
->offset
) / Vcb
->superblock
.sector_size
],
2465 (ULONG
)(ned2
->num_bytes
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
));
2467 newext
->csum
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(ed2
->size
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
), ALLOC_TAG
);
2468 if (!newext
->csum
) {
2469 ERR("out of memory\n");
2470 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2475 RtlCopyMemory(newext
->csum
, ext
->csum
, (ULONG
)(ed2
->size
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
));
2478 newext
->csum
= NULL
;
2480 add_extent(fcb
, &ext
->list_entry
, newext
);
2482 remove_fcb_extent(fcb
, ext
, rollback
);
2483 } else if (start_data
> ext
->offset
&& end_data
>= ext
->offset
+ len
) { // remove end
2487 if (ed2
->size
!= 0) {
2488 fcb
->inode_item
.st_blocks
-= ext
->offset
+ len
- start_data
;
2489 fcb
->inode_item_changed
= TRUE
;
2492 newext
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + sizeof(EXTENT_DATA
) - 1 + sizeof(EXTENT_DATA2
), ALLOC_TAG
);
2494 ERR("out of memory\n");
2495 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2499 ned2
= (EXTENT_DATA2
*)newext
->extent_data
.data
;
2501 newext
->extent_data
.generation
= Vcb
->superblock
.generation
;
2502 newext
->extent_data
.decoded_size
= ed
->decoded_size
;
2503 newext
->extent_data
.compression
= ed
->compression
;
2504 newext
->extent_data
.encryption
= ed
->encryption
;
2505 newext
->extent_data
.encoding
= ed
->encoding
;
2506 newext
->extent_data
.type
= ed
->type
;
2507 ned2
->address
= ed2
->address
;
2508 ned2
->size
= ed2
->size
;
2509 ned2
->offset
= ed2
->offset
;
2510 ned2
->num_bytes
= start_data
- ext
->offset
;
2512 newext
->offset
= ext
->offset
;
2513 newext
->datalen
= sizeof(EXTENT_DATA
) - 1 + sizeof(EXTENT_DATA2
);
2514 newext
->unique
= ext
->unique
;
2515 newext
->ignore
= FALSE
;
2516 newext
->inserted
= TRUE
;
2519 if (ed
->compression
== BTRFS_COMPRESSION_NONE
) {
2520 newext
->csum
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(ned2
->num_bytes
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
), ALLOC_TAG
);
2521 if (!newext
->csum
) {
2522 ERR("out of memory\n");
2523 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2528 RtlCopyMemory(newext
->csum
, ext
->csum
, (ULONG
)(ned2
->num_bytes
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
));
2530 newext
->csum
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(ed2
->size
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
), ALLOC_TAG
);
2531 if (!newext
->csum
) {
2532 ERR("out of memory\n");
2533 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2538 RtlCopyMemory(newext
->csum
, ext
->csum
, (ULONG
)(ed2
->size
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
));
2541 newext
->csum
= NULL
;
2543 InsertHeadList(&ext
->list_entry
, &newext
->list_entry
);
2545 remove_fcb_extent(fcb
, ext
, rollback
);
2546 } else if (start_data
> ext
->offset
&& end_data
< ext
->offset
+ len
) { // remove middle
2547 EXTENT_DATA2
*neda2
, *nedb2
;
2548 extent
*newext1
, *newext2
;
2550 if (ed2
->size
!= 0) {
2553 fcb
->inode_item
.st_blocks
-= end_data
- start_data
;
2554 fcb
->inode_item_changed
= TRUE
;
2556 c
= get_chunk_from_address(Vcb
, ed2
->address
);
2559 ERR("get_chunk_from_address(%llx) failed\n", ed2
->address
);
2561 Status
= update_changed_extent_ref(Vcb
, c
, ed2
->address
, ed2
->size
, fcb
->subvol
->id
, fcb
->inode
, ext
->offset
- ed2
->offset
, 1,
2562 fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
, FALSE
, Irp
);
2563 if (!NT_SUCCESS(Status
)) {
2564 ERR("update_changed_extent_ref returned %08x\n", Status
);
2570 newext1
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + sizeof(EXTENT_DATA
) - 1 + sizeof(EXTENT_DATA2
), ALLOC_TAG
);
2572 ERR("out of memory\n");
2573 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2577 newext2
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + sizeof(EXTENT_DATA
) - 1 + sizeof(EXTENT_DATA2
), ALLOC_TAG
);
2579 ERR("out of memory\n");
2580 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2581 ExFreePool(newext1
);
2585 neda2
= (EXTENT_DATA2
*)newext1
->extent_data
.data
;
2587 newext1
->extent_data
.generation
= Vcb
->superblock
.generation
;
2588 newext1
->extent_data
.decoded_size
= ed
->decoded_size
;
2589 newext1
->extent_data
.compression
= ed
->compression
;
2590 newext1
->extent_data
.encryption
= ed
->encryption
;
2591 newext1
->extent_data
.encoding
= ed
->encoding
;
2592 newext1
->extent_data
.type
= ed
->type
;
2593 neda2
->address
= ed2
->address
;
2594 neda2
->size
= ed2
->size
;
2595 neda2
->offset
= ed2
->offset
;
2596 neda2
->num_bytes
= start_data
- ext
->offset
;
2598 nedb2
= (EXTENT_DATA2
*)newext2
->extent_data
.data
;
2600 newext2
->extent_data
.generation
= Vcb
->superblock
.generation
;
2601 newext2
->extent_data
.decoded_size
= ed
->decoded_size
;
2602 newext2
->extent_data
.compression
= ed
->compression
;
2603 newext2
->extent_data
.encryption
= ed
->encryption
;
2604 newext2
->extent_data
.encoding
= ed
->encoding
;
2605 newext2
->extent_data
.type
= ed
->type
;
2606 nedb2
->address
= ed2
->address
;
2607 nedb2
->size
= ed2
->size
;
2608 nedb2
->offset
= ed2
->offset
+ (end_data
- ext
->offset
);
2609 nedb2
->num_bytes
= ext
->offset
+ len
- end_data
;
2611 newext1
->offset
= ext
->offset
;
2612 newext1
->datalen
= sizeof(EXTENT_DATA
) - 1 + sizeof(EXTENT_DATA2
);
2613 newext1
->unique
= ext
->unique
;
2614 newext1
->ignore
= FALSE
;
2615 newext1
->inserted
= TRUE
;
2617 newext2
->offset
= end_data
;
2618 newext2
->datalen
= sizeof(EXTENT_DATA
) - 1 + sizeof(EXTENT_DATA2
);
2619 newext2
->unique
= ext
->unique
;
2620 newext2
->ignore
= FALSE
;
2621 newext2
->inserted
= TRUE
;
2624 if (ed
->compression
== BTRFS_COMPRESSION_NONE
) {
2625 newext1
->csum
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(neda2
->num_bytes
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
), ALLOC_TAG
);
2626 if (!newext1
->csum
) {
2627 ERR("out of memory\n");
2628 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2629 ExFreePool(newext1
);
2630 ExFreePool(newext2
);
2634 newext2
->csum
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(nedb2
->num_bytes
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
), ALLOC_TAG
);
2635 if (!newext2
->csum
) {
2636 ERR("out of memory\n");
2637 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2638 ExFreePool(newext1
->csum
);
2639 ExFreePool(newext1
);
2640 ExFreePool(newext2
);
2644 RtlCopyMemory(newext1
->csum
, ext
->csum
, (ULONG
)(neda2
->num_bytes
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
));
2645 RtlCopyMemory(newext2
->csum
, &ext
->csum
[(end_data
- ext
->offset
) / Vcb
->superblock
.sector_size
],
2646 (ULONG
)(nedb2
->num_bytes
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
));
2648 newext1
->csum
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(ed2
->size
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
), ALLOC_TAG
);
2649 if (!newext1
->csum
) {
2650 ERR("out of memory\n");
2651 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2652 ExFreePool(newext1
);
2653 ExFreePool(newext2
);
2657 newext2
->csum
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(ed2
->size
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
), ALLOC_TAG
);
2658 if (!newext2
->csum
) {
2659 ERR("out of memory\n");
2660 Status
= STATUS_INSUFFICIENT_RESOURCES
;
2661 ExFreePool(newext1
->csum
);
2662 ExFreePool(newext1
);
2663 ExFreePool(newext2
);
2667 RtlCopyMemory(newext1
->csum
, ext
->csum
, (ULONG
)(ed2
->size
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
));
2668 RtlCopyMemory(newext2
->csum
, ext
->csum
, (ULONG
)(ed2
->size
* sizeof(UINT32
) / Vcb
->superblock
.sector_size
));
2671 newext1
->csum
= NULL
;
2672 newext2
->csum
= NULL
;
2675 InsertHeadList(&ext
->list_entry
, &newext1
->list_entry
);
2676 add_extent(fcb
, &newext1
->list_entry
, newext2
);
2678 remove_fcb_extent(fcb
, ext
, rollback
);
2687 Status
= STATUS_SUCCESS
;
2690 fcb
->extents_changed
= TRUE
;
2691 mark_fcb_dirty(fcb
);
2696 void add_insert_extent_rollback(LIST_ENTRY
* rollback
, fcb
* fcb
, extent
* ext
) {
2697 rollback_extent
* re
;
2699 re
= ExAllocatePoolWithTag(NonPagedPool
, sizeof(rollback_extent
), ALLOC_TAG
);
2701 ERR("out of memory\n");
2708 add_rollback(rollback
, ROLLBACK_INSERT_EXTENT
, re
);
2712 #pragma warning(push)
2713 #pragma warning(suppress: 28194)
2715 NTSTATUS
add_extent_to_fcb(_In_ fcb
* fcb
, _In_ UINT64 offset
, _In_reads_bytes_(edsize
) EXTENT_DATA
* ed
, _In_ UINT16 edsize
,
2716 _In_ BOOL unique
, _In_opt_
_When_(return >= 0, __drv_aliasesMem
) UINT32
* csum
, _In_ LIST_ENTRY
* rollback
) {
2720 ext
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + edsize
, ALLOC_TAG
);
2722 ERR("out of memory\n");
2723 return STATUS_INSUFFICIENT_RESOURCES
;
2726 ext
->offset
= offset
;
2727 ext
->datalen
= edsize
;
2728 ext
->unique
= unique
;
2729 ext
->ignore
= FALSE
;
2730 ext
->inserted
= TRUE
;
2733 RtlCopyMemory(&ext
->extent_data
, ed
, edsize
);
2735 le
= fcb
->extents
.Flink
;
2736 while (le
!= &fcb
->extents
) {
2737 extent
* oldext
= CONTAINING_RECORD(le
, extent
, list_entry
);
2739 if (oldext
->offset
>= offset
) {
2740 InsertHeadList(le
->Blink
, &ext
->list_entry
);
2747 InsertTailList(&fcb
->extents
, &ext
->list_entry
);
2750 add_insert_extent_rollback(rollback
, fcb
, ext
);
2752 return STATUS_SUCCESS
;
2755 #pragma warning(pop)
2758 static void remove_fcb_extent(fcb
* fcb
, extent
* ext
, LIST_ENTRY
* rollback
) {
2760 rollback_extent
* re
;
2764 re
= ExAllocatePoolWithTag(NonPagedPool
, sizeof(rollback_extent
), ALLOC_TAG
);
2766 ERR("out of memory\n");
2773 add_rollback(rollback
, ROLLBACK_DELETE_EXTENT
, re
);
2777 NTSTATUS
calc_csum(_In_ device_extension
* Vcb
, _In_reads_bytes_(sectors
*Vcb
->superblock
.sector_size
) UINT8
* data
,
2778 _In_ UINT32 sectors
, _Out_writes_bytes_(sectors
*sizeof(UINT32
)) UINT32
* csum
) {
2782 // From experimenting, it seems that 40 sectors is roughly the crossover
2783 // point where offloading the crc32 calculation becomes worth it.
2785 if (sectors
< 40 || KeQueryActiveProcessorCount(NULL
) < 2) {
2788 for (j
= 0; j
< sectors
; j
++) {
2789 csum
[j
] = ~calc_crc32c(0xffffffff, data
+ (j
* Vcb
->superblock
.sector_size
), Vcb
->superblock
.sector_size
);
2792 return STATUS_SUCCESS
;
2795 Status
= add_calc_job(Vcb
, data
, sectors
, csum
, &cj
);
2796 if (!NT_SUCCESS(Status
)) {
2797 ERR("add_calc_job returned %08x\n", Status
);
2801 KeWaitForSingleObject(&cj
->event
, Executive
, KernelMode
, FALSE
, NULL
);
2804 return STATUS_SUCCESS
;
2807 _Requires_lock_held_(c
->lock
)
2808 _When_(return != 0, _Releases_lock_(c
->lock
))
2809 BOOL
insert_extent_chunk(_In_ device_extension
* Vcb
, _In_ fcb
* fcb
, _In_ chunk
* c
, _In_ UINT64 start_data
, _In_ UINT64 length
, _In_ BOOL prealloc
, _In_opt_
void* data
,
2810 _In_opt_ PIRP Irp
, _In_ LIST_ENTRY
* rollback
, _In_ UINT8 compression
, _In_ UINT64 decoded_size
, _In_ BOOL file_write
, _In_ UINT64 irp_offset
) {
2815 UINT16 edsize
= (UINT16
)(offsetof(EXTENT_DATA
, data
[0]) + sizeof(EXTENT_DATA2
));
2816 UINT32
* csum
= NULL
;
2818 TRACE("(%p, (%llx, %llx), %llx, %llx, %llx, %u, %p, %p)\n", Vcb
, fcb
->subvol
->id
, fcb
->inode
, c
->offset
, start_data
, length
, prealloc
, data
, rollback
);
2820 if (!find_data_address_in_chunk(Vcb
, c
, length
, &address
))
2823 // add extent data to inode
2824 ed
= ExAllocatePoolWithTag(PagedPool
, edsize
, ALLOC_TAG
);
2826 ERR("out of memory\n");
2830 ed
->generation
= Vcb
->superblock
.generation
;
2831 ed
->decoded_size
= decoded_size
;
2832 ed
->compression
= compression
;
2833 ed
->encryption
= BTRFS_ENCRYPTION_NONE
;
2834 ed
->encoding
= BTRFS_ENCODING_NONE
;
2835 ed
->type
= prealloc
? EXTENT_TYPE_PREALLOC
: EXTENT_TYPE_REGULAR
;
2837 ed2
= (EXTENT_DATA2
*)ed
->data
;
2838 ed2
->address
= address
;
2841 ed2
->num_bytes
= decoded_size
;
2843 if (!prealloc
&& data
&& !(fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
)) {
2844 ULONG sl
= (ULONG
)(length
/ Vcb
->superblock
.sector_size
);
2846 csum
= ExAllocatePoolWithTag(PagedPool
, sl
* sizeof(UINT32
), ALLOC_TAG
);
2848 ERR("out of memory\n");
2853 Status
= calc_csum(Vcb
, data
, sl
, csum
);
2854 if (!NT_SUCCESS(Status
)) {
2855 ERR("calc_csum returned %08x\n", Status
);
2862 Status
= add_extent_to_fcb(fcb
, start_data
, ed
, edsize
, TRUE
, csum
, rollback
);
2863 if (!NT_SUCCESS(Status
)) {
2864 ERR("add_extent_to_fcb returned %08x\n", Status
);
2865 if (csum
) ExFreePool(csum
);
2873 space_list_subtract(c
, FALSE
, address
, length
, rollback
);
2875 fcb
->inode_item
.st_blocks
+= decoded_size
;
2877 fcb
->extents_changed
= TRUE
;
2878 fcb
->inode_item_changed
= TRUE
;
2879 mark_fcb_dirty(fcb
);
2881 ExAcquireResourceExclusiveLite(&c
->changed_extents_lock
, TRUE
);
2883 add_changed_extent_ref(c
, address
, length
, fcb
->subvol
->id
, fcb
->inode
, start_data
, 1, fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
);
2885 ExReleaseResourceLite(&c
->changed_extents_lock
);
2887 ExReleaseResourceLite(&c
->lock
);
2890 Status
= write_data_complete(Vcb
, address
, data
, (UINT32
)length
, Irp
, NULL
, file_write
, irp_offset
,
2891 fcb
->Header
.Flags2
& FSRTL_FLAG2_IS_PAGING_FILE
? HighPagePriority
: NormalPagePriority
);
2892 if (!NT_SUCCESS(Status
))
2893 ERR("write_data_complete returned %08x\n", Status
);
2899 static BOOL
try_extend_data(device_extension
* Vcb
, fcb
* fcb
, UINT64 start_data
, UINT64 length
, void* data
,
2900 PIRP Irp
, UINT64
* written
, BOOL file_write
, UINT64 irp_offset
, LIST_ENTRY
* rollback
) {
2901 BOOL success
= FALSE
;
2908 le
= fcb
->extents
.Flink
;
2910 while (le
!= &fcb
->extents
) {
2911 extent
* nextext
= CONTAINING_RECORD(le
, extent
, list_entry
);
2913 if (!nextext
->ignore
) {
2914 if (nextext
->offset
== start_data
) {
2917 } else if (nextext
->offset
> start_data
)
2929 ed
= &ext
->extent_data
;
2931 if (ed
->type
!= EXTENT_TYPE_REGULAR
&& ed
->type
!= EXTENT_TYPE_PREALLOC
) {
2932 TRACE("not extending extent which is not regular or prealloc\n");
2936 ed2
= (EXTENT_DATA2
*)ed
->data
;
2938 if (ext
->offset
+ ed2
->num_bytes
!= start_data
) {
2939 TRACE("last EXTENT_DATA does not run up to start_data (%llx + %llx != %llx)\n", ext
->offset
, ed2
->num_bytes
, start_data
);
2943 c
= get_chunk_from_address(Vcb
, ed2
->address
);
2945 if (c
->reloc
|| c
->readonly
|| c
->chunk_item
->type
!= Vcb
->data_flags
)
2948 ExAcquireResourceExclusiveLite(&c
->lock
, TRUE
);
2950 if (length
> c
->chunk_item
->size
- c
->used
) {
2951 ExReleaseResourceLite(&c
->lock
);
2955 if (!c
->cache_loaded
) {
2956 NTSTATUS Status
= load_cache_chunk(Vcb
, c
, NULL
);
2958 if (!NT_SUCCESS(Status
)) {
2959 ERR("load_cache_chunk returned %08x\n", Status
);
2960 ExReleaseResourceLite(&c
->lock
);
2965 le
= c
->space
.Flink
;
2966 while (le
!= &c
->space
) {
2967 space
* s
= CONTAINING_RECORD(le
, space
, list_entry
);
2969 if (s
->address
== ed2
->address
+ ed2
->size
) {
2970 UINT64 newlen
= min(min(s
->size
, length
), MAX_EXTENT_SIZE
);
2972 success
= insert_extent_chunk(Vcb
, fcb
, c
, start_data
, newlen
, FALSE
, data
, Irp
, rollback
, BTRFS_COMPRESSION_NONE
, newlen
, file_write
, irp_offset
);
2977 ExReleaseResourceLite(&c
->lock
);
2980 } else if (s
->address
> ed2
->address
+ ed2
->size
)
2986 ExReleaseResourceLite(&c
->lock
);
2991 static NTSTATUS
insert_chunk_fragmented(fcb
* fcb
, UINT64 start
, UINT64 length
, UINT8
* data
, BOOL prealloc
, LIST_ENTRY
* rollback
) {
2993 UINT64 flags
= fcb
->Vcb
->data_flags
;
2994 BOOL page_file
= fcb
->Header
.Flags2
& FSRTL_FLAG2_IS_PAGING_FILE
;
2998 ExAcquireResourceSharedLite(&fcb
->Vcb
->chunk_lock
, TRUE
);
3000 // first create as many chunks as we can
3002 Status
= alloc_chunk(fcb
->Vcb
, flags
, &c
, FALSE
);
3003 } while (NT_SUCCESS(Status
));
3005 if (Status
!= STATUS_DISK_FULL
) {
3006 ERR("alloc_chunk returned %08x\n", Status
);
3007 ExReleaseResourceLite(&fcb
->Vcb
->chunk_lock
);
3011 le
= fcb
->Vcb
->chunks
.Flink
;
3012 while (le
!= &fcb
->Vcb
->chunks
) {
3013 c
= CONTAINING_RECORD(le
, chunk
, list_entry
);
3015 if (!c
->readonly
&& !c
->reloc
) {
3016 ExAcquireResourceExclusiveLite(&c
->lock
, TRUE
);
3018 if (c
->chunk_item
->type
== flags
) {
3019 while (!IsListEmpty(&c
->space_size
) && length
> 0) {
3020 space
* s
= CONTAINING_RECORD(c
->space_size
.Flink
, space
, list_entry_size
);
3021 UINT64 extlen
= min(length
, s
->size
);
3023 if (insert_extent_chunk(fcb
->Vcb
, fcb
, c
, start
, extlen
, prealloc
&& !page_file
, data
, NULL
, rollback
, BTRFS_COMPRESSION_NONE
, extlen
, FALSE
, 0)) {
3026 if (data
) data
+= extlen
;
3028 ExAcquireResourceExclusiveLite(&c
->lock
, TRUE
);
3033 ExReleaseResourceLite(&c
->lock
);
3042 ExReleaseResourceLite(&fcb
->Vcb
->chunk_lock
);
3044 return length
== 0 ? STATUS_SUCCESS
: STATUS_DISK_FULL
;
3047 static NTSTATUS
insert_prealloc_extent(fcb
* fcb
, UINT64 start
, UINT64 length
, LIST_ENTRY
* rollback
) {
3052 BOOL page_file
= fcb
->Header
.Flags2
& FSRTL_FLAG2_IS_PAGING_FILE
;
3054 flags
= fcb
->Vcb
->data_flags
;
3057 UINT64 extlen
= min(MAX_EXTENT_SIZE
, length
);
3059 ExAcquireResourceSharedLite(&fcb
->Vcb
->chunk_lock
, TRUE
);
3061 le
= fcb
->Vcb
->chunks
.Flink
;
3062 while (le
!= &fcb
->Vcb
->chunks
) {
3063 c
= CONTAINING_RECORD(le
, chunk
, list_entry
);
3065 if (!c
->readonly
&& !c
->reloc
) {
3066 ExAcquireResourceExclusiveLite(&c
->lock
, TRUE
);
3068 if (c
->chunk_item
->type
== flags
&& (c
->chunk_item
->size
- c
->used
) >= extlen
) {
3069 if (insert_extent_chunk(fcb
->Vcb
, fcb
, c
, start
, extlen
, !page_file
, NULL
, NULL
, rollback
, BTRFS_COMPRESSION_NONE
, extlen
, FALSE
, 0)) {
3070 ExReleaseResourceLite(&fcb
->Vcb
->chunk_lock
);
3075 ExReleaseResourceLite(&c
->lock
);
3081 ExReleaseResourceLite(&fcb
->Vcb
->chunk_lock
);
3083 ExAcquireResourceExclusiveLite(&fcb
->Vcb
->chunk_lock
, TRUE
);
3085 Status
= alloc_chunk(fcb
->Vcb
, flags
, &c
, FALSE
);
3087 ExReleaseResourceLite(&fcb
->Vcb
->chunk_lock
);
3089 if (!NT_SUCCESS(Status
)) {
3090 ERR("alloc_chunk returned %08x\n", Status
);
3094 ExAcquireResourceExclusiveLite(&c
->lock
, TRUE
);
3096 if (c
->chunk_item
->type
== flags
&& (c
->chunk_item
->size
- c
->used
) >= extlen
) {
3097 if (insert_extent_chunk(fcb
->Vcb
, fcb
, c
, start
, extlen
, !page_file
, NULL
, NULL
, rollback
, BTRFS_COMPRESSION_NONE
, extlen
, FALSE
, 0))
3101 ExReleaseResourceLite(&c
->lock
);
3103 Status
= insert_chunk_fragmented(fcb
, start
, length
, NULL
, TRUE
, rollback
);
3104 if (!NT_SUCCESS(Status
))
3105 ERR("insert_chunk_fragmented returned %08x\n", Status
);
3112 } while (length
> 0);
3114 Status
= STATUS_SUCCESS
;
3120 static NTSTATUS
insert_extent(device_extension
* Vcb
, fcb
* fcb
, UINT64 start_data
, UINT64 length
, void* data
,
3121 PIRP Irp
, BOOL file_write
, UINT64 irp_offset
, LIST_ENTRY
* rollback
) {
3125 UINT64 flags
, orig_length
= length
, written
= 0;
3127 TRACE("(%p, (%llx, %llx), %llx, %llx, %p)\n", Vcb
, fcb
->subvol
->id
, fcb
->inode
, start_data
, length
, data
);
3129 if (start_data
> 0) {
3130 try_extend_data(Vcb
, fcb
, start_data
, length
, data
, Irp
, &written
, file_write
, irp_offset
, rollback
);
3132 if (written
== length
)
3133 return STATUS_SUCCESS
;
3134 else if (written
> 0) {
3135 start_data
+= written
;
3136 irp_offset
+= written
;
3138 data
= &((UINT8
*)data
)[written
];
3142 flags
= Vcb
->data_flags
;
3144 while (written
< orig_length
) {
3145 UINT64 newlen
= min(length
, MAX_EXTENT_SIZE
);
3148 // Rather than necessarily writing the whole extent at once, we deal with it in blocks of 128 MB.
3149 // First, see if we can write the extent part to an existing chunk.
3151 ExAcquireResourceSharedLite(&Vcb
->chunk_lock
, TRUE
);
3153 le
= Vcb
->chunks
.Flink
;
3154 while (le
!= &Vcb
->chunks
) {
3155 c
= CONTAINING_RECORD(le
, chunk
, list_entry
);
3157 if (!c
->readonly
&& !c
->reloc
) {
3158 ExAcquireResourceExclusiveLite(&c
->lock
, TRUE
);
3160 if (c
->chunk_item
->type
== flags
&& (c
->chunk_item
->size
- c
->used
) >= newlen
&&
3161 insert_extent_chunk(Vcb
, fcb
, c
, start_data
, newlen
, FALSE
, data
, Irp
, rollback
, BTRFS_COMPRESSION_NONE
, newlen
, file_write
, irp_offset
)) {
3164 if (written
== orig_length
) {
3165 ExReleaseResourceLite(&Vcb
->chunk_lock
);
3166 return STATUS_SUCCESS
;
3169 start_data
+= newlen
;
3170 irp_offset
+= newlen
;
3172 data
= &((UINT8
*)data
)[newlen
];
3176 ExReleaseResourceLite(&c
->lock
);
3182 ExReleaseResourceLite(&Vcb
->chunk_lock
);
3186 // Otherwise, see if we can put it in a new chunk.
3188 ExAcquireResourceExclusiveLite(&Vcb
->chunk_lock
, TRUE
);
3190 Status
= alloc_chunk(Vcb
, flags
, &c
, FALSE
);
3192 ExReleaseResourceLite(&Vcb
->chunk_lock
);
3194 if (!NT_SUCCESS(Status
)) {
3195 ERR("alloc_chunk returned %08x\n", Status
);
3200 ExAcquireResourceExclusiveLite(&c
->lock
, TRUE
);
3202 if (c
->chunk_item
->type
== flags
&& (c
->chunk_item
->size
- c
->used
) >= newlen
&&
3203 insert_extent_chunk(Vcb
, fcb
, c
, start_data
, newlen
, FALSE
, data
, Irp
, rollback
, BTRFS_COMPRESSION_NONE
, newlen
, file_write
, irp_offset
)) {
3206 if (written
== orig_length
)
3207 return STATUS_SUCCESS
;
3210 start_data
+= newlen
;
3211 irp_offset
+= newlen
;
3213 data
= &((UINT8
*)data
)[newlen
];
3216 ExReleaseResourceLite(&c
->lock
);
3220 Status
= insert_chunk_fragmented(fcb
, start_data
, length
, data
, FALSE
, rollback
);
3221 if (!NT_SUCCESS(Status
))
3222 ERR("insert_chunk_fragmented returned %08x\n", Status
);
3228 return STATUS_DISK_FULL
;
3231 NTSTATUS
truncate_file(fcb
* fcb
, UINT64 end
, PIRP Irp
, LIST_ENTRY
* rollback
) {
3234 // FIXME - convert into inline extent if short enough
3236 if (end
> 0 && fcb_is_inline(fcb
)) {
3238 BOOL make_inline
= end
<= fcb
->Vcb
->options
.max_inline
;
3240 buf
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(make_inline
? (offsetof(EXTENT_DATA
, data
[0]) + end
) : sector_align(end
, fcb
->Vcb
->superblock
.sector_size
)), ALLOC_TAG
);
3242 ERR("out of memory\n");
3243 return STATUS_INSUFFICIENT_RESOURCES
;
3246 Status
= read_file(fcb
, make_inline
? (buf
+ offsetof(EXTENT_DATA
, data
[0])) : buf
, 0, end
, NULL
, Irp
);
3247 if (!NT_SUCCESS(Status
)) {
3248 ERR("read_file returned %08x\n", Status
);
3253 Status
= excise_extents(fcb
->Vcb
, fcb
, 0, fcb
->inode_item
.st_size
, Irp
, rollback
);
3254 if (!NT_SUCCESS(Status
)) {
3255 ERR("excise_extents returned %08x\n", Status
);
3261 RtlZeroMemory(buf
+ end
, (ULONG
)(sector_align(end
, fcb
->Vcb
->superblock
.sector_size
) - end
));
3263 Status
= do_write_file(fcb
, 0, sector_align(end
, fcb
->Vcb
->superblock
.sector_size
), buf
, Irp
, FALSE
, 0, rollback
);
3264 if (!NT_SUCCESS(Status
)) {
3265 ERR("do_write_file returned %08x\n", Status
);
3270 EXTENT_DATA
* ed
= (EXTENT_DATA
*)buf
;
3272 ed
->generation
= fcb
->Vcb
->superblock
.generation
;
3273 ed
->decoded_size
= end
;
3274 ed
->compression
= BTRFS_COMPRESSION_NONE
;
3275 ed
->encryption
= BTRFS_ENCRYPTION_NONE
;
3276 ed
->encoding
= BTRFS_ENCODING_NONE
;
3277 ed
->type
= EXTENT_TYPE_INLINE
;
3279 Status
= add_extent_to_fcb(fcb
, 0, ed
, (UINT16
)(offsetof(EXTENT_DATA
, data
[0]) + end
), FALSE
, NULL
, rollback
);
3280 if (!NT_SUCCESS(Status
)) {
3281 ERR("add_extent_to_fcb returned %08x\n", Status
);
3286 fcb
->inode_item
.st_blocks
+= end
;
3290 return STATUS_SUCCESS
;
3293 Status
= excise_extents(fcb
->Vcb
, fcb
, sector_align(end
, fcb
->Vcb
->superblock
.sector_size
),
3294 sector_align(fcb
->inode_item
.st_size
, fcb
->Vcb
->superblock
.sector_size
), Irp
, rollback
);
3295 if (!NT_SUCCESS(Status
)) {
3296 ERR("excise_extents returned %08x\n", Status
);
3300 fcb
->inode_item
.st_size
= end
;
3301 fcb
->inode_item_changed
= TRUE
;
3302 TRACE("setting st_size to %llx\n", end
);
3304 fcb
->Header
.AllocationSize
.QuadPart
= sector_align(fcb
->inode_item
.st_size
, fcb
->Vcb
->superblock
.sector_size
);
3305 fcb
->Header
.FileSize
.QuadPart
= fcb
->inode_item
.st_size
;
3306 fcb
->Header
.ValidDataLength
.QuadPart
= fcb
->inode_item
.st_size
;
3307 // FIXME - inform cache manager of this
3309 TRACE("fcb %p FileSize = %llx\n", fcb
, fcb
->Header
.FileSize
.QuadPart
);
3311 return STATUS_SUCCESS
;
3314 NTSTATUS
extend_file(fcb
* fcb
, file_ref
* fileref
, UINT64 end
, BOOL prealloc
, PIRP Irp
, LIST_ENTRY
* rollback
) {
3315 UINT64 oldalloc
, newalloc
;
3319 TRACE("(%p, %p, %x, %u)\n", fcb
, fileref
, end
, prealloc
);
3323 return STATUS_DISK_FULL
;
3325 return stream_set_end_of_file_information(fcb
->Vcb
, (UINT16
)end
, fcb
, fileref
, FALSE
);
3330 le
= fcb
->extents
.Blink
;
3331 while (le
!= &fcb
->extents
) {
3332 extent
* ext2
= CONTAINING_RECORD(le
, extent
, list_entry
);
3334 if (!ext2
->ignore
) {
3344 EXTENT_DATA
* ed
= &ext
->extent_data
;
3345 EXTENT_DATA2
* ed2
= (EXTENT_DATA2
*)ed
->data
;
3347 oldalloc
= ext
->offset
+ (ed
->type
== EXTENT_TYPE_INLINE
? ed
->decoded_size
: ed2
->num_bytes
);
3348 cur_inline
= ed
->type
== EXTENT_TYPE_INLINE
;
3350 if (cur_inline
&& end
> fcb
->Vcb
->options
.max_inline
) {
3351 UINT64 origlength
, length
;
3354 TRACE("giving inline file proper extents\n");
3356 origlength
= ed
->decoded_size
;
3360 length
= sector_align(origlength
, fcb
->Vcb
->superblock
.sector_size
);
3362 data
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)length
, ALLOC_TAG
);
3364 ERR("could not allocate %llx bytes for data\n", length
);
3365 return STATUS_INSUFFICIENT_RESOURCES
;
3368 Status
= read_file(fcb
, data
, 0, origlength
, NULL
, Irp
);
3369 if (!NT_SUCCESS(Status
)) {
3370 ERR("read_file returned %08x\n", Status
);
3375 RtlZeroMemory(data
+ origlength
, (ULONG
)(length
- origlength
));
3377 Status
= excise_extents(fcb
->Vcb
, fcb
, 0, fcb
->inode_item
.st_size
, Irp
, rollback
);
3378 if (!NT_SUCCESS(Status
)) {
3379 ERR("excise_extents returned %08x\n", Status
);
3384 Status
= do_write_file(fcb
, 0, length
, data
, Irp
, FALSE
, 0, rollback
);
3385 if (!NT_SUCCESS(Status
)) {
3386 ERR("do_write_file returned %08x\n", Status
);
3391 oldalloc
= ext
->offset
+ length
;
3399 if (end
> oldalloc
) {
3400 edsize
= (UINT16
)(offsetof(EXTENT_DATA
, data
[0]) + end
- ext
->offset
);
3401 ed
= ExAllocatePoolWithTag(PagedPool
, edsize
, ALLOC_TAG
);
3404 ERR("out of memory\n");
3405 return STATUS_INSUFFICIENT_RESOURCES
;
3408 ed
->generation
= fcb
->Vcb
->superblock
.generation
;
3409 ed
->decoded_size
= end
- ext
->offset
;
3410 ed
->compression
= BTRFS_COMPRESSION_NONE
;
3411 ed
->encryption
= BTRFS_ENCRYPTION_NONE
;
3412 ed
->encoding
= BTRFS_ENCODING_NONE
;
3413 ed
->type
= EXTENT_TYPE_INLINE
;
3415 Status
= read_file(fcb
, ed
->data
, ext
->offset
, oldalloc
, NULL
, Irp
);
3416 if (!NT_SUCCESS(Status
)) {
3417 ERR("read_file returned %08x\n", Status
);
3422 RtlZeroMemory(ed
->data
+ oldalloc
- ext
->offset
, (ULONG
)(end
- oldalloc
));
3424 remove_fcb_extent(fcb
, ext
, rollback
);
3426 Status
= add_extent_to_fcb(fcb
, ext
->offset
, ed
, edsize
, ext
->unique
, NULL
, rollback
);
3427 if (!NT_SUCCESS(Status
)) {
3428 ERR("add_extent_to_fcb returned %08x\n", Status
);
3435 fcb
->extents_changed
= TRUE
;
3436 mark_fcb_dirty(fcb
);
3439 TRACE("extending inline file (oldalloc = %llx, end = %llx)\n", oldalloc
, end
);
3441 fcb
->inode_item
.st_size
= end
;
3442 TRACE("setting st_size to %llx\n", end
);
3444 fcb
->inode_item
.st_blocks
= end
;
3446 fcb
->Header
.AllocationSize
.QuadPart
= fcb
->Header
.FileSize
.QuadPart
= fcb
->Header
.ValidDataLength
.QuadPart
= end
;
3448 newalloc
= sector_align(end
, fcb
->Vcb
->superblock
.sector_size
);
3450 if (newalloc
> oldalloc
) {
3452 // FIXME - try and extend previous extent first
3454 Status
= insert_prealloc_extent(fcb
, oldalloc
, newalloc
- oldalloc
, rollback
);
3456 if (!NT_SUCCESS(Status
)) {
3457 ERR("insert_prealloc_extent returned %08x\n", Status
);
3462 fcb
->extents_changed
= TRUE
;
3465 fcb
->inode_item
.st_size
= end
;
3466 fcb
->inode_item_changed
= TRUE
;
3467 mark_fcb_dirty(fcb
);
3469 TRACE("setting st_size to %llx\n", end
);
3471 TRACE("newalloc = %llx\n", newalloc
);
3473 fcb
->Header
.AllocationSize
.QuadPart
= newalloc
;
3474 fcb
->Header
.FileSize
.QuadPart
= fcb
->Header
.ValidDataLength
.QuadPart
= end
;
3477 if (end
> fcb
->Vcb
->options
.max_inline
) {
3478 newalloc
= sector_align(end
, fcb
->Vcb
->superblock
.sector_size
);
3481 Status
= insert_prealloc_extent(fcb
, 0, newalloc
, rollback
);
3483 if (!NT_SUCCESS(Status
)) {
3484 ERR("insert_prealloc_extent returned %08x\n", Status
);
3489 fcb
->extents_changed
= TRUE
;
3490 fcb
->inode_item_changed
= TRUE
;
3491 mark_fcb_dirty(fcb
);
3493 fcb
->inode_item
.st_size
= end
;
3494 TRACE("setting st_size to %llx\n", end
);
3496 TRACE("newalloc = %llx\n", newalloc
);
3498 fcb
->Header
.AllocationSize
.QuadPart
= newalloc
;
3499 fcb
->Header
.FileSize
.QuadPart
= fcb
->Header
.ValidDataLength
.QuadPart
= end
;
3504 edsize
= (UINT16
)(offsetof(EXTENT_DATA
, data
[0]) + end
);
3505 ed
= ExAllocatePoolWithTag(PagedPool
, edsize
, ALLOC_TAG
);
3508 ERR("out of memory\n");
3509 return STATUS_INSUFFICIENT_RESOURCES
;
3512 ed
->generation
= fcb
->Vcb
->superblock
.generation
;
3513 ed
->decoded_size
= end
;
3514 ed
->compression
= BTRFS_COMPRESSION_NONE
;
3515 ed
->encryption
= BTRFS_ENCRYPTION_NONE
;
3516 ed
->encoding
= BTRFS_ENCODING_NONE
;
3517 ed
->type
= EXTENT_TYPE_INLINE
;
3519 RtlZeroMemory(ed
->data
, (ULONG
)end
);
3521 Status
= add_extent_to_fcb(fcb
, 0, ed
, edsize
, FALSE
, NULL
, rollback
);
3522 if (!NT_SUCCESS(Status
)) {
3523 ERR("add_extent_to_fcb returned %08x\n", Status
);
3530 fcb
->extents_changed
= TRUE
;
3531 fcb
->inode_item_changed
= TRUE
;
3532 mark_fcb_dirty(fcb
);
3534 fcb
->inode_item
.st_size
= end
;
3535 TRACE("setting st_size to %llx\n", end
);
3537 fcb
->inode_item
.st_blocks
= end
;
3539 fcb
->Header
.AllocationSize
.QuadPart
= fcb
->Header
.FileSize
.QuadPart
= fcb
->Header
.ValidDataLength
.QuadPart
= end
;
3544 return STATUS_SUCCESS
;
3547 static NTSTATUS
do_write_file_prealloc(fcb
* fcb
, extent
* ext
, UINT64 start_data
, UINT64 end_data
, void* data
, UINT64
* written
,
3548 PIRP Irp
, BOOL file_write
, UINT64 irp_offset
, ULONG priority
, LIST_ENTRY
* rollback
) {
3549 EXTENT_DATA
* ed
= &ext
->extent_data
;
3550 EXTENT_DATA2
* ed2
= (EXTENT_DATA2
*)ed
->data
;
3554 if (start_data
<= ext
->offset
&& end_data
>= ext
->offset
+ ed2
->num_bytes
) { // replace all
3557 newext
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + ext
->datalen
, ALLOC_TAG
);
3559 ERR("out of memory\n");
3560 return STATUS_INSUFFICIENT_RESOURCES
;
3563 RtlCopyMemory(&newext
->extent_data
, &ext
->extent_data
, ext
->datalen
);
3565 newext
->extent_data
.type
= EXTENT_TYPE_REGULAR
;
3567 Status
= write_data_complete(fcb
->Vcb
, ed2
->address
+ ed2
->offset
, (UINT8
*)data
+ ext
->offset
- start_data
, (UINT32
)ed2
->num_bytes
, Irp
,
3568 NULL
, file_write
, irp_offset
+ ext
->offset
- start_data
, priority
);
3569 if (!NT_SUCCESS(Status
)) {
3570 ERR("write_data_complete returned %08x\n", Status
);
3574 if (!(fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
)) {
3575 ULONG sl
= (ULONG
)(ed2
->num_bytes
/ fcb
->Vcb
->superblock
.sector_size
);
3576 UINT32
* csum
= ExAllocatePoolWithTag(PagedPool
, sl
* sizeof(UINT32
), ALLOC_TAG
);
3579 ERR("out of memory\n");
3581 return STATUS_INSUFFICIENT_RESOURCES
;
3584 Status
= calc_csum(fcb
->Vcb
, (UINT8
*)data
+ ext
->offset
- start_data
, sl
, csum
);
3585 if (!NT_SUCCESS(Status
)) {
3586 ERR("calc_csum returned %08x\n", Status
);
3592 newext
->csum
= csum
;
3594 newext
->csum
= NULL
;
3596 *written
= ed2
->num_bytes
;
3598 newext
->offset
= ext
->offset
;
3599 newext
->datalen
= ext
->datalen
;
3600 newext
->unique
= ext
->unique
;
3601 newext
->ignore
= FALSE
;
3602 newext
->inserted
= TRUE
;
3603 InsertHeadList(&ext
->list_entry
, &newext
->list_entry
);
3605 add_insert_extent_rollback(rollback
, fcb
, newext
);
3607 remove_fcb_extent(fcb
, ext
, rollback
);
3609 c
= get_chunk_from_address(fcb
->Vcb
, ed2
->address
);
3610 } else if (start_data
<= ext
->offset
&& end_data
< ext
->offset
+ ed2
->num_bytes
) { // replace beginning
3612 extent
*newext1
, *newext2
;
3614 newext1
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + ext
->datalen
, ALLOC_TAG
);
3616 ERR("out of memory\n");
3617 return STATUS_INSUFFICIENT_RESOURCES
;
3620 newext2
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + ext
->datalen
, ALLOC_TAG
);
3622 ERR("out of memory\n");
3623 ExFreePool(newext1
);
3624 return STATUS_INSUFFICIENT_RESOURCES
;
3627 RtlCopyMemory(&newext1
->extent_data
, &ext
->extent_data
, ext
->datalen
);
3628 newext1
->extent_data
.type
= EXTENT_TYPE_REGULAR
;
3629 ned2
= (EXTENT_DATA2
*)newext1
->extent_data
.data
;
3630 ned2
->num_bytes
= end_data
- ext
->offset
;
3632 RtlCopyMemory(&newext2
->extent_data
, &ext
->extent_data
, ext
->datalen
);
3633 ned2
= (EXTENT_DATA2
*)newext2
->extent_data
.data
;
3634 ned2
->offset
+= end_data
- ext
->offset
;
3635 ned2
->num_bytes
-= end_data
- ext
->offset
;
3637 Status
= write_data_complete(fcb
->Vcb
, ed2
->address
+ ed2
->offset
, (UINT8
*)data
+ ext
->offset
- start_data
, (UINT32
)(end_data
- ext
->offset
),
3638 Irp
, NULL
, file_write
, irp_offset
+ ext
->offset
- start_data
, priority
);
3639 if (!NT_SUCCESS(Status
)) {
3640 ERR("write_data_complete returned %08x\n", Status
);
3644 if (!(fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
)) {
3645 ULONG sl
= (ULONG
)((end_data
- ext
->offset
) / fcb
->Vcb
->superblock
.sector_size
);
3646 UINT32
* csum
= ExAllocatePoolWithTag(PagedPool
, sl
* sizeof(UINT32
), ALLOC_TAG
);
3649 ERR("out of memory\n");
3650 ExFreePool(newext1
);
3651 ExFreePool(newext2
);
3652 return STATUS_INSUFFICIENT_RESOURCES
;
3655 Status
= calc_csum(fcb
->Vcb
, (UINT8
*)data
+ ext
->offset
- start_data
, sl
, csum
);
3656 if (!NT_SUCCESS(Status
)) {
3657 ERR("calc_csum returned %08x\n", Status
);
3658 ExFreePool(newext1
);
3659 ExFreePool(newext2
);
3664 newext1
->csum
= csum
;
3666 newext1
->csum
= NULL
;
3668 *written
= end_data
- ext
->offset
;
3670 newext1
->offset
= ext
->offset
;
3671 newext1
->datalen
= ext
->datalen
;
3672 newext1
->unique
= ext
->unique
;
3673 newext1
->ignore
= FALSE
;
3674 newext1
->inserted
= TRUE
;
3675 InsertHeadList(&ext
->list_entry
, &newext1
->list_entry
);
3677 add_insert_extent_rollback(rollback
, fcb
, newext1
);
3679 newext2
->offset
= end_data
;
3680 newext2
->datalen
= ext
->datalen
;
3681 newext2
->unique
= ext
->unique
;
3682 newext2
->ignore
= FALSE
;
3683 newext2
->inserted
= TRUE
;
3684 newext2
->csum
= NULL
;
3685 add_extent(fcb
, &newext1
->list_entry
, newext2
);
3687 add_insert_extent_rollback(rollback
, fcb
, newext2
);
3689 c
= get_chunk_from_address(fcb
->Vcb
, ed2
->address
);
3692 ERR("get_chunk_from_address(%llx) failed\n", ed2
->address
);
3694 Status
= update_changed_extent_ref(fcb
->Vcb
, c
, ed2
->address
, ed2
->size
, fcb
->subvol
->id
, fcb
->inode
, ext
->offset
- ed2
->offset
, 1,
3695 fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
, FALSE
, Irp
);
3697 if (!NT_SUCCESS(Status
)) {
3698 ERR("update_changed_extent_ref returned %08x\n", Status
);
3703 remove_fcb_extent(fcb
, ext
, rollback
);
3704 } else if (start_data
> ext
->offset
&& end_data
>= ext
->offset
+ ed2
->num_bytes
) { // replace end
3706 extent
*newext1
, *newext2
;
3708 newext1
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + ext
->datalen
, ALLOC_TAG
);
3710 ERR("out of memory\n");
3711 return STATUS_INSUFFICIENT_RESOURCES
;
3714 newext2
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + ext
->datalen
, ALLOC_TAG
);
3716 ERR("out of memory\n");
3717 ExFreePool(newext1
);
3718 return STATUS_INSUFFICIENT_RESOURCES
;
3721 RtlCopyMemory(&newext1
->extent_data
, &ext
->extent_data
, ext
->datalen
);
3723 ned2
= (EXTENT_DATA2
*)newext1
->extent_data
.data
;
3724 ned2
->num_bytes
= start_data
- ext
->offset
;
3726 RtlCopyMemory(&newext2
->extent_data
, &ext
->extent_data
, ext
->datalen
);
3728 newext2
->extent_data
.type
= EXTENT_TYPE_REGULAR
;
3729 ned2
= (EXTENT_DATA2
*)newext2
->extent_data
.data
;
3730 ned2
->offset
+= start_data
- ext
->offset
;
3731 ned2
->num_bytes
= ext
->offset
+ ed2
->num_bytes
- start_data
;
3733 Status
= write_data_complete(fcb
->Vcb
, ed2
->address
+ ned2
->offset
, data
, (UINT32
)ned2
->num_bytes
, Irp
, NULL
, file_write
, irp_offset
, priority
);
3734 if (!NT_SUCCESS(Status
)) {
3735 ERR("write_data_complete returned %08x\n", Status
);
3739 if (!(fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
)) {
3740 ULONG sl
= (ULONG
)(ned2
->num_bytes
/ fcb
->Vcb
->superblock
.sector_size
);
3741 UINT32
* csum
= ExAllocatePoolWithTag(PagedPool
, sl
* sizeof(UINT32
), ALLOC_TAG
);
3744 ERR("out of memory\n");
3745 ExFreePool(newext1
);
3746 ExFreePool(newext2
);
3747 return STATUS_INSUFFICIENT_RESOURCES
;
3750 Status
= calc_csum(fcb
->Vcb
, data
, sl
, csum
);
3751 if (!NT_SUCCESS(Status
)) {
3752 ERR("calc_csum returned %08x\n", Status
);
3753 ExFreePool(newext1
);
3754 ExFreePool(newext2
);
3759 newext2
->csum
= csum
;
3761 newext2
->csum
= NULL
;
3763 *written
= ned2
->num_bytes
;
3765 newext1
->offset
= ext
->offset
;
3766 newext1
->datalen
= ext
->datalen
;
3767 newext1
->unique
= ext
->unique
;
3768 newext1
->ignore
= FALSE
;
3769 newext1
->inserted
= TRUE
;
3770 newext1
->csum
= NULL
;
3771 InsertHeadList(&ext
->list_entry
, &newext1
->list_entry
);
3773 add_insert_extent_rollback(rollback
, fcb
, newext1
);
3775 newext2
->offset
= start_data
;
3776 newext2
->datalen
= ext
->datalen
;
3777 newext2
->unique
= ext
->unique
;
3778 newext2
->ignore
= FALSE
;
3779 newext2
->inserted
= TRUE
;
3780 add_extent(fcb
, &newext1
->list_entry
, newext2
);
3782 add_insert_extent_rollback(rollback
, fcb
, newext2
);
3784 c
= get_chunk_from_address(fcb
->Vcb
, ed2
->address
);
3787 ERR("get_chunk_from_address(%llx) failed\n", ed2
->address
);
3789 Status
= update_changed_extent_ref(fcb
->Vcb
, c
, ed2
->address
, ed2
->size
, fcb
->subvol
->id
, fcb
->inode
, ext
->offset
- ed2
->offset
, 1,
3790 fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
, FALSE
, Irp
);
3792 if (!NT_SUCCESS(Status
)) {
3793 ERR("update_changed_extent_ref returned %08x\n", Status
);
3798 remove_fcb_extent(fcb
, ext
, rollback
);
3799 } else if (start_data
> ext
->offset
&& end_data
< ext
->offset
+ ed2
->num_bytes
) { // replace middle
3801 extent
*newext1
, *newext2
, *newext3
;
3803 newext1
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + ext
->datalen
, ALLOC_TAG
);
3805 ERR("out of memory\n");
3806 return STATUS_INSUFFICIENT_RESOURCES
;
3809 newext2
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + ext
->datalen
, ALLOC_TAG
);
3811 ERR("out of memory\n");
3812 ExFreePool(newext1
);
3813 return STATUS_INSUFFICIENT_RESOURCES
;
3816 newext3
= ExAllocatePoolWithTag(PagedPool
, offsetof(extent
, extent_data
) + ext
->datalen
, ALLOC_TAG
);
3818 ERR("out of memory\n");
3819 ExFreePool(newext1
);
3820 ExFreePool(newext2
);
3821 return STATUS_INSUFFICIENT_RESOURCES
;
3824 RtlCopyMemory(&newext1
->extent_data
, &ext
->extent_data
, ext
->datalen
);
3825 RtlCopyMemory(&newext2
->extent_data
, &ext
->extent_data
, ext
->datalen
);
3826 RtlCopyMemory(&newext3
->extent_data
, &ext
->extent_data
, ext
->datalen
);
3828 ned2
= (EXTENT_DATA2
*)newext1
->extent_data
.data
;
3829 ned2
->num_bytes
= start_data
- ext
->offset
;
3831 newext2
->extent_data
.type
= EXTENT_TYPE_REGULAR
;
3832 ned2
= (EXTENT_DATA2
*)newext2
->extent_data
.data
;
3833 ned2
->offset
+= start_data
- ext
->offset
;
3834 ned2
->num_bytes
= end_data
- start_data
;
3836 ned2
= (EXTENT_DATA2
*)newext3
->extent_data
.data
;
3837 ned2
->offset
+= end_data
- ext
->offset
;
3838 ned2
->num_bytes
-= end_data
- ext
->offset
;
3840 ned2
= (EXTENT_DATA2
*)newext2
->extent_data
.data
;
3841 Status
= write_data_complete(fcb
->Vcb
, ed2
->address
+ ned2
->offset
, data
, (UINT32
)(end_data
- start_data
), Irp
, NULL
, file_write
, irp_offset
, priority
);
3842 if (!NT_SUCCESS(Status
)) {
3843 ERR("write_data_complete returned %08x\n", Status
);
3844 ExFreePool(newext1
);
3845 ExFreePool(newext2
);
3846 ExFreePool(newext3
);
3850 if (!(fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
)) {
3851 ULONG sl
= (ULONG
)((end_data
- start_data
) / fcb
->Vcb
->superblock
.sector_size
);
3852 UINT32
* csum
= ExAllocatePoolWithTag(PagedPool
, sl
* sizeof(UINT32
), ALLOC_TAG
);
3855 ERR("out of memory\n");
3856 ExFreePool(newext1
);
3857 ExFreePool(newext2
);
3858 ExFreePool(newext3
);
3859 return STATUS_INSUFFICIENT_RESOURCES
;
3862 Status
= calc_csum(fcb
->Vcb
, data
, sl
, csum
);
3863 if (!NT_SUCCESS(Status
)) {
3864 ERR("calc_csum returned %08x\n", Status
);
3865 ExFreePool(newext1
);
3866 ExFreePool(newext2
);
3867 ExFreePool(newext3
);
3872 newext2
->csum
= csum
;
3874 newext2
->csum
= NULL
;
3876 *written
= end_data
- start_data
;
3878 newext1
->offset
= ext
->offset
;
3879 newext1
->datalen
= ext
->datalen
;
3880 newext1
->unique
= ext
->unique
;
3881 newext1
->ignore
= FALSE
;
3882 newext1
->inserted
= TRUE
;
3883 newext1
->csum
= NULL
;
3884 InsertHeadList(&ext
->list_entry
, &newext1
->list_entry
);
3886 add_insert_extent_rollback(rollback
, fcb
, newext1
);
3888 newext2
->offset
= start_data
;
3889 newext2
->datalen
= ext
->datalen
;
3890 newext2
->unique
= ext
->unique
;
3891 newext2
->ignore
= FALSE
;
3892 newext2
->inserted
= TRUE
;
3893 add_extent(fcb
, &newext1
->list_entry
, newext2
);
3895 add_insert_extent_rollback(rollback
, fcb
, newext2
);
3897 newext3
->offset
= end_data
;
3898 newext3
->datalen
= ext
->datalen
;
3899 newext3
->unique
= ext
->unique
;
3900 newext3
->ignore
= FALSE
;
3901 newext3
->inserted
= TRUE
;
3902 newext3
->csum
= NULL
;
3903 add_extent(fcb
, &newext2
->list_entry
, newext3
);
3905 add_insert_extent_rollback(rollback
, fcb
, newext3
);
3907 c
= get_chunk_from_address(fcb
->Vcb
, ed2
->address
);
3910 ERR("get_chunk_from_address(%llx) failed\n", ed2
->address
);
3912 Status
= update_changed_extent_ref(fcb
->Vcb
, c
, ed2
->address
, ed2
->size
, fcb
->subvol
->id
, fcb
->inode
, ext
->offset
- ed2
->offset
, 2,
3913 fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
, FALSE
, Irp
);
3915 if (!NT_SUCCESS(Status
)) {
3916 ERR("update_changed_extent_ref returned %08x\n", Status
);
3921 remove_fcb_extent(fcb
, ext
, rollback
);
3927 return STATUS_SUCCESS
;
3930 NTSTATUS
do_write_file(fcb
* fcb
, UINT64 start
, UINT64 end_data
, void* data
, PIRP Irp
, BOOL file_write
, UINT32 irp_offset
, LIST_ENTRY
* rollback
) {
3932 LIST_ENTRY
*le
, *le2
;
3933 UINT64 written
= 0, length
= end_data
- start
;
3934 UINT64 last_cow_start
;
3935 ULONG priority
= fcb
->Header
.Flags2
& FSRTL_FLAG2_IS_PAGING_FILE
? HighPagePriority
: NormalPagePriority
;
3936 #ifdef DEBUG_PARANOID
3942 le
= fcb
->extents
.Flink
;
3943 while (le
!= &fcb
->extents
) {
3944 extent
* ext
= CONTAINING_RECORD(le
, extent
, list_entry
);
3949 EXTENT_DATA
* ed
= &ext
->extent_data
;
3950 EXTENT_DATA2
* ed2
= ed
->type
== EXTENT_TYPE_INLINE
? NULL
: (EXTENT_DATA2
*)ed
->data
;
3953 len
= ed
->type
== EXTENT_TYPE_INLINE
? ed
->decoded_size
: ed2
->num_bytes
;
3955 if (ext
->offset
+ len
<= start
)
3958 if (ext
->offset
> start
+ written
+ length
)
3961 if ((fcb
->inode_item
.flags
& BTRFS_INODE_NODATACOW
|| ed
->type
== EXTENT_TYPE_PREALLOC
) && ext
->unique
&& ed
->compression
== BTRFS_COMPRESSION_NONE
) {
3962 if (max(last_cow_start
, start
+ written
) < ext
->offset
) {
3963 UINT64 start_write
= max(last_cow_start
, start
+ written
);
3965 Status
= excise_extents(fcb
->Vcb
, fcb
, start_write
, ext
->offset
, Irp
, rollback
);
3966 if (!NT_SUCCESS(Status
)) {
3967 ERR("excise_extents returned %08x\n", Status
);
3971 Status
= insert_extent(fcb
->Vcb
, fcb
, start_write
, ext
->offset
- start_write
, (UINT8
*)data
+ written
, Irp
, file_write
, irp_offset
+ written
, rollback
);
3972 if (!NT_SUCCESS(Status
)) {
3973 ERR("insert_extent returned %08x\n", Status
);
3977 written
+= ext
->offset
- start_write
;
3978 length
-= ext
->offset
- start_write
;
3984 if (ed
->type
== EXTENT_TYPE_REGULAR
) {
3985 UINT64 writeaddr
= ed2
->address
+ ed2
->offset
+ start
+ written
- ext
->offset
;
3986 UINT64 write_len
= min(len
, length
);
3989 TRACE("doing non-COW write to %llx\n", writeaddr
);
3991 Status
= write_data_complete(fcb
->Vcb
, writeaddr
, (UINT8
*)data
+ written
, (UINT32
)write_len
, Irp
, NULL
, file_write
, irp_offset
+ written
, priority
);
3992 if (!NT_SUCCESS(Status
)) {
3993 ERR("write_data_complete returned %08x\n", Status
);
3997 c
= get_chunk_from_address(fcb
->Vcb
, writeaddr
);
4001 // This shouldn't ever get called - nocow files should always also be nosum.
4002 if (!(fcb
->inode_item
.flags
& BTRFS_INODE_NODATASUM
)) {
4003 calc_csum(fcb
->Vcb
, (UINT8
*)data
+ written
, (UINT32
)(write_len
/ fcb
->Vcb
->superblock
.sector_size
),
4004 &ext
->csum
[(start
+ written
- ext
->offset
) / fcb
->Vcb
->superblock
.sector_size
]);
4006 ext
->inserted
= TRUE
;
4009 written
+= write_len
;
4010 length
-= write_len
;
4014 } else if (ed
->type
== EXTENT_TYPE_PREALLOC
) {
4017 Status
= do_write_file_prealloc(fcb
, ext
, start
+ written
, end_data
, (UINT8
*)data
+ written
, &write_len
,
4018 Irp
, file_write
, irp_offset
+ written
, priority
, rollback
);
4019 if (!NT_SUCCESS(Status
)) {
4020 ERR("do_write_file_prealloc returned %08x\n", Status
);
4024 written
+= write_len
;
4025 length
-= write_len
;
4031 last_cow_start
= ext
->offset
+ len
;
4040 UINT64 start_write
= max(last_cow_start
, start
+ written
);
4042 Status
= excise_extents(fcb
->Vcb
, fcb
, start_write
, end_data
, Irp
, rollback
);
4043 if (!NT_SUCCESS(Status
)) {
4044 ERR("excise_extents returned %08x\n", Status
);
4048 Status
= insert_extent(fcb
->Vcb
, fcb
, start_write
, end_data
- start_write
, (UINT8
*)data
+ written
, Irp
, file_write
, irp_offset
+ written
, rollback
);
4049 if (!NT_SUCCESS(Status
)) {
4050 ERR("insert_extent returned %08x\n", Status
);
4055 #ifdef DEBUG_PARANOID
4056 last_off
= 0xffffffffffffffff;
4058 le
= fcb
->extents
.Flink
;
4059 while (le
!= &fcb
->extents
) {
4060 extent
* ext
= CONTAINING_RECORD(le
, extent
, list_entry
);
4063 if (ext
->offset
== last_off
) {
4064 ERR("offset %llx duplicated\n", ext
->offset
);
4066 } else if (ext
->offset
< last_off
&& last_off
!= 0xffffffffffffffff) {
4067 ERR("offsets out of order\n");
4071 last_off
= ext
->offset
;
4078 fcb
->extents_changed
= TRUE
;
4079 mark_fcb_dirty(fcb
);
4081 return STATUS_SUCCESS
;
4084 NTSTATUS
write_compressed(fcb
* fcb
, UINT64 start_data
, UINT64 end_data
, void* data
, PIRP Irp
, LIST_ENTRY
* rollback
) {
4088 for (i
= 0; i
< sector_align(end_data
- start_data
, COMPRESSED_EXTENT_SIZE
) / COMPRESSED_EXTENT_SIZE
; i
++) {
4092 s2
= start_data
+ (i
* COMPRESSED_EXTENT_SIZE
);
4093 e2
= min(s2
+ COMPRESSED_EXTENT_SIZE
, end_data
);
4095 Status
= write_compressed_bit(fcb
, s2
, e2
, (UINT8
*)data
+ (i
* COMPRESSED_EXTENT_SIZE
), &compressed
, Irp
, rollback
);
4097 if (!NT_SUCCESS(Status
)) {
4098 ERR("write_compressed_bit returned %08x\n", Status
);
4102 // If the first 128 KB of a file is incompressible, we set the nocompress flag so we don't
4103 // bother with the rest of it.
4104 if (s2
== 0 && e2
== COMPRESSED_EXTENT_SIZE
&& !compressed
&& !fcb
->Vcb
->options
.compress_force
) {
4105 fcb
->inode_item
.flags
|= BTRFS_INODE_NOCOMPRESS
;
4106 fcb
->inode_item_changed
= TRUE
;
4107 mark_fcb_dirty(fcb
);
4109 // write subsequent data non-compressed
4110 if (e2
< end_data
) {
4111 Status
= do_write_file(fcb
, e2
, end_data
, (UINT8
*)data
+ e2
, Irp
, FALSE
, 0, rollback
);
4113 if (!NT_SUCCESS(Status
)) {
4114 ERR("do_write_file returned %08x\n", Status
);
4119 return STATUS_SUCCESS
;
4123 return STATUS_SUCCESS
;
4126 NTSTATUS
write_file2(device_extension
* Vcb
, PIRP Irp
, LARGE_INTEGER offset
, void* buf
, ULONG
* length
, BOOLEAN paging_io
, BOOLEAN no_cache
,
4127 BOOLEAN wait
, BOOLEAN deferred_write
, BOOLEAN write_irp
, LIST_ENTRY
* rollback
) {
4128 PIO_STACK_LOCATION IrpSp
= IoGetCurrentIrpStackLocation(Irp
);
4129 PFILE_OBJECT FileObject
= IrpSp
->FileObject
;
4131 UINT64 off64
, newlength
, start_data
, end_data
;
4136 BOOL changed_length
= FALSE
;
4143 BOOL paging_lock
= FALSE
, fcb_lock
= FALSE
, tree_lock
= FALSE
, pagefile
;
4146 TRACE("(%p, %p, %llx, %p, %x, %u, %u)\n", Vcb
, FileObject
, offset
.QuadPart
, buf
, *length
, paging_io
, no_cache
);
4149 WARN("returning success for zero-length write\n");
4150 return STATUS_SUCCESS
;
4154 ERR("error - FileObject was NULL\n");
4155 return STATUS_ACCESS_DENIED
;
4158 fcb
= FileObject
->FsContext
;
4159 ccb
= FileObject
->FsContext2
;
4160 fileref
= ccb
? ccb
->fileref
: NULL
;
4162 if (!fcb
->ads
&& fcb
->type
!= BTRFS_TYPE_FILE
&& fcb
->type
!= BTRFS_TYPE_SYMLINK
) {
4163 WARN("tried to write to something other than a file or symlink (inode %llx, type %u, %p, %p)\n", fcb
->inode
, fcb
->type
, &fcb
->type
, fcb
);
4164 return STATUS_INVALID_DEVICE_REQUEST
;
4167 if (offset
.LowPart
== FILE_WRITE_TO_END_OF_FILE
&& offset
.HighPart
== -1)
4168 offset
= fcb
->Header
.FileSize
;
4170 off64
= offset
.QuadPart
;
4172 TRACE("fcb->Header.Flags = %x\n", fcb
->Header
.Flags
);
4174 if (!no_cache
&& !CcCanIWrite(FileObject
, *length
, wait
, deferred_write
))
4175 return STATUS_PENDING
;
4177 if (!wait
&& no_cache
)
4178 return STATUS_PENDING
;
4180 if (no_cache
&& !paging_io
&& FileObject
->SectionObjectPointer
->DataSectionObject
) {
4181 IO_STATUS_BLOCK iosb
;
4183 ExAcquireResourceExclusiveLite(fcb
->Header
.PagingIoResource
, TRUE
);
4185 CcFlushCache(FileObject
->SectionObjectPointer
, &offset
, *length
, &iosb
);
4187 if (!NT_SUCCESS(iosb
.Status
)) {
4188 ExReleaseResourceLite(fcb
->Header
.PagingIoResource
);
4189 ERR("CcFlushCache returned %08x\n", iosb
.Status
);
4195 CcPurgeCacheSection(FileObject
->SectionObjectPointer
, &offset
, *length
, FALSE
);
4199 if (!ExAcquireResourceSharedLite(fcb
->Header
.PagingIoResource
, wait
)) {
4200 Status
= STATUS_PENDING
;
4206 pagefile
= fcb
->Header
.Flags2
& FSRTL_FLAG2_IS_PAGING_FILE
&& paging_io
;
4208 if (!pagefile
&& !ExIsResourceAcquiredExclusiveLite(&Vcb
->tree_lock
)) {
4209 if (!ExAcquireResourceSharedLite(&Vcb
->tree_lock
, wait
)) {
4210 Status
= STATUS_PENDING
;
4218 if (!ExAcquireResourceSharedLite(fcb
->Header
.Resource
, wait
)) {
4219 Status
= STATUS_PENDING
;
4223 } else if (!ExIsResourceAcquiredExclusiveLite(fcb
->Header
.Resource
)) {
4224 if (!ExAcquireResourceExclusiveLite(fcb
->Header
.Resource
, wait
)) {
4225 Status
= STATUS_PENDING
;
4232 newlength
= fcb
->ads
? fcb
->adsdata
.Length
: fcb
->inode_item
.st_size
;
4237 TRACE("newlength = %llx\n", newlength
);
4239 if (off64
+ *length
> newlength
) {
4241 if (off64
>= newlength
) {
4242 TRACE("paging IO tried to write beyond end of file (file size = %llx, offset = %llx, length = %x)\n", newlength
, off64
, *length
);
4243 TRACE("filename %S\n", file_desc(FileObject
));
4244 TRACE("FileObject: AllocationSize = %llx, FileSize = %llx, ValidDataLength = %llx\n",
4245 fcb
->Header
.AllocationSize
.QuadPart
, fcb
->Header
.FileSize
.QuadPart
, fcb
->Header
.ValidDataLength
.QuadPart
);
4246 Status
= STATUS_SUCCESS
;
4250 *length
= (ULONG
)(newlength
- off64
);
4252 newlength
= off64
+ *length
;
4253 changed_length
= TRUE
;
4255 TRACE("extending length to %llx\n", newlength
);
4260 make_inline
= FALSE
;
4261 else if (fcb
->type
== BTRFS_TYPE_SYMLINK
)
4262 make_inline
= newlength
<= (Vcb
->superblock
.node_size
- sizeof(tree_header
) - sizeof(leaf_node
) - offsetof(EXTENT_DATA
, data
[0]));
4264 make_inline
= newlength
<= fcb
->Vcb
->options
.max_inline
;
4266 if (changed_length
) {
4267 if (newlength
> (UINT64
)fcb
->Header
.AllocationSize
.QuadPart
) {
4269 // We need to acquire the tree lock if we don't have it already -
4270 // we can't give an inline file proper extents at the same time as we're
4272 if (!ExAcquireResourceSharedLite(&Vcb
->tree_lock
, wait
)) {
4273 Status
= STATUS_PENDING
;
4279 Status
= extend_file(fcb
, fileref
, newlength
, FALSE
, Irp
, rollback
);
4280 if (!NT_SUCCESS(Status
)) {
4281 ERR("extend_file returned %08x\n", Status
);
4284 } else if (!fcb
->ads
)
4285 fcb
->inode_item
.st_size
= newlength
;
4287 fcb
->Header
.FileSize
.QuadPart
= newlength
;
4288 fcb
->Header
.ValidDataLength
.QuadPart
= newlength
;
4290 TRACE("AllocationSize = %llx\n", fcb
->Header
.AllocationSize
.QuadPart
);
4291 TRACE("FileSize = %llx\n", fcb
->Header
.FileSize
.QuadPart
);
4292 TRACE("ValidDataLength = %llx\n", fcb
->Header
.ValidDataLength
.QuadPart
);
4296 Status
= STATUS_SUCCESS
;
4299 if (!FileObject
->PrivateCacheMap
|| changed_length
) {
4302 ccfs
.AllocationSize
= fcb
->Header
.AllocationSize
;
4303 ccfs
.FileSize
= fcb
->Header
.FileSize
;
4304 ccfs
.ValidDataLength
= fcb
->Header
.ValidDataLength
;
4306 if (!FileObject
->PrivateCacheMap
)
4307 init_file_cache(FileObject
, &ccfs
);
4309 CcSetFileSizes(FileObject
, &ccfs
);
4312 if (IrpSp
->MinorFunction
& IRP_MN_MDL
) {
4313 CcPrepareMdlWrite(FileObject
, &offset
, *length
, &Irp
->MdlAddress
, &Irp
->IoStatus
);
4315 Status
= Irp
->IoStatus
.Status
;
4318 if (fCcCopyWriteEx
) {
4319 TRACE("CcCopyWriteEx(%p, %llx, %x, %u, %p, %p)\n", FileObject
, off64
, *length
, wait
, buf
, Irp
->Tail
.Overlay
.Thread
);
4320 if (!fCcCopyWriteEx(FileObject
, &offset
, *length
, wait
, buf
, Irp
->Tail
.Overlay
.Thread
)) {
4321 Status
= STATUS_PENDING
;
4324 TRACE("CcCopyWriteEx finished\n");
4326 TRACE("CcCopyWrite(%p, %llx, %x, %u, %p)\n", FileObject
, off64
, *length
, wait
, buf
);
4327 if (!CcCopyWrite(FileObject
, &offset
, *length
, wait
, buf
)) {
4328 Status
= STATUS_PENDING
;
4331 TRACE("CcCopyWrite finished\n");
4334 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
4335 Status
= _SEH2_GetExceptionCode();
4342 if (changed_length
) {
4345 if (newlength
> fcb
->adsmaxlen
) {
4346 ERR("error - xattr too long (%llu > %u)\n", newlength
, fcb
->adsmaxlen
);
4347 Status
= STATUS_DISK_FULL
;
4351 data2
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)newlength
, ALLOC_TAG
);
4353 ERR("out of memory\n");
4354 Status
= STATUS_INSUFFICIENT_RESOURCES
;
4358 if (fcb
->adsdata
.Buffer
) {
4359 RtlCopyMemory(data2
, fcb
->adsdata
.Buffer
, fcb
->adsdata
.Length
);
4360 ExFreePool(fcb
->adsdata
.Buffer
);
4363 if (newlength
> fcb
->adsdata
.Length
)
4364 RtlZeroMemory(&data2
[fcb
->adsdata
.Length
], (ULONG
)(newlength
- fcb
->adsdata
.Length
));
4367 fcb
->adsdata
.Buffer
= data2
;
4368 fcb
->adsdata
.Length
= fcb
->adsdata
.MaximumLength
= (USHORT
)newlength
;
4370 fcb
->Header
.AllocationSize
.QuadPart
= newlength
;
4371 fcb
->Header
.FileSize
.QuadPart
= newlength
;
4372 fcb
->Header
.ValidDataLength
.QuadPart
= newlength
;
4376 RtlCopyMemory(&fcb
->adsdata
.Buffer
[off64
], buf
, *length
);
4378 fcb
->Header
.ValidDataLength
.QuadPart
= newlength
;
4380 mark_fcb_dirty(fcb
);
4383 mark_fileref_dirty(fileref
);
4385 BOOL compress
= write_fcb_compressed(fcb
), no_buf
= FALSE
;
4389 end_data
= sector_align(newlength
, fcb
->Vcb
->superblock
.sector_size
);
4390 bufhead
= sizeof(EXTENT_DATA
) - 1;
4391 } else if (compress
) {
4392 start_data
= off64
& ~(UINT64
)(COMPRESSED_EXTENT_SIZE
- 1);
4393 end_data
= min(sector_align(off64
+ *length
, COMPRESSED_EXTENT_SIZE
),
4394 sector_align(newlength
, fcb
->Vcb
->superblock
.sector_size
));
4397 start_data
= off64
& ~(UINT64
)(fcb
->Vcb
->superblock
.sector_size
- 1);
4398 end_data
= sector_align(off64
+ *length
, fcb
->Vcb
->superblock
.sector_size
);
4402 if (fcb_is_inline(fcb
))
4403 end_data
= max(end_data
, sector_align(fcb
->inode_item
.st_size
, Vcb
->superblock
.sector_size
));
4405 fcb
->Header
.ValidDataLength
.QuadPart
= newlength
;
4406 TRACE("fcb %p FileSize = %llx\n", fcb
, fcb
->Header
.FileSize
.QuadPart
);
4408 if (!make_inline
&& !compress
&& off64
== start_data
&& off64
+ *length
== end_data
) {
4412 data
= ExAllocatePoolWithTag(PagedPool
, (ULONG
)(end_data
- start_data
+ bufhead
), ALLOC_TAG
);
4414 ERR("out of memory\n");
4415 Status
= STATUS_INSUFFICIENT_RESOURCES
;
4419 RtlZeroMemory(data
+ bufhead
, (ULONG
)(end_data
- start_data
));
4421 TRACE("start_data = %llx\n", start_data
);
4422 TRACE("end_data = %llx\n", end_data
);
4424 if (off64
> start_data
|| off64
+ *length
< end_data
) {
4425 if (changed_length
) {
4426 if (fcb
->inode_item
.st_size
> start_data
)
4427 Status
= read_file(fcb
, data
+ bufhead
, start_data
, fcb
->inode_item
.st_size
- start_data
, NULL
, Irp
);
4429 Status
= STATUS_SUCCESS
;
4431 Status
= read_file(fcb
, data
+ bufhead
, start_data
, end_data
- start_data
, NULL
, Irp
);
4433 if (!NT_SUCCESS(Status
)) {
4434 ERR("read_file returned %08x\n", Status
);
4440 RtlCopyMemory(data
+ bufhead
+ off64
- start_data
, buf
, *length
);
4444 Status
= excise_extents(fcb
->Vcb
, fcb
, start_data
, end_data
, Irp
, rollback
);
4445 if (!NT_SUCCESS(Status
)) {
4446 ERR("error - excise_extents returned %08x\n", Status
);
4451 ed2
= (EXTENT_DATA
*)data
;
4452 ed2
->generation
= fcb
->Vcb
->superblock
.generation
;
4453 ed2
->decoded_size
= newlength
;
4454 ed2
->compression
= BTRFS_COMPRESSION_NONE
;
4455 ed2
->encryption
= BTRFS_ENCRYPTION_NONE
;
4456 ed2
->encoding
= BTRFS_ENCODING_NONE
;
4457 ed2
->type
= EXTENT_TYPE_INLINE
;
4459 Status
= add_extent_to_fcb(fcb
, 0, ed2
, (UINT16
)(offsetof(EXTENT_DATA
, data
[0]) + newlength
), FALSE
, NULL
, rollback
);
4460 if (!NT_SUCCESS(Status
)) {
4461 ERR("add_extent_to_fcb returned %08x\n", Status
);
4466 fcb
->inode_item
.st_blocks
+= newlength
;
4467 } else if (compress
) {
4468 Status
= write_compressed(fcb
, start_data
, end_data
, data
, Irp
, rollback
);
4470 if (!NT_SUCCESS(Status
)) {
4471 ERR("write_compressed returned %08x\n", Status
);
4478 if (write_irp
&& Irp
->MdlAddress
&& no_buf
) {
4479 BOOL locked
= Irp
->MdlAddress
->MdlFlags
& MDL_PAGES_LOCKED
;
4482 Status
= STATUS_SUCCESS
;
4485 MmProbeAndLockPages(Irp
->MdlAddress
, KernelMode
, IoReadAccess
);
4486 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
4487 Status
= _SEH2_GetExceptionCode();
4490 if (!NT_SUCCESS(Status
)) {
4491 ERR("MmProbeAndLockPages threw exception %08x\n", Status
);
4497 Status
= do_write_file(fcb
, start_data
, end_data
, data
, Irp
, TRUE
, 0, rollback
);
4498 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
4499 Status
= _SEH2_GetExceptionCode();
4503 MmUnlockPages(Irp
->MdlAddress
);
4506 Status
= do_write_file(fcb
, start_data
, end_data
, data
, Irp
, FALSE
, 0, rollback
);
4507 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
4508 Status
= _SEH2_GetExceptionCode();
4512 if (!NT_SUCCESS(Status
)) {
4513 ERR("do_write_file returned %08x\n", Status
);
4514 if (!no_buf
) ExFreePool(data
);
4523 KeQuerySystemTime(&time
);
4524 win_time_to_unix(time
, &now
);
4528 if (fileref
&& fileref
->parent
)
4529 origii
= &fileref
->parent
->fcb
->inode_item
;
4531 ERR("no parent fcb found for stream\n");
4532 Status
= STATUS_INTERNAL_ERROR
;
4536 origii
= &fcb
->inode_item
;
4538 origii
->transid
= Vcb
->superblock
.generation
;
4541 if (!ccb
->user_set_change_time
)
4542 origii
->st_ctime
= now
;
4545 if (changed_length
) {
4546 TRACE("setting st_size to %llx\n", newlength
);
4547 origii
->st_size
= newlength
;
4548 filter
|= FILE_NOTIFY_CHANGE_SIZE
;
4551 fcb
->inode_item_changed
= TRUE
;
4553 fileref
->parent
->fcb
->inode_item_changed
= TRUE
;
4556 filter
|= FILE_NOTIFY_CHANGE_SIZE
;
4559 if (!ccb
->user_set_write_time
) {
4560 origii
->st_mtime
= now
;
4561 filter
|= FILE_NOTIFY_CHANGE_LAST_WRITE
;
4564 mark_fcb_dirty(fcb
->ads
? fileref
->parent
->fcb
: fcb
);
4567 if (changed_length
) {
4570 ccfs
.AllocationSize
= fcb
->Header
.AllocationSize
;
4571 ccfs
.FileSize
= fcb
->Header
.FileSize
;
4572 ccfs
.ValidDataLength
= fcb
->Header
.ValidDataLength
;
4575 CcSetFileSizes(FileObject
, &ccfs
);
4576 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
4577 Status
= _SEH2_GetExceptionCode();
4582 fcb
->subvol
->root_item
.ctransid
= Vcb
->superblock
.generation
;
4583 fcb
->subvol
->root_item
.ctime
= now
;
4585 Status
= STATUS_SUCCESS
;
4588 send_notification_fcb(fcb
->ads
? fileref
->parent
: fileref
, filter
, FILE_ACTION_MODIFIED
, fcb
->ads
&& fileref
->dc
? &fileref
->dc
->name
: NULL
);
4591 if (NT_SUCCESS(Status
) && FileObject
->Flags
& FO_SYNCHRONOUS_IO
&& !paging_io
) {
4592 TRACE("CurrentByteOffset was: %llx\n", FileObject
->CurrentByteOffset
.QuadPart
);
4593 FileObject
->CurrentByteOffset
.QuadPart
= offset
.QuadPart
+ (NT_SUCCESS(Status
) ? *length
: 0);
4594 TRACE("CurrentByteOffset now: %llx\n", FileObject
->CurrentByteOffset
.QuadPart
);
4598 ExReleaseResourceLite(fcb
->Header
.Resource
);
4601 ExReleaseResourceLite(&Vcb
->tree_lock
);
4604 ExReleaseResourceLite(fcb
->Header
.PagingIoResource
);
4609 NTSTATUS
write_file(device_extension
* Vcb
, PIRP Irp
, BOOLEAN wait
, BOOLEAN deferred_write
) {
4610 PIO_STACK_LOCATION IrpSp
= IoGetCurrentIrpStackLocation(Irp
);
4613 LARGE_INTEGER offset
= IrpSp
->Parameters
.Write
.ByteOffset
;
4614 PFILE_OBJECT FileObject
= IrpSp
->FileObject
;
4615 fcb
* fcb
= FileObject
? FileObject
->FsContext
: NULL
;
4616 LIST_ENTRY rollback
;
4618 InitializeListHead(&rollback
);
4622 Irp
->IoStatus
.Information
= 0;
4624 TRACE("offset = %llx\n", offset
.QuadPart
);
4625 TRACE("length = %x\n", IrpSp
->Parameters
.Write
.Length
);
4627 if (!Irp
->AssociatedIrp
.SystemBuffer
) {
4628 buf
= map_user_buffer(Irp
, fcb
&& fcb
->Header
.Flags2
& FSRTL_FLAG2_IS_PAGING_FILE
? HighPagePriority
: NormalPagePriority
);
4630 if (Irp
->MdlAddress
&& !buf
) {
4631 ERR("MmGetSystemAddressForMdlSafe returned NULL\n");
4632 Status
= STATUS_INSUFFICIENT_RESOURCES
;
4636 buf
= Irp
->AssociatedIrp
.SystemBuffer
;
4638 TRACE("buf = %p\n", buf
);
4640 if (fcb
&& !(Irp
->Flags
& IRP_PAGING_IO
) && !FsRtlCheckLockForWriteAccess(&fcb
->lock
, Irp
)) {
4641 WARN("tried to write to locked region\n");
4642 Status
= STATUS_FILE_LOCK_CONFLICT
;
4646 Status
= write_file2(Vcb
, Irp
, offset
, buf
, &IrpSp
->Parameters
.Write
.Length
, Irp
->Flags
& IRP_PAGING_IO
, Irp
->Flags
& IRP_NOCACHE
,
4647 wait
, deferred_write
, TRUE
, &rollback
);
4649 if (Status
== STATUS_PENDING
)
4651 else if (!NT_SUCCESS(Status
)) {
4652 ERR("write_file2 returned %08x\n", Status
);
4656 if (NT_SUCCESS(Status
)) {
4657 Irp
->IoStatus
.Information
= IrpSp
->Parameters
.Write
.Length
;
4659 if (diskacc
&& Status
!= STATUS_PENDING
&& Irp
->Flags
& IRP_NOCACHE
) {
4660 PETHREAD thread
= NULL
;
4662 if (Irp
->Tail
.Overlay
.Thread
&& !IoIsSystemThread(Irp
->Tail
.Overlay
.Thread
))
4663 thread
= Irp
->Tail
.Overlay
.Thread
;
4664 else if (!IoIsSystemThread(PsGetCurrentThread()))
4665 thread
= PsGetCurrentThread();
4666 else if (IoIsSystemThread(PsGetCurrentThread()) && IoGetTopLevelIrp() == Irp
)
4667 thread
= PsGetCurrentThread();
4670 fPsUpdateDiskCounters(PsGetThreadProcess(thread
), 0, IrpSp
->Parameters
.Write
.Length
, 0, 1, 0);
4675 if (NT_SUCCESS(Status
))
4676 clear_rollback(&rollback
);
4678 do_rollback(Vcb
, &rollback
);
4683 _Dispatch_type_(IRP_MJ_WRITE
)
4684 _Function_class_(DRIVER_DISPATCH
)
4685 NTSTATUS
drv_write(IN PDEVICE_OBJECT DeviceObject
, IN PIRP Irp
) {
4688 PIO_STACK_LOCATION IrpSp
= IoGetCurrentIrpStackLocation(Irp
);
4689 device_extension
* Vcb
= DeviceObject
->DeviceExtension
;
4690 PFILE_OBJECT FileObject
= IrpSp
->FileObject
;
4691 fcb
* fcb
= FileObject
? FileObject
->FsContext
: NULL
;
4692 ccb
* ccb
= FileObject
? FileObject
->FsContext2
: NULL
;
4693 BOOLEAN wait
= FileObject
? IoIsOperationSynchronous(Irp
) : TRUE
;
4695 FsRtlEnterFileSystem();
4697 top_level
= is_top_level(Irp
);
4699 if (Vcb
&& Vcb
->type
== VCB_TYPE_VOLUME
) {
4700 Status
= vol_write(DeviceObject
, Irp
);
4702 } else if (!Vcb
|| Vcb
->type
!= VCB_TYPE_FS
) {
4703 Status
= STATUS_INVALID_PARAMETER
;
4708 ERR("fcb was NULL\n");
4709 Status
= STATUS_INVALID_PARAMETER
;
4714 ERR("ccb was NULL\n");
4715 Status
= STATUS_INVALID_PARAMETER
;
4719 if (Irp
->RequestorMode
== UserMode
&& !(ccb
->access
& (FILE_WRITE_DATA
| FILE_APPEND_DATA
))) {
4720 WARN("insufficient permissions\n");
4721 Status
= STATUS_ACCESS_DENIED
;
4725 if (fcb
== Vcb
->volume_fcb
) {
4726 if (!Vcb
->locked
|| Vcb
->locked_fileobj
!= FileObject
) {
4727 ERR("trying to write to volume when not locked, or locked with another FileObject\n");
4728 Status
= STATUS_ACCESS_DENIED
;
4732 TRACE("writing directly to volume\n");
4734 IoSkipCurrentIrpStackLocation(Irp
);
4736 Status
= IoCallDriver(Vcb
->Vpb
->RealDevice
, Irp
);
4740 if (is_subvol_readonly(fcb
->subvol
, Irp
)) {
4741 Status
= STATUS_ACCESS_DENIED
;
4745 if (Vcb
->readonly
) {
4746 Status
= STATUS_MEDIA_WRITE_PROTECTED
;
4751 if (IrpSp
->MinorFunction
& IRP_MN_COMPLETE
) {
4752 CcMdlWriteComplete(IrpSp
->FileObject
, &IrpSp
->Parameters
.Write
.ByteOffset
, Irp
->MdlAddress
);
4754 Irp
->MdlAddress
= NULL
;
4755 Status
= STATUS_SUCCESS
;
4757 // Don't offload jobs when doing paging IO - otherwise this can lead to
4758 // deadlocks in CcCopyWrite.
4759 if (Irp
->Flags
& IRP_PAGING_IO
)
4762 Status
= write_file(Vcb
, Irp
, wait
, FALSE
);
4764 } _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER
) {
4765 Status
= _SEH2_GetExceptionCode();
4769 Irp
->IoStatus
.Status
= Status
;
4771 TRACE("wrote %u bytes\n", Irp
->IoStatus
.Information
);
4773 if (Status
!= STATUS_PENDING
)
4774 IoCompleteRequest(Irp
, IO_NO_INCREMENT
);
4776 IoMarkIrpPending(Irp
);
4778 if (!add_thread_job(Vcb
, Irp
))
4779 do_write_job(Vcb
, Irp
);
4784 IoSetTopLevelIrp(NULL
);
4786 TRACE("returning %08x\n", Status
);
4788 FsRtlExitFileSystem();