2 * xzlib.c: front end for the transparent suport of lzma compression
3 * at the I/O layer, based on an example file from lzma project
5 * See Copyright for the status of this software.
7 * Anders F Bjorklund <afb@users.sourceforge.net>
11 #ifdef LIBXML_LZMA_ENABLED
19 #ifdef HAVE_SYS_TYPES_H
20 #include <sys/types.h>
22 #ifdef HAVE_SYS_STAT_H
34 #ifdef LIBXML_ZLIB_ENABLED
37 #ifdef LIBXML_LZMA_ENABLED
42 #include <libxml/xmlmemory.h>
44 /* values for xz_state how */
45 #define LOOK 0 /* look for a gzip/lzma header */
46 #define COPY 1 /* copy input directly */
47 #define GZIP 2 /* decompress a gzip stream */
48 #define LZMA 3 /* decompress a lzma stream */
50 /* internal lzma file state data structure */
52 int mode
; /* see lzma modes above */
53 int fd
; /* file descriptor */
54 char *path
; /* path or fd for error messages */
55 uint64_t pos
; /* current position in uncompressed data */
56 unsigned int size
; /* buffer size, zero if not allocated yet */
57 unsigned int want
; /* requested buffer size, default is BUFSIZ */
58 unsigned char *in
; /* input buffer */
59 unsigned char *out
; /* output buffer (double-sized when reading) */
60 unsigned char *next
; /* next output data to deliver or write */
61 unsigned int have
; /* amount of output data unused at next */
62 int eof
; /* true if end of input file reached */
63 uint64_t start
; /* where the lzma data started, for rewinding */
64 uint64_t raw
; /* where the raw data started, for seeking */
65 int how
; /* 0: get header, 1: copy, 2: decompress */
66 int direct
; /* true if last read direct, false if lzma */
68 uint64_t skip
; /* amount to skip (already rewound if backwards) */
69 int seek
; /* true if seek request pending */
70 /* error information */
71 int err
; /* error code */
72 char *msg
; /* error message */
74 int init
; /* is the iniflate stream initialized */
75 lzma_stream strm
; /* stream structure in-place (not a pointer) */
76 char padding1
[32]; /* padding allowing to cope with possible
77 extensions of above structure without
78 too much side effect */
79 #ifdef LIBXML_ZLIB_ENABLED
80 /* zlib inflate or deflate stream */
81 z_stream zstrm
; /* stream structure in-place (not a pointer) */
83 char padding2
[32]; /* padding allowing to cope with possible
84 extensions of above structure without
85 too much side effect */
86 } xz_state
, *xz_statep
;
89 xz_error(xz_statep state
, int err
, const char *msg
)
91 /* free previously allocated message and clear */
92 if (state
->msg
!= NULL
) {
93 if (state
->err
!= LZMA_MEM_ERROR
)
98 /* set error code, and if no message, then done */
103 /* for an out of memory error, save as static string */
104 if (err
== LZMA_MEM_ERROR
) {
105 state
->msg
= (char *) msg
;
109 /* construct error message with path */
111 xmlMalloc(strlen(state
->path
) + strlen(msg
) + 3)) == NULL
) {
112 state
->err
= LZMA_MEM_ERROR
;
113 state
->msg
= (char *) "out of memory";
116 strcpy(state
->msg
, state
->path
);
117 strcat(state
->msg
, ": ");
118 strcat(state
->msg
, msg
);
123 xz_reset(xz_statep state
)
125 state
->have
= 0; /* no output data available */
126 state
->eof
= 0; /* not at end of file */
127 state
->how
= LOOK
; /* look for gzip header */
128 state
->direct
= 1; /* default for empty file */
129 state
->seek
= 0; /* no seek request pending */
130 xz_error(state
, LZMA_OK
, NULL
); /* clear error */
131 state
->pos
= 0; /* no uncompressed data yet */
132 state
->strm
.avail_in
= 0; /* no input data yet */
133 #ifdef LIBXML_ZLIB_ENABLED
134 state
->zstrm
.avail_in
= 0; /* no input data yet */
139 xz_open(const char *path
, int fd
, const char *mode ATTRIBUTE_UNUSED
)
143 /* allocate xzFile structure to return */
144 state
= xmlMalloc(sizeof(xz_state
));
147 state
->size
= 0; /* no buffers allocated yet */
148 state
->want
= BUFSIZ
; /* requested buffer size */
149 state
->msg
= NULL
; /* no error message yet */
150 state
->init
= 0; /* initialization of zlib data */
152 /* save the path name for error messages */
153 state
->path
= xmlMalloc(strlen(path
) + 1);
154 if (state
->path
== NULL
) {
158 strcpy(state
->path
, path
);
160 /* open the file with the appropriate mode (or just use fd) */
161 state
->fd
= fd
!= -1 ? fd
: open(path
,
169 if (state
->fd
== -1) {
170 xmlFree(state
->path
);
175 /* save the current position for rewinding (only if reading) */
176 state
->start
= lseek(state
->fd
, 0, SEEK_CUR
);
177 if (state
->start
== (uint64_t) - 1)
180 /* initialize stream */
184 return (xzFile
) state
;
188 xz_compressed(xzFile f
) {
193 state
= (xz_statep
) f
;
194 if (state
->init
<= 0)
197 switch (state
->how
) {
208 __libxml2_xzopen(const char *path
, const char *mode
)
210 return xz_open(path
, -1, mode
);
214 __libxml2_xzcompressed(xzFile f
) {
215 return xz_compressed(f
);
219 __libxml2_xzdopen(int fd
, const char *mode
)
221 char *path
; /* identifier for error messages */
224 if (fd
== -1 || (path
= xmlMalloc(7 + 3 * sizeof(int))) == NULL
)
226 sprintf(path
, "<fd:%d>", fd
); /* for debugging */
227 xz
= xz_open(path
, fd
, mode
);
233 xz_load(xz_statep state
, unsigned char *buf
, unsigned int len
,
240 ret
= read(state
->fd
, buf
+ *have
, len
- *have
);
244 } while (*have
< len
);
246 xz_error(state
, -1, strerror(errno
));
255 xz_avail(xz_statep state
)
257 lzma_stream
*strm
= &(state
->strm
);
259 if (state
->err
!= LZMA_OK
)
261 if (state
->eof
== 0) {
262 /* avail_in is size_t, which is not necessary sizeof(unsigned) */
263 unsigned tmp
= strm
->avail_in
;
265 if (xz_load(state
, state
->in
, state
->size
, &tmp
) == -1) {
266 strm
->avail_in
= tmp
;
269 strm
->avail_in
= tmp
;
270 strm
->next_in
= state
->in
;
275 #ifdef LIBXML_ZLIB_ENABLED
277 xz_avail_zstrm(xz_statep state
)
280 state
->strm
.avail_in
= state
->zstrm
.avail_in
;
281 state
->strm
.next_in
= state
->zstrm
.next_in
;
282 ret
= xz_avail(state
);
283 state
->zstrm
.avail_in
= (uInt
) state
->strm
.avail_in
;
284 state
->zstrm
.next_in
= (Bytef
*) state
->strm
.next_in
;
290 is_format_xz(xz_statep state
)
292 lzma_stream
*strm
= &(state
->strm
);
294 return strm
->avail_in
>= 6 && memcmp(state
->in
, "\3757zXZ", 6) == 0;
298 is_format_lzma(xz_statep state
)
300 lzma_stream
*strm
= &(state
->strm
);
303 lzma_options_lzma
*opt
;
305 uint64_t uncompressed_size
;
308 if (strm
->avail_in
< 13)
311 filter
.id
= LZMA_FILTER_LZMA1
;
312 if (lzma_properties_decode(&filter
, NULL
, state
->in
, 5) != LZMA_OK
)
315 opt
= filter
.options
;
316 dict_size
= opt
->dict_size
;
317 free(opt
); /* we can't use xmlFree on a string returned by zlib */
319 /* A hack to ditch tons of false positives: We allow only dictionary
320 * sizes that are 2^n or 2^n + 2^(n-1) or UINT32_MAX. LZMA_Alone
321 * created only files with 2^n, but accepts any dictionary size.
322 * If someone complains, this will be reconsidered.
324 if (dict_size
!= UINT32_MAX
) {
325 uint32_t d
= dict_size
- 1;
333 if (d
!= dict_size
|| dict_size
== 0)
337 /* Another hack to ditch false positives: Assume that if the
338 * uncompressed size is known, it must be less than 256 GiB.
339 * Again, if someone complains, this will be reconsidered.
341 uncompressed_size
= 0;
342 for (i
= 0; i
< 8; ++i
)
343 uncompressed_size
|= (uint64_t) (state
->in
[5 + i
]) << (i
* 8);
345 if (uncompressed_size
!= UINT64_MAX
346 && uncompressed_size
> (UINT64_C(1) << 38))
352 #ifdef LIBXML_ZLIB_ENABLED
354 /* Get next byte from input, or -1 if end or error. */
355 #define NEXT() ((strm->avail_in == 0 && xz_avail(state) == -1) ? -1 : \
356 (strm->avail_in == 0 ? -1 : \
357 (strm->avail_in--, *(strm->next_in)++)))
358 /* Same thing, but from zstrm */
359 #define NEXTZ() ((strm->avail_in == 0 && xz_avail_zstrm(state) == -1) ? -1 : \
360 (strm->avail_in == 0 ? -1 : \
361 (strm->avail_in--, *(strm->next_in)++)))
363 /* Get a four-byte little-endian integer and return 0 on success and the value
364 in *ret. Otherwise -1 is returned and *ret is not modified. */
366 gz_next4(xz_statep state
, unsigned long *ret
)
370 z_streamp strm
= &(state
->zstrm
);
373 val
+= (unsigned) NEXTZ() << 8;
374 val
+= (unsigned long) NEXTZ() << 16;
378 val
+= (unsigned long) ch
<< 24;
385 xz_head(xz_statep state
)
387 lzma_stream
*strm
= &(state
->strm
);
388 lzma_stream init
= LZMA_STREAM_INIT
;
392 /* allocate read buffers and inflate memory */
393 if (state
->size
== 0) {
394 /* allocate buffers */
395 state
->in
= xmlMalloc(state
->want
);
396 state
->out
= xmlMalloc(state
->want
<< 1);
397 if (state
->in
== NULL
|| state
->out
== NULL
) {
398 if (state
->out
!= NULL
)
400 if (state
->in
!= NULL
)
402 xz_error(state
, LZMA_MEM_ERROR
, "out of memory");
405 state
->size
= state
->want
;
407 /* allocate decoder memory */
409 state
->strm
.avail_in
= 0;
410 state
->strm
.next_in
= NULL
;
411 if (lzma_auto_decoder(&state
->strm
, 100000000, 0) != LZMA_OK
) {
415 xz_error(state
, LZMA_MEM_ERROR
, "out of memory");
418 #ifdef LIBXML_ZLIB_ENABLED
419 /* allocate inflate memory */
420 state
->zstrm
.zalloc
= Z_NULL
;
421 state
->zstrm
.zfree
= Z_NULL
;
422 state
->zstrm
.opaque
= Z_NULL
;
423 state
->zstrm
.avail_in
= 0;
424 state
->zstrm
.next_in
= Z_NULL
;
425 if (state
->init
== 0) {
426 if (inflateInit2(&(state
->zstrm
), -15) != Z_OK
) {/* raw inflate */
430 xz_error(state
, LZMA_MEM_ERROR
, "out of memory");
438 /* get some data in the input buffer */
439 if (strm
->avail_in
== 0) {
440 if (xz_avail(state
) == -1)
442 if (strm
->avail_in
== 0)
446 /* look for the xz magic header bytes */
447 if (is_format_xz(state
) || is_format_lzma(state
)) {
452 #ifdef LIBXML_ZLIB_ENABLED
453 /* look for the gzip magic header bytes 31 and 139 */
454 if (strm
->next_in
[0] == 31) {
457 if (strm
->avail_in
== 0 && xz_avail(state
) == -1)
459 if (strm
->avail_in
&& strm
->next_in
[0] == 139) {
460 /* we have a gzip header, woo hoo! */
464 /* skip rest of header */
465 if (NEXT() != 8) { /* compression method */
466 xz_error(state
, LZMA_DATA_ERROR
,
467 "unknown compression method");
471 if (flags
& 0xe0) { /* reserved flag bits */
472 xz_error(state
, LZMA_DATA_ERROR
,
473 "unknown header flags set");
476 NEXT(); /* modification time */
480 NEXT(); /* extra flags */
481 NEXT(); /* operating system */
482 if (flags
& 4) { /* extra field */
483 len
= (unsigned) NEXT();
484 len
+= (unsigned) NEXT() << 8;
489 if (flags
& 8) /* file name */
491 if (flags
& 16) /* comment */
493 if (flags
& 2) { /* header crc */
497 /* an unexpected end of file is not checked for here -- it will be
498 * noticed on the first request for uncompressed data */
500 /* set up for decompression */
501 inflateReset(&state
->zstrm
);
502 state
->zstrm
.adler
= crc32(0L, Z_NULL
, 0);
507 /* not a gzip file -- save first byte (31) and fall to raw i/o */
514 /* doing raw i/o, save start of raw data for seeking, copy any leftover
515 * input to output -- this assumes that the output buffer is larger than
516 * the input buffer, which also assures space for gzungetc() */
517 state
->raw
= state
->pos
;
518 state
->next
= state
->out
;
519 if (strm
->avail_in
) {
520 memcpy(state
->next
+ state
->have
, strm
->next_in
, strm
->avail_in
);
521 state
->have
+= strm
->avail_in
;
530 xz_decomp(xz_statep state
)
534 unsigned long crc
, len
;
535 lzma_stream
*strm
= &(state
->strm
);
537 lzma_action action
= LZMA_RUN
;
539 /* fill output buffer up to end of deflate stream */
540 had
= strm
->avail_out
;
542 /* get more input for inflate() */
543 if (strm
->avail_in
== 0 && xz_avail(state
) == -1)
545 if (strm
->avail_in
== 0) {
546 xz_error(state
, LZMA_DATA_ERROR
, "unexpected end of file");
550 action
= LZMA_FINISH
;
552 /* decompress and handle errors */
553 #ifdef LIBXML_ZLIB_ENABLED
554 if (state
->how
== GZIP
) {
555 state
->zstrm
.avail_in
= (uInt
) state
->strm
.avail_in
;
556 state
->zstrm
.next_in
= (Bytef
*) state
->strm
.next_in
;
557 state
->zstrm
.avail_out
= (uInt
) state
->strm
.avail_out
;
558 state
->zstrm
.next_out
= (Bytef
*) state
->strm
.next_out
;
559 ret
= inflate(&state
->zstrm
, Z_NO_FLUSH
);
560 if (ret
== Z_STREAM_ERROR
|| ret
== Z_NEED_DICT
) {
561 xz_error(state
, Z_STREAM_ERROR
,
562 "internal error: inflate stream corrupt");
566 * FIXME: Remapping a couple of error codes and falling through
567 * to the LZMA error handling looks fragile.
569 if (ret
== Z_MEM_ERROR
)
570 ret
= LZMA_MEM_ERROR
;
571 if (ret
== Z_DATA_ERROR
)
572 ret
= LZMA_DATA_ERROR
;
573 if (ret
== Z_STREAM_END
)
574 ret
= LZMA_STREAM_END
;
575 state
->strm
.avail_in
= state
->zstrm
.avail_in
;
576 state
->strm
.next_in
= state
->zstrm
.next_in
;
577 state
->strm
.avail_out
= state
->zstrm
.avail_out
;
578 state
->strm
.next_out
= state
->zstrm
.next_out
;
579 } else /* state->how == LZMA */
581 ret
= lzma_code(strm
, action
);
582 if (ret
== LZMA_MEM_ERROR
) {
583 xz_error(state
, LZMA_MEM_ERROR
, "out of memory");
586 if (ret
== LZMA_DATA_ERROR
) {
587 xz_error(state
, LZMA_DATA_ERROR
, "compressed data error");
590 if (ret
== LZMA_PROG_ERROR
) {
591 xz_error(state
, LZMA_PROG_ERROR
, "compression error");
594 if ((state
->how
!= GZIP
) &&
595 (ret
!= LZMA_OK
) && (ret
!= LZMA_STREAM_END
)) {
596 xz_error(state
, ret
, "lzma error");
599 } while (strm
->avail_out
&& ret
!= LZMA_STREAM_END
);
601 /* update available output and crc check value */
602 state
->have
= had
- strm
->avail_out
;
603 state
->next
= strm
->next_out
- state
->have
;
604 #ifdef LIBXML_ZLIB_ENABLED
606 crc32(state
->zstrm
.adler
, state
->next
, state
->have
);
609 if (ret
== LZMA_STREAM_END
) {
610 #ifdef LIBXML_ZLIB_ENABLED
611 if (state
->how
== GZIP
) {
612 if (gz_next4(state
, &crc
) == -1 || gz_next4(state
, &len
) == -1) {
613 xz_error(state
, LZMA_DATA_ERROR
, "unexpected end of file");
616 if (crc
!= state
->zstrm
.adler
) {
617 xz_error(state
, LZMA_DATA_ERROR
, "incorrect data check");
620 if (len
!= (state
->zstrm
.total_out
& 0xffffffffL
)) {
621 xz_error(state
, LZMA_DATA_ERROR
, "incorrect length check");
624 state
->strm
.avail_in
= 0;
625 state
->strm
.next_in
= NULL
;
626 state
->strm
.avail_out
= 0;
627 state
->strm
.next_out
= NULL
;
630 if (strm
->avail_in
!= 0 || !state
->eof
) {
631 xz_error(state
, LZMA_DATA_ERROR
, "trailing garbage");
634 state
->how
= LOOK
; /* ready for next stream, once have is 0 (leave
635 * state->direct unchanged to remember how) */
638 /* good decompression */
643 xz_make(xz_statep state
)
645 lzma_stream
*strm
= &(state
->strm
);
647 if (state
->how
== LOOK
) { /* look for lzma / gzip header */
648 if (xz_head(state
) == -1)
650 if (state
->have
) /* got some data from xz_head() */
653 if (state
->how
== COPY
) { /* straight copy */
654 if (xz_load(state
, state
->out
, state
->size
<< 1, &(state
->have
)) ==
657 state
->next
= state
->out
;
658 } else if (state
->how
== LZMA
|| state
->how
== GZIP
) { /* decompress */
659 strm
->avail_out
= state
->size
<< 1;
660 strm
->next_out
= state
->out
;
661 if (xz_decomp(state
) == -1)
668 xz_skip(xz_statep state
, uint64_t len
)
672 /* skip over len bytes or reach end-of-file, whichever comes first */
674 /* skip over whatever is in output buffer */
676 n
= (uint64_t) state
->have
> len
?
677 (unsigned) len
: state
->have
;
684 /* output buffer empty -- return if we're at the end of the input */
685 else if (state
->eof
&& state
->strm
.avail_in
== 0)
688 /* need more data to skip -- load up output buffer */
690 /* get more output, looking for header if required */
691 if (xz_make(state
) == -1)
698 __libxml2_xzread(xzFile file
, void *buf
, unsigned len
)
704 /* get internal structure */
707 state
= (xz_statep
) file
;
708 strm
= &(state
->strm
);
710 /* check that we're reading and that there's no error */
711 if (state
->err
!= LZMA_OK
)
714 /* since an int is returned, make sure len fits in one, otherwise return
715 * with an error (this avoids the flaw in the interface) */
717 xz_error(state
, LZMA_BUF_ERROR
,
718 "requested length does not fit in int");
722 /* if len is zero, avoid unnecessary operations */
726 /* process a skip request */
729 if (xz_skip(state
, state
->skip
) == -1)
733 /* get len bytes to buf, or less than len if at the end */
736 /* first just try copying data from the output buffer */
738 n
= state
->have
> len
? len
: state
->have
;
739 memcpy(buf
, state
->next
, n
);
744 /* output buffer empty -- return if we're at the end of the input */
745 else if (state
->eof
&& strm
->avail_in
== 0)
748 /* need output data -- for small len or new stream load up our output
750 else if (state
->how
== LOOK
|| len
< (state
->size
<< 1)) {
751 /* get more output, looking for header if required */
752 if (xz_make(state
) == -1)
754 continue; /* no progress yet -- go back to memcpy() above */
755 /* the copy above assures that we will leave with space in the
756 * output buffer, allowing at least one gzungetc() to succeed */
759 /* large len -- read directly into user buffer */
760 else if (state
->how
== COPY
) { /* read directly */
761 if (xz_load(state
, buf
, len
, &n
) == -1)
765 /* large len -- decompress directly into user buffer */
766 else { /* state->how == LZMA */
767 strm
->avail_out
= len
;
768 strm
->next_out
= buf
;
769 if (xz_decomp(state
) == -1)
775 /* update progress */
777 buf
= (char *) buf
+ n
;
782 /* return number of bytes read into user buffer (will fit in int) */
787 __libxml2_xzclose(xzFile file
)
792 /* get internal structure */
794 return LZMA_DATA_ERROR
;
795 state
= (xz_statep
) file
;
797 /* free memory and close file */
799 lzma_end(&(state
->strm
));
800 #ifdef LIBXML_ZLIB_ENABLED
801 if (state
->init
== 1)
802 inflateEnd(&(state
->zstrm
));
808 xmlFree(state
->path
);
809 if ((state
->msg
!= NULL
) && (state
->err
!= LZMA_MEM_ERROR
))
811 ret
= close(state
->fd
);
813 return ret
? ret
: LZMA_OK
;
815 #endif /* LIBXML_LZMA_ENABLED */