support compress & decompress by stream（block by block）

.cirrus.yml

@@ -9,4 +9,4 @@ task:
     - cmake --build build
     - ctest --test-dir build
     # Direct compilation without official build system
-    - cc -O2 -Wall -Werror lib/*.c lib/*/*.c programs/gzip.c programs/prog_util.c programs/tgetopt.c -o libdeflate-gzip
+    - cc -O2 -Wall -Werror lib/*.c lib/*/*.c programs/gzip.c programs/gzip_compress_by_stream.c programs/gzip_decompress_by_stream.c programs/prog_util.c programs/tgetopt.c -o libdeflate-gzip

lib/decompress_template.h

@@ -41,13 +41,25 @@
 #  define EXTRACT_VARBITS8(word, count)	((word) & BITMASK((u8)(count)))
 #endif
 
+#define _DEF_bitstream_byte_restore() do{	\
+		bitsleft = (u8)bitsleft;			\
+		SAFETY_CHECK(overread_count <= (bitsleft >> 3));	\
+		in_next -= (bitsleft >> 3) - overread_count; } while(0)
+#define _DEF_bitstream_byte_align() do{	\
+		_DEF_bitstream_byte_restore();  \
+		overread_count = 0;	\
+		bitbuf = 0;			\
+		bitsleft = 0; } while(0)
+
 static ATTRIBUTES MAYBE_UNUSED enum libdeflate_result
 FUNCNAME(struct libdeflate_decompressor * restrict d,
 	 const void * restrict in, size_t in_nbytes,
-	 void * restrict out, size_t out_nbytes_avail,
-	 size_t *actual_in_nbytes_ret, size_t *actual_out_nbytes_ret)
+	 void * restrict out, size_t in_dict_nbytes, size_t out_nbytes_avail,
+	 size_t *actual_in_nbytes_ret,size_t *actual_out_nbytes_ret,
+	 enum libdeflate_decompress_stop_by stop_type,int* is_final_block_ret)
 {
-	u8 *out_next = out;
+	enum libdeflate_result result = LIBDEFLATE_SUCCESS;
+	u8 *out_next = ((u8 *)out)+in_dict_nbytes;
 	u8 * const out_end = out_next + out_nbytes_avail;
 	u8 * const out_fastloop_end =
 		out_end - MIN(out_nbytes_avail, FASTLOOP_MAX_BYTES_WRITTEN);
@@ -57,9 +69,9 @@ FUNCNAME(struct libdeflate_decompressor * restrict d,
 	const u8 * const in_end = in_next + in_nbytes;
 	const u8 * const in_fastloop_end =
 		in_end - MIN(in_nbytes, FASTLOOP_MAX_BYTES_READ);
-	bitbuf_t bitbuf = 0;
+	bitbuf_t bitbuf = d->bitbuf_back;
 	bitbuf_t saved_bitbuf;
-	u32 bitsleft = 0;
+	u32 bitsleft = d->bitsleft_back;
 	size_t overread_count = 0;
 
 	bool is_final_block;
@@ -69,6 +81,8 @@ FUNCNAME(struct libdeflate_decompressor * restrict d,
 	bitbuf_t litlen_tablemask;
 	u32 entry;
 
+	_decompress_block_init(d);
+
 next_block:
 	/* Starting to read the next block */
 	;
@@ -261,12 +275,7 @@ FUNCNAME(struct libdeflate_decompressor * restrict d,
 		 * that have been refilled but not actually consumed yet (not
 		 * counting overread bytes, which don't increment 'in_next').
 		 */
-		bitsleft = (u8)bitsleft;
-		SAFETY_CHECK(overread_count <= (bitsleft >> 3));
-		in_next -= (bitsleft >> 3) - overread_count;
-		overread_count = 0;
-		bitbuf = 0;
-		bitsleft = 0;
+		_DEF_bitstream_byte_align();
 
 		SAFETY_CHECK(in_end - in_next >= 4);
 		len = get_unaligned_le16(in_next);
@@ -275,7 +284,7 @@ FUNCNAME(struct libdeflate_decompressor * restrict d,
 
 		SAFETY_CHECK(len == (u16)~nlen);
 		if (unlikely(len > out_end - out_next))
-			return LIBDEFLATE_INSUFFICIENT_SPACE;
+			goto _on_insufficient_space;
 		SAFETY_CHECK(len <= in_end - in_next);
 
 		memcpy(out_next, in_next, len);
@@ -698,15 +707,15 @@ FUNCNAME(struct libdeflate_decompressor * restrict d,
 		length = entry >> 16;
 		if (entry & HUFFDEC_LITERAL) {
 			if (unlikely(out_next == out_end))
-				return LIBDEFLATE_INSUFFICIENT_SPACE;
+				goto _on_insufficient_space;
 			*out_next++ = length;
 			continue;
 		}
 		if (unlikely(entry & HUFFDEC_END_OF_BLOCK))
 			goto block_done;
 		length += EXTRACT_VARBITS8(saved_bitbuf, entry) >> (u8)(entry >> 8);
 		if (unlikely(length > out_end - out_next))
-			return LIBDEFLATE_INSUFFICIENT_SPACE;
+			goto _on_insufficient_space;
 
 		if (!CAN_CONSUME(LENGTH_MAXBITS + OFFSET_MAXBITS))
 			REFILL_BITS();
@@ -739,36 +748,62 @@ FUNCNAME(struct libdeflate_decompressor * restrict d,
 
 block_done:
 	/* Finished decoding a block */
-
-	if (!is_final_block)
-		goto next_block;
+	if (is_final_block)
+		_DEF_bitstream_byte_align();
+
+	switch (stop_type){
+		case LIBDEFLATE_STOP_BY_FINAL_BLOCK:{
+			if (!is_final_block)
+				goto next_block;
+		} break;
+		case LIBDEFLATE_STOP_BY_ANY_BLOCK:{
+			// stop, not next block
+		} break;
+		case LIBDEFLATE_STOP_BY_ANY_BLOCK_AND_FULL_INPUT:{
+			if (in_next-((((u8)bitsleft)>>3)-overread_count)<in_end)
+				goto next_block;
+		} break;
+		case LIBDEFLATE_STOP_BY_ANY_BLOCK_AND_FULL_OUTPUT:{
+			if (out_next<out_end)
+				goto next_block;
+		} break;
+		case LIBDEFLATE_STOP_BY_ANY_BLOCK_AND_FULL_OUTPUT_AND_IN_BYTE_ALIGN:{
+			if ((out_next<out_end)|((bitsleft&7)!=0))
+				goto next_block;
+		} break;
+	}
 
 	/* That was the last block. */
+	if (is_final_block_ret)
+		*is_final_block_ret=is_final_block;
+	if (!is_final_block){
+		_DEF_bitstream_byte_restore();
+		//backup for next block
+		d->bitsleft_back=bitsleft&7;
+		d->bitbuf_back=bitbuf&((1<<(bitsleft&7))-1);
+	}
 
-	bitsleft = (u8)bitsleft;
-
-	/*
-	 * If any of the implicit appended zero bytes were consumed (not just
-	 * refilled) before hitting end of stream, then the data is bad.
-	 */
-	SAFETY_CHECK(overread_count <= (bitsleft >> 3));
-
+_on_return:
 	/* Optionally return the actual number of bytes consumed. */
 	if (actual_in_nbytes_ret) {
-		/* Don't count bytes that were refilled but not consumed. */
-		in_next -= (bitsleft >> 3) - overread_count;
-
 		*actual_in_nbytes_ret = in_next - (u8 *)in;
 	}
 
 	/* Optionally return the actual number of bytes written. */
 	if (actual_out_nbytes_ret) {
-		*actual_out_nbytes_ret = out_next - (u8 *)out;
+		*actual_out_nbytes_ret = out_next - (((u8 *)out)+in_dict_nbytes);
 	} else {
-		if (out_next != out_end)
-			return LIBDEFLATE_SHORT_OUTPUT;
+		if ( (out_next != out_end) && (result==LIBDEFLATE_SUCCESS) )
+			result = LIBDEFLATE_SHORT_OUTPUT;
 	}
-	return LIBDEFLATE_SUCCESS;
+	return result;
+
+_on_bad_data:
+	result=LIBDEFLATE_BAD_DATA;
+	goto _on_return;
+_on_insufficient_space:
+	result=LIBDEFLATE_INSUFFICIENT_SPACE;
+	goto _on_return;
 }
 
 #undef FUNCNAME