1NOTE: this patch is for _gzip_! 2 3This is pending for actual inclusion in gzip. It is currently being 4tried out in the default gzip for Debian Sarge, and may go into the 5upstream gzip at somepoint in the not-too-distant future. 6 7--- gzip-1.3.5/deflate.c 1999-10-07 23:46:28 -0700 8+++ rsyncable/deflate.c 2005-02-05 09:40:33 -0800 9@@ -122,6 +122,14 @@ 10 #endif 11 /* Matches of length 3 are discarded if their distance exceeds TOO_FAR */ 12 13+#ifndef RSYNC_WIN 14+# define RSYNC_WIN 4096 15+#endif 16+/* Size of rsync window, must be < MAX_DIST */ 17+ 18+#define RSYNC_SUM_MATCH(sum) ((sum) % RSYNC_WIN == 0) 19+/* Whether window sum matches magic value */ 20+ 21 /* =========================================================================== 22 * Local data used by the "longest match" routines. 23 */ 24@@ -203,6 +211,8 @@ 25 unsigned near good_match; 26 /* Use a faster search when the previous match is longer than this */ 27 28+local ulg rsync_sum; /* rolling sum of rsync window */ 29+local ulg rsync_chunk_end; /* next rsync sequence point */ 30 31 /* Values for max_lazy_match, good_match and max_chain_length, depending on 32 * the desired pack level (0..9). The values given below have been tuned to 33@@ -301,6 +311,10 @@ 34 #endif 35 /* prev will be initialized on the fly */ 36 37+ /* rsync params */ 38+ rsync_chunk_end = 0xFFFFFFFFUL; 39+ rsync_sum = 0; 40+ 41 /* Set the default configuration parameters: 42 */ 43 max_lazy_match = configuration_table[pack_level].max_lazy; 44@@ -537,6 +551,8 @@ 45 memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE); 46 match_start -= WSIZE; 47 strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */ 48+ if (rsync_chunk_end != 0xFFFFFFFFUL) 49+ rsync_chunk_end -= WSIZE; 50 51 block_start -= (long) WSIZE; 52 53@@ -564,13 +580,46 @@ 54 } 55 } 56 57+local void rsync_roll(start, num) 58+ unsigned start; 59+ unsigned num; 60+{ 61+ unsigned i; 62+ 63+ if (start < RSYNC_WIN) { 64+ /* before window fills. */ 65+ for (i = start; i < RSYNC_WIN; i++) { 66+ if (i == start + num) return; 67+ rsync_sum += (ulg)window[i]; 68+ } 69+ num -= (RSYNC_WIN - start); 70+ start = RSYNC_WIN; 71+ } 72+ 73+ /* buffer after window full */ 74+ for (i = start; i < start+num; i++) { 75+ /* New character in */ 76+ rsync_sum += (ulg)window[i]; 77+ /* Old character out */ 78+ rsync_sum -= (ulg)window[i - RSYNC_WIN]; 79+ if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum)) 80+ rsync_chunk_end = i; 81+ } 82+} 83+ 84+/* =========================================================================== 85+ * Set rsync_chunk_end if window sum matches magic value. 86+ */ 87+#define RSYNC_ROLL(s, n) \ 88+ do { if (rsync) rsync_roll((s), (n)); } while(0) 89+ 90 /* =========================================================================== 91 * Flush the current block, with given end-of-file flag. 92 * IN assertion: strstart is set to the end of the current match. 93 */ 94 #define FLUSH_BLOCK(eof) \ 95 flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \ 96- (char*)NULL, (long)strstart - block_start, (eof)) 97+ (char*)NULL, (long)strstart - block_start, flush-1, (eof)) 98 99 /* =========================================================================== 100 * Processes a new input file and return its compressed length. This 101@@ -581,7 +630,7 @@ 102 local off_t deflate_fast() 103 { 104 IPos hash_head; /* head of the hash chain */ 105- int flush; /* set if current block must be flushed */ 106+ int flush; /* set if current block must be flushed, 2=>and padded */ 107 unsigned match_length = 0; /* length of best match */ 108 109 prev_length = MIN_MATCH-1; 110@@ -610,6 +659,7 @@ 111 112 lookahead -= match_length; 113 114+ RSYNC_ROLL(strstart, match_length); 115 /* Insert new strings in the hash table only if the match length 116 * is not too large. This saves time but degrades compression. 117 */ 118@@ -638,9 +688,14 @@ 119 /* No match, output a literal byte */ 120 Tracevv((stderr,"%c",window[strstart])); 121 flush = ct_tally (0, window[strstart]); 122+ RSYNC_ROLL(strstart, 1); 123 lookahead--; 124 strstart++; 125 } 126+ if (rsync && strstart > rsync_chunk_end) { 127+ rsync_chunk_end = 0xFFFFFFFFUL; 128+ flush = 2; 129+ } 130 if (flush) FLUSH_BLOCK(0), block_start = strstart; 131 132 /* Make sure that we always have enough lookahead, except 133@@ -713,6 +768,7 @@ 134 */ 135 lookahead -= prev_length-1; 136 prev_length -= 2; 137+ RSYNC_ROLL(strstart, prev_length+1); 138 do { 139 strstart++; 140 INSERT_STRING(strstart, hash_head); 141@@ -725,24 +781,39 @@ 142 match_available = 0; 143 match_length = MIN_MATCH-1; 144 strstart++; 145- if (flush) FLUSH_BLOCK(0), block_start = strstart; 146 147+ if (rsync && strstart > rsync_chunk_end) { 148+ rsync_chunk_end = 0xFFFFFFFFUL; 149+ flush = 2; 150+ } 151+ if (flush) FLUSH_BLOCK(0), block_start = strstart; 152 } else if (match_available) { 153 /* If there was no match at the previous position, output a 154 * single literal. If there was a match but the current match 155 * is longer, truncate the previous match to a single literal. 156 */ 157 Tracevv((stderr,"%c",window[strstart-1])); 158- if (ct_tally (0, window[strstart-1])) { 159- FLUSH_BLOCK(0), block_start = strstart; 160- } 161+ flush = ct_tally (0, window[strstart-1]); 162+ if (rsync && strstart > rsync_chunk_end) { 163+ rsync_chunk_end = 0xFFFFFFFFUL; 164+ flush = 2; 165+ } 166+ if (flush) FLUSH_BLOCK(0), block_start = strstart; 167+ RSYNC_ROLL(strstart, 1); 168 strstart++; 169 lookahead--; 170 } else { 171 /* There is no previous match to compare with, wait for 172 * the next step to decide. 173 */ 174+ if (rsync && strstart > rsync_chunk_end) { 175+ /* Reset huffman tree */ 176+ rsync_chunk_end = 0xFFFFFFFFUL; 177+ flush = 2; 178+ FLUSH_BLOCK(0), block_start = strstart; 179+ } 180 match_available = 1; 181+ RSYNC_ROLL(strstart, 1); 182 strstart++; 183 lookahead--; 184 } 185--- gzip-1.3.5/gzip.c 2002-09-28 00:38:43 -0700 186+++ rsyncable/gzip.c 2005-02-05 09:40:33 -0800 187@@ -256,6 +256,7 @@ 188 unsigned insize; /* valid bytes in inbuf */ 189 unsigned inptr; /* index of next byte to be processed in inbuf */ 190 unsigned outcnt; /* bytes in output buffer */ 191+int rsync = 0; /* make ryncable chunks */ 192 193 struct option longopts[] = 194 { 195@@ -285,6 +286,7 @@ 196 {"best", 0, 0, '9'}, /* compress better */ 197 {"lzw", 0, 0, 'Z'}, /* make output compatible with old compress */ 198 {"bits", 1, 0, 'b'}, /* max number of bits per code (implies -Z) */ 199+ {"rsyncable", 0, 0, 'R'}, /* make rsync-friendly archive */ 200 { 0, 0, 0, 0 } 201 }; 202 203@@ -365,6 +367,7 @@ 204 " -Z --lzw produce output compatible with old compress", 205 " -b --bits maxbits max number of bits per code (implies -Z)", 206 #endif 207+ " --rsyncable Make rsync-friendly archive", 208 " file... files to (de)compress. If none given, use standard input.", 209 "Report bugs to <bug-gzip@gnu.org>.", 210 0}; 211@@ -543,6 +546,9 @@ 212 #else 213 recursive = 1; break; 214 #endif 215+ case 'R': 216+ rsync = 1; break; 217+ 218 case 'S': 219 #ifdef NO_MULTIPLE_DOTS 220 if (*optarg == '.') optarg++; 221--- gzip-1.3.5/gzip.h 2001-09-30 23:53:41 -0700 222+++ rsyncable/gzip.h 2005-02-05 09:40:33 -0800 223@@ -133,6 +133,7 @@ 224 extern unsigned insize; /* valid bytes in inbuf */ 225 extern unsigned inptr; /* index of next byte to be processed in inbuf */ 226 extern unsigned outcnt; /* bytes in output buffer */ 227+extern int rsync; /* deflate into rsyncable chunks */ 228 229 extern off_t bytes_in; /* number of input bytes */ 230 extern off_t bytes_out; /* number of output bytes */ 231@@ -281,7 +282,7 @@ 232 /* in trees.c */ 233 void ct_init OF((ush *attr, int *method)); 234 int ct_tally OF((int dist, int lc)); 235-off_t flush_block OF((char *buf, ulg stored_len, int eof)); 236+off_t flush_block OF((char *buf, ulg stored_len, int pad, int eof)); 237 238 /* in bits.c */ 239 void bi_init OF((file_t zipfile)); 240--- gzip-1.3.5/gzip.texi 2002-09-29 23:57:29 -0700 241+++ rsyncable/gzip.texi 2005-02-05 09:40:33 -0800 242@@ -334,6 +334,14 @@ 243 into the directory and compress all the files it finds there (or 244 decompress them in the case of @code{gunzip}). 245 246+@item --rsyncable 247+While compressing, synchronize the output occasionally based on the 248+input. This increases size by less than 1 percent most cases, but 249+means that the @code{rsync} program can take advantage of similarities 250+in the uncompressed input when syncronizing two files compressed with 251+this flag. @code{gunzip} cannot tell the difference between a 252+compressed file created with this option, and one created without it. 253+ 254 @item --suffix @var{suf} 255 @itemx -S @var{suf} 256 Use suffix @samp{@var{suf}} instead of @samp{.gz}. Any suffix can be 257--- gzip-1.3.5/trees.c 1999-10-05 22:00:00 -0700 258+++ rsyncable/trees.c 2005-02-05 09:40:33 -0800 259@@ -46,12 +46,13 @@ 260 * void ct_tally (int dist, int lc); 261 * Save the match info and tally the frequency counts. 262 * 263- * off_t flush_block (char *buf, ulg stored_len, int eof) 264+ * off_t flush_block (char *buf, ulg stored_len, int pad, int eof) 265 * Determine the best encoding for the current block: dynamic trees, 266 * static trees or store, and output the encoded block to the zip 267- * file. Returns the total compressed length for the file so far. 268- * 269- */ 270+ * file. If pad is set, pads the block to the next 271+ * byte. Returns the total compressed length for the file so 272+ * far. 273+ * */ 274 275 #include <config.h> 276 #include <ctype.h> 277@@ -847,9 +848,10 @@ 278 * trees or store, and output the encoded block to the zip file. This function 279 * returns the total compressed length for the file so far. 280 */ 281-off_t flush_block(buf, stored_len, eof) 282+off_t flush_block(buf, stored_len, pad, eof) 283 char *buf; /* input block, or NULL if too old */ 284 ulg stored_len; /* length of input block */ 285+ int pad; /* pad output to byte boundary */ 286 int eof; /* true if this is the last block for a file */ 287 { 288 ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ 289@@ -941,6 +943,10 @@ 290 Assert (input_len == bytes_in, "bad input size"); 291 bi_windup(); 292 compressed_len += 7; /* align on byte boundary */ 293+ } else if (pad && (compressed_len % 8) != 0) { 294+ send_bits((STORED_BLOCK<<1)+eof, 3); /* send block type */ 295+ compressed_len = (compressed_len + 3 + 7) & ~7L; 296+ copy_block(buf, 0, 1); /* with header */ 297 } 298 299 return compressed_len >> 3; 300