1 #! /bin/sh /usr/share/dpatch/dpatch-run
2 ## 16_gzip.dpatch by Wil Mahan <wmahan@users.sourceforge.net>
4 ## All lines beginning with `## DP:' are a description of the patch.
5 ## DP: Add deflate-filter support to privoxy, see sf bug id 895531
6 ## DP: Adapted from privoxy 3.1 to 3.0.x by Roland Rosenfeld.
7 ## DP: This requires --enable-zlib as configure option in debian/rules
8 ## DP: and a build dependeny on zlib1g-dev.
11 diff -urNad privoxy~/actionlist.h privoxy/actionlist.h
12 --- privoxy~/actionlist.h
13 +++ privoxy/actionlist.h
15 DEFINE_ACTION_STRING ("deanimate-gifs", ACTION_DEANIMATE, ACTION_STRING_DEANIMATE)
16 DEFINE_CGI_PARAM_RADIO ("deanimate-gifs", ACTION_DEANIMATE, ACTION_STRING_DEANIMATE, "first", 0)
17 DEFINE_CGI_PARAM_RADIO ("deanimate-gifs", ACTION_DEANIMATE, ACTION_STRING_DEANIMATE, "last", 1)
18 +DEFINE_ACTION_BOOL ("decompress-from-server", ACTION_DECOMPRESS_IN)
19 DEFINE_ACTION_BOOL ("downgrade-http-version", ACTION_DOWNGRADE)
20 DEFINE_ACTION_BOOL ("fast-redirects", ACTION_FAST_REDIRECTS)
21 DEFINE_ACTION_MULTI ("filter", ACTION_MULTI_FILTER)
22 diff -urNad privoxy~/configure.in privoxy/configure.in
23 --- privoxy~/configure.in
24 +++ privoxy/configure.in
25 @@ -1234,6 +1234,20 @@
26 libpcrs is available],
27 [ if test $enableval = "no"; then have_pcrs=no; fi ])
30 +[ --enable-zlib Use the zlib library to allow compressing or
31 + decompressing data on the fly.],
32 +[enableval2=$enableval],
34 +if test $enableval2 = yes; then
35 + AC_CHECK_LIB(z, zlibVersion, , [
36 + AC_MSG_ERROR([Unable to find a copy of zlib. The zlib library
37 +is necessary to enable compresion support. ])
39 + AC_DEFINE(FEATURE_ZLIB,1,
40 + [ Define to 1 to use compression through the zlib library. ])
44 # If we have libpcre and either we also have pcreposix or
45 # we don't need pcreposix, then link pcre dynamically; else
46 diff -urNad privoxy~/default.action.master privoxy/default.action.master
47 --- privoxy~/default.action.master
48 +++ privoxy/default.action.master
53 -+prevent-compression \
54 +-prevent-compression \
57 +session-cookies-only \
58 diff -urNad privoxy~/filters.c privoxy/filters.c
59 --- privoxy~/filters.c
61 @@ -1325,6 +1325,38 @@
66 + /* If the body has a compressed transfer-encoding, uncompress
67 + * it first, adjusting size and iob->eod. Note that
68 + * decompression occurs after de-chunking.
70 + if (csp->content_type & CT_GZIP || csp->content_type & CT_DEFLATE)
72 + /* Notice that we at least tried to decompress. */
73 + if (JB_ERR_OK != decompress_iob(csp))
75 + /* We failed to decompress the data; there's no point
76 + * in continuing since we can't filter. This is
77 + * slightly tricky because we need to remember not to
78 + * modify the Content-Encoding header later; using
79 + * CT_TABOO flag is a kludge for this purpose.
81 + csp->content_type |= CT_TABOO;
84 + log_error(LOG_LEVEL_RE_FILTER, "Decompressing successful");
86 + /* Decompression gives us a completely new iob, so we
89 + size = csp->iob->eod - csp->iob->cur;
90 + old = csp->iob->cur;
92 + csp->flags |= CSP_FLAG_MODIFIED;
98 * If the body has a "chunked" transfer-encoding,
99 * get rid of it first, adjusting size and iob->eod
100 diff -urNad privoxy~/jcc.c privoxy/jcc.c
114 @@ -1609,6 +1613,8 @@
116 if ((csp->content_type & CT_TEXT) && /* It's a text / * MIME-Type */
117 !http->ssl && /* We talk plaintext */
118 + !(csp->content_type & CT_GZIP) &&
119 + !(csp->content_type & CT_DEFLATE) &&
120 block_popups) /* Policy allows */
122 block_popups_now = 1;
123 diff -urNad privoxy~/parsers.c privoxy/parsers.c
124 --- privoxy~/parsers.c
125 +++ privoxy/parsers.c
134 #if !defined(_WIN32) && !defined(__OS2__)
137 @@ -632,6 +636,281 @@
142 +/*********************************************************************
144 + * Function : decompress_iob
146 + * Description : Decompress buffered page, expanding the
147 + * buffer as necessary. csp->iob->cur
148 + * should point to the the beginning of the
149 + * compressed data block.
152 + * 1 : csp = Current client state (buffers, headers, etc...)
154 + * Returns : JB_ERR_OK on success, JB_ERR_MEMORY if out-of-memory
155 + * limit reached, JB_ERR_COMPRESS if error decompressing
158 + *********************************************************************/
159 +jb_err decompress_iob(struct client_state *csp)
161 + char *buf; /* new, uncompressed buffer */
162 + int bufsize = csp->iob->size; /* allocated size of the new buffer */
163 + /* Number of bytes at the beginning
164 + * of the iob that we should NOT
167 + int skip_size = csp->iob->cur - csp->iob->buf;
168 + int status; /* return status of the inflate() call */
169 + z_stream zstr; /* used by calls to zlib */
171 + /* This is to protect the parsing of gzipped data, but it should(?)
172 + * be valid for deflated data also.
176 + log_error (LOG_LEVEL_ERROR, "Buffer too small decompressing iob");
177 + return JB_ERR_COMPRESS;
180 + if (csp->content_type & CT_GZIP)
182 + /* Our task is slightly complicated by the facts that data
183 + * compressed by gzip does not include a zlib header, and
184 + * that there is no easily accessible interface in zlib to
185 + * handle a gzip header. We strip off the gzip header by
186 + * hand, and later inform zlib not to expect a header.
189 + /* Strip off the gzip header. Please see RFC 1952 for more
190 + * explanation of the appropriate fields.
192 + if ((*csp->iob->cur++ != (char)0x1f)
193 + || (*csp->iob->cur++ != (char)0x8b)
194 + || (*csp->iob->cur++ != Z_DEFLATED))
196 + log_error (LOG_LEVEL_ERROR,
197 + "Invalid gzip header when decompressing");
198 + return JB_ERR_COMPRESS;
201 + int flags = *csp->iob->cur++;
204 + /* The gzip header has reserved bits set; bail out. */
205 + log_error (LOG_LEVEL_ERROR,
206 + "Invalid gzip header when decompressing");
207 + return JB_ERR_COMPRESS;
209 + csp->iob->cur += 6;
211 + /* Skip extra fields if necessary. */
214 + /* Skip a given number of bytes, specified as a 16-bit
215 + * little-endian value.
217 + csp->iob->cur += *csp->iob->cur++ + (*csp->iob->cur++ << 8);
220 + /* Skip the filename if necessary. */
223 + /* A null-terminated string follows. */
224 + while (*csp->iob->cur++);
227 + /* Skip the comment if necessary. */
230 + while (*csp->iob->cur++);
233 + /* Skip the CRC if necessary. */
236 + csp->iob->cur += 2;
240 + else if (csp->content_type & CT_DEFLATE)
242 + log_error (LOG_LEVEL_INFO, "Decompressing deflated iob: %d", *csp->iob->cur);
243 + /* In theory (that is, according to RFC 1950), deflate-compressed
244 + * data should begin with a two-byte zlib header and have an
245 + * adler32 checksum at the end. It seems that in practice the
246 + * only the raw compressed data is sent. Note that this means that
247 + * we are not RFC 1950-compliant here, but the advantage is that
248 + * this actually works. :)
250 + * We add a dummy null byte to tell zlib where the data ends,
251 + * and later inform it not to expect a header.
253 + * Fortunately, add_to_iob() has thoughtfully null-terminated
254 + * the buffer; we can just increment the end pointer to include
261 + log_error (LOG_LEVEL_ERROR,
262 + "Unable to determine compression format for decompression");
263 + return JB_ERR_COMPRESS;
266 + /* Set up the fields required by zlib. */
267 + zstr.next_in = csp->iob->cur;
268 + zstr.avail_in = csp->iob->eod - csp->iob->cur;
269 + zstr.zalloc = Z_NULL;
270 + zstr.zfree = Z_NULL;
271 + zstr.opaque = Z_NULL;
273 + /* Passing -MAX_WBITS to inflateInit2 tells the library
274 + * that there is no zlib header.
276 + if (inflateInit2 (&zstr, -MAX_WBITS) != Z_OK)
278 + log_error (LOG_LEVEL_ERROR,
279 + "Error initializing decompression");
280 + return JB_ERR_COMPRESS;
283 + /* Next, we allocate new storage for the inflated data.
284 + * We don't modify the existing iob yet, so in case there
285 + * is error in decompression we can recover gracefully.
287 + buf = zalloc (bufsize);
290 + log_error (LOG_LEVEL_ERROR,
291 + "Out of memory decompressing iob");
292 + return JB_ERR_MEMORY;
295 + assert(bufsize >= skip_size);
296 + memcpy(buf, csp->iob->buf, skip_size);
297 + zstr.avail_out = bufsize - skip_size;
298 + zstr.next_out = buf + skip_size;
300 + /* Try to decompress the whole stream in one shot. */
301 + while (Z_BUF_ERROR == (status = inflate(&zstr, Z_FINISH)))
303 + /* We need to allocate more memory for the output buffer. */
305 + char *tmpbuf; /* used for realloc'ing the buffer */
306 + int oldbufsize = bufsize; /* keep track of the old bufsize */
308 + /* If zlib wants more data then there's a problem, because
309 + * the complete compressed file should have been buffered.
311 + if (0 == zstr.avail_in)
313 + log_error(LOG_LEVEL_ERROR,
314 + "Unexpected end of compressed iob");
315 + return JB_ERR_COMPRESS;
318 + /* If we tried the limit and still didn't have enough
319 + * memory, just give up.
321 + if (bufsize == csp->config->buffer_limit)
323 + log_error(LOG_LEVEL_ERROR, "Out of memory decompressing iob");
324 + return JB_ERR_MEMORY;
327 + /* Try doubling the buffer size each time. */
330 + /* Don't exceed the buffer limit. */
331 + if (bufsize > csp->config->buffer_limit)
333 + bufsize = csp->config->buffer_limit;
336 + /* Try to allocate the new buffer. */
337 + tmpbuf = realloc(buf, bufsize);
338 + if (NULL == tmpbuf)
340 + log_error(LOG_LEVEL_ERROR, "Out of memory decompressing iob");
342 + return JB_ERR_MEMORY;
346 + char *oldnext_out = zstr.next_out;
348 + /* Update the fields for inflate() to use the new
349 + * buffer, which may be in a different location from
352 + zstr.avail_out += bufsize - oldbufsize;
353 + zstr.next_out = tmpbuf + bufsize - zstr.avail_out;
355 + /* Compare with an uglier method of calculating these values
356 + * that doesn't require the extra oldbufsize variable.
358 + assert(zstr.avail_out ==
359 + tmpbuf + bufsize - (char *)zstr.next_out);
360 + assert((char *)zstr.next_out ==
361 + tmpbuf + ((char *)oldnext_out - buf));
362 + assert(zstr.avail_out > 0);
369 + if (status != Z_STREAM_END)
371 + /* We failed to decompress the stream. */
372 + log_error(LOG_LEVEL_ERROR,
373 + "Error in decompressing to the buffer (iob): %s",
375 + return JB_ERR_COMPRESS;
378 + /* Finally, we can actually update the iob, since the
379 + * decompression was successful. First, free the old
382 + freez(csp->iob->buf);
384 + /* Now, update the iob to use the new buffer. */
385 + csp->iob->buf = buf;
386 + csp->iob->cur = csp->iob->buf + skip_size;
387 + csp->iob->eod = zstr.next_out;
388 + csp->iob->size = bufsize;
390 + /* Make sure the new uncompressed iob obeys some minimal
391 + * consistency conditions.
393 + if ((csp->iob->buf < csp->iob->cur)
394 + && (csp->iob->cur <= csp->iob->eod)
395 + && (csp->iob->eod <= csp->iob->buf + csp->iob->size))
397 + char t = csp->iob->cur[100];
398 + csp->iob->cur[100] = 0;
399 + log_error(LOG_LEVEL_INFO,
400 + "Sucessfully decompressed: %s", csp->iob->cur);
401 + csp->iob->cur[100] = t;
406 + /* It seems that zlib did something weird. */
407 + log_error(LOG_LEVEL_ERROR,
408 + "Unexpected error decompressing the buffer (iob): %d==%d, %d>%d, %d<%d", csp->iob->cur, csp->iob->buf + skip_size, csp->iob->eod, csp->iob->buf, csp->iob->eod, csp->iob->buf + csp->iob->size);
409 + return JB_ERR_COMPRESS;
413 +#endif /* defined(FEATURE_ZLIB) */
416 /*********************************************************************
418 * Function : get_header
419 @@ -936,13 +1215,59 @@
420 *********************************************************************/
421 jb_err server_content_encoding(struct client_state *csp, char **header)
424 + if (strstr(*header, "gzip"))
427 + * If the body was modified, we have tried to
428 + * decompress it, so adjust the header if necessary.
430 + if ((csp->flags & CSP_FLAG_MODIFIED) /* we attempted to decompress */
431 + && !(csp->content_type & CT_TABOO)) /* decompression was successful */
434 + *header = strdup("Content-Encoding: identity");
435 + return (header == NULL) ? JB_ERR_MEMORY : JB_ERR_OK;
439 + csp->content_type |= CT_GZIP;
442 + else if (strstr(*header, "deflate"))
445 + * If the body was modified, we have tried to
446 + * decompress it, so adjust the header if necessary.
448 + if ((csp->flags & CSP_FLAG_MODIFIED) /* we attempted to decompress */
449 + && !(csp->content_type & CT_TABOO)) /* decompression was successful */
452 + *header = strdup("Content-Encoding: identity");
453 + return (header == NULL) ? JB_ERR_MEMORY : JB_ERR_OK;
457 + csp->content_type |= CT_DEFLATE;
460 + else if (strstr(*header, "compress"))
462 + /* We can't decompress this; therefore we can't filter
465 + csp->content_type |= CT_TABOO;
467 +#else /* !defined(FEATURE_GZIP) */
469 * Turn off pcrs and gif filtering if body compressed
471 if (strstr(*header, "gzip") || strstr(*header, "compress") || strstr(*header, "deflate"))
473 - csp->content_type = CT_TABOO;
474 + csp->content_type |= CT_TABOO;
476 +#endif /* !defined(FEATURE_GZIP) */
480 diff -urNad privoxy~/parsers.h privoxy/parsers.h
481 --- privoxy~/parsers.h
482 +++ privoxy/parsers.h
485 extern int flush_socket(jb_socket fd, struct client_state *csp);
486 extern jb_err add_to_iob(struct client_state *csp, char *buf, int n);
487 +extern jb_err decompress_iob(struct client_state *csp);
488 extern char *get_header(struct client_state *csp);
489 extern char *get_header_value(const struct list *header_list, const char *header_name);
490 extern char *sed(const struct parsers pats[], const add_header_func_ptr more_headers[], struct client_state *csp);
491 diff -urNad privoxy~/project.h privoxy/project.h
492 --- privoxy~/project.h
493 +++ privoxy/project.h
495 #define JB_ERR_PARSE 4 /**< Error parsing file */
496 #define JB_ERR_MODIFIED 5 /**< File has been modified outside of the
497 CGI actions editor. */
499 +#define JB_ERR_COMPRESS 6 /**< Error on decompression */
502 * This macro is used to free a pointer that may be NULL.
504 #define CT_TABOO 4 /**< csp->content_type bitmask:
505 DO NOT filter, irrespective of other flags. */
507 +/* Although these are not, strictly speaking, content types
508 + * (they are content encodings), it is simple to handle
511 +#define CT_GZIP 8 /**< csp->content_type bitmask:
512 + gzip-compressed data. */
513 +#define CT_DEFLATE 16 /**< csp->content_type bitmask:
514 + zlib-compressed data. */
517 * The mask which includes all actions.
520 #define ACTION_VANILLA_WAFER 0x00008000UL
521 /** Action bitmap: Limit CONNECT requests to safe ports. */
522 #define ACTION_LIMIT_CONNECT 0x00010000UL
523 +/** Action bitmap: Uncompress incoming text for filtering. */
524 +#define ACTION_DECOMPRESS_IN 0x00020000UL
526 /** Action string index: How to deanimate GIFs */
527 #define ACTION_STRING_DEANIMATE 0