1 const char parsers_rcs[] = "$Id: parsers.c,v 1.238 2011/12/31 14:47:45 fabiankeil Exp $";
2 /*********************************************************************
4 * File : $Source: /cvsroot/ijbswa/current/parsers.c,v $
6 * Purpose : Declares functions to parse/crunch headers and pages.
8 * Copyright : Written by and Copyright (C) 2001-2011 the
9 * Privoxy team. http://www.privoxy.org/
11 * Based on the Internet Junkbuster originally written
12 * by and Copyright (C) 1997 Anonymous Coders and
13 * Junkbusters Corporation. http://www.junkbusters.com
15 * This program is free software; you can redistribute it
16 * and/or modify it under the terms of the GNU General
17 * Public License as published by the Free Software
18 * Foundation; either version 2 of the License, or (at
19 * your option) any later version.
21 * This program is distributed in the hope that it will
22 * be useful, but WITHOUT ANY WARRANTY; without even the
23 * implied warranty of MERCHANTABILITY or FITNESS FOR A
24 * PARTICULAR PURPOSE. See the GNU General Public
25 * License for more details.
27 * The GNU General Public License should be included with
28 * this file. If not, you can view it at
29 * http://www.gnu.org/copyleft/gpl.html
30 * or write to the Free Software Foundation, Inc., 59
31 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
33 *********************************************************************/
40 #include <sys/types.h>
50 * Convince GNU's libc to provide a strptime prototype.
59 #define GZIP_IDENTIFIER_1 0x1f
60 #define GZIP_IDENTIFIER_2 0x8b
62 #define GZIP_FLAG_CHECKSUM 0x02
63 #define GZIP_FLAG_EXTRA_FIELDS 0x04
64 #define GZIP_FLAG_FILE_NAME 0x08
65 #define GZIP_FLAG_COMMENT 0x10
66 #define GZIP_FLAG_RESERVED_BITS 0xe0
69 #if !defined(_WIN32) && !defined(__OS2__)
75 #ifdef FEATURE_PTHREAD
77 /* jcc.h is for mutex semapores only */
78 #endif /* def FEATURE_PTHREAD */
83 #include "jbsockets.h"
93 const char parsers_h_rcs[] = PARSERS_H_VERSION;
95 /* Fix a problem with Solaris. There should be no effect on other
97 * Solaris's isspace() is a macro which uses its argument directly
98 * as an array index. Therefore we need to make sure that high-bit
99 * characters generate +ve values, and ideally we also want to make
100 * the argument match the declared parameter type of "int".
102 * Why did they write a character function that can't take a simple
103 * "char" argument? Doh!
105 #define ijb_isupper(__X) isupper((int)(unsigned char)(__X))
106 #define ijb_tolower(__X) tolower((int)(unsigned char)(__X))
108 static char *get_header_line(struct iob *iob);
109 static jb_err scan_headers(struct client_state *csp);
110 static jb_err header_tagger(struct client_state *csp, char *header);
111 static jb_err parse_header_time(const char *header_time, time_t *result);
113 static jb_err crumble (struct client_state *csp, char **header);
114 static jb_err filter_header (struct client_state *csp, char **header);
115 static jb_err client_connection (struct client_state *csp, char **header);
116 static jb_err client_referrer (struct client_state *csp, char **header);
117 static jb_err client_uagent (struct client_state *csp, char **header);
118 static jb_err client_ua (struct client_state *csp, char **header);
119 static jb_err client_from (struct client_state *csp, char **header);
120 static jb_err client_send_cookie (struct client_state *csp, char **header);
121 static jb_err client_x_forwarded (struct client_state *csp, char **header);
122 static jb_err client_accept_encoding (struct client_state *csp, char **header);
123 static jb_err client_te (struct client_state *csp, char **header);
124 static jb_err client_max_forwards (struct client_state *csp, char **header);
125 static jb_err client_host (struct client_state *csp, char **header);
126 static jb_err client_if_modified_since (struct client_state *csp, char **header);
127 static jb_err client_accept_language (struct client_state *csp, char **header);
128 static jb_err client_if_none_match (struct client_state *csp, char **header);
129 static jb_err crunch_client_header (struct client_state *csp, char **header);
130 static jb_err client_x_filter (struct client_state *csp, char **header);
131 static jb_err client_range (struct client_state *csp, char **header);
132 static jb_err server_set_cookie (struct client_state *csp, char **header);
133 static jb_err server_connection (struct client_state *csp, char **header);
134 static jb_err server_content_type (struct client_state *csp, char **header);
135 static jb_err server_adjust_content_length(struct client_state *csp, char **header);
136 static jb_err server_content_md5 (struct client_state *csp, char **header);
137 static jb_err server_content_encoding (struct client_state *csp, char **header);
138 static jb_err server_transfer_coding (struct client_state *csp, char **header);
139 static jb_err server_http (struct client_state *csp, char **header);
140 static jb_err crunch_server_header (struct client_state *csp, char **header);
141 static jb_err server_last_modified (struct client_state *csp, char **header);
142 static jb_err server_content_disposition(struct client_state *csp, char **header);
144 static jb_err server_adjust_content_encoding(struct client_state *csp, char **header);
147 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
148 static jb_err server_save_content_length(struct client_state *csp, char **header);
149 static jb_err server_keep_alive(struct client_state *csp, char **header);
150 static jb_err server_proxy_connection(struct client_state *csp, char **header);
151 static jb_err client_keep_alive(struct client_state *csp, char **header);
152 static jb_err client_save_content_length(struct client_state *csp, char **header);
153 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
155 static jb_err client_host_adder (struct client_state *csp);
156 static jb_err client_xtra_adder (struct client_state *csp);
157 static jb_err client_x_forwarded_for_adder(struct client_state *csp);
158 static jb_err client_connection_header_adder(struct client_state *csp);
159 static jb_err server_connection_adder(struct client_state *csp);
160 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
161 static jb_err server_proxy_connection_adder(struct client_state *csp);
162 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
164 static jb_err create_forged_referrer(char **header, const char *hostport);
165 static jb_err create_fake_referrer(char **header, const char *fake_referrer);
166 static jb_err handle_conditional_hide_referrer_parameter(char **header,
167 const char *host, const int parameter_conditional_block);
168 static void create_content_length_header(unsigned long long content_length,
169 char *header, size_t buffer_length);
172 * List of functions to run on a list of headers.
176 /** The header prefix to match */
179 /** The length of the prefix to match */
182 /** The function to apply to this line */
183 const parser_func_ptr parser;
186 static const struct parsers client_patterns[] = {
187 { "referer:", 8, client_referrer },
188 { "user-agent:", 11, client_uagent },
189 { "ua-", 3, client_ua },
190 { "from:", 5, client_from },
191 { "cookie:", 7, client_send_cookie },
192 { "x-forwarded-for:", 16, client_x_forwarded },
193 { "Accept-Encoding:", 16, client_accept_encoding },
194 { "TE:", 3, client_te },
195 { "Host:", 5, client_host },
196 { "if-modified-since:", 18, client_if_modified_since },
197 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
198 { "Keep-Alive:", 11, client_keep_alive },
199 { "Content-Length:", 15, client_save_content_length },
201 { "Keep-Alive:", 11, crumble },
203 { "connection:", 11, client_connection },
204 { "proxy-connection:", 17, crumble },
205 { "max-forwards:", 13, client_max_forwards },
206 { "Accept-Language:", 16, client_accept_language },
207 { "if-none-match:", 14, client_if_none_match },
208 { "Range:", 6, client_range },
209 { "Request-Range:", 14, client_range },
210 { "If-Range:", 9, client_range },
211 { "X-Filter:", 9, client_x_filter },
212 { "*", 0, crunch_client_header },
213 { "*", 0, filter_header },
217 static const struct parsers server_patterns[] = {
218 { "HTTP/", 5, server_http },
219 { "set-cookie:", 11, server_set_cookie },
220 { "connection:", 11, server_connection },
221 { "Content-Type:", 13, server_content_type },
222 { "Content-MD5:", 12, server_content_md5 },
223 { "Content-Encoding:", 17, server_content_encoding },
224 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
225 { "Content-Length:", 15, server_save_content_length },
226 { "Keep-Alive:", 11, server_keep_alive },
227 { "Proxy-Connection:", 17, server_proxy_connection },
229 { "Keep-Alive:", 11, crumble },
230 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
231 { "Transfer-Encoding:", 18, server_transfer_coding },
232 { "content-disposition:", 20, server_content_disposition },
233 { "Last-Modified:", 14, server_last_modified },
234 { "*", 0, crunch_server_header },
235 { "*", 0, filter_header },
239 static const add_header_func_ptr add_client_headers[] = {
241 client_x_forwarded_for_adder,
243 client_connection_header_adder,
247 static const add_header_func_ptr add_server_headers[] = {
248 server_connection_adder,
249 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
250 server_proxy_connection_adder,
251 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
255 /*********************************************************************
257 * Function : flush_socket
259 * Description : Write any pending "buffered" content.
262 * 1 : fd = file descriptor of the socket to read
263 * 2 : iob = The I/O buffer to flush, usually csp->iob.
265 * Returns : On success, the number of bytes written are returned (zero
266 * indicates nothing was written). On error, -1 is returned,
267 * and errno is set appropriately. If count is zero and the
268 * file descriptor refers to a regular file, 0 will be
269 * returned without causing any other effect. For a special
270 * file, the results are not portable.
272 *********************************************************************/
273 long flush_socket(jb_socket fd, struct iob *iob)
275 long len = iob->eod - iob->cur;
282 if (write_socket(fd, iob->cur, (size_t)len))
286 iob->eod = iob->cur = iob->buf;
292 /*********************************************************************
294 * Function : add_to_iob
296 * Description : Add content to the buffered page, expanding the
297 * buffer if necessary.
300 * 1 : csp = Current client state (buffers, headers, etc...)
301 * 2 : buf = holds the content to be added to the page
302 * 3 : n = number of bytes to be added
304 * Returns : JB_ERR_OK on success, JB_ERR_MEMORY if out-of-memory
305 * or buffer limit reached.
307 *********************************************************************/
308 jb_err add_to_iob(struct client_state *csp, char *buf, long n)
310 struct iob *iob = csp->iob;
311 size_t used, offset, need;
314 if (n <= 0) return JB_ERR_OK;
316 used = (size_t)(iob->eod - iob->buf);
317 offset = (size_t)(iob->cur - iob->buf);
318 need = used + (size_t)n + 1;
321 * If the buffer can't hold the new data, extend it first.
322 * Use the next power of two if possible, else use the actual need.
324 if (need > csp->config->buffer_limit)
326 log_error(LOG_LEVEL_INFO,
327 "Buffer limit reached while extending the buffer (iob). Needed: %d. Limit: %d",
328 need, csp->config->buffer_limit);
329 return JB_ERR_MEMORY;
332 if (need > iob->size)
334 size_t want = csp->iob->size ? csp->iob->size : 512;
341 if (want <= csp->config->buffer_limit && NULL != (p = (char *)realloc(iob->buf, want)))
345 else if (NULL != (p = (char *)realloc(iob->buf, need)))
351 log_error(LOG_LEVEL_ERROR, "Extending the buffer (iob) failed: %E");
352 return JB_ERR_MEMORY;
355 /* Update the iob pointers */
356 iob->cur = p + offset;
361 /* copy the new data into the iob buffer */
362 memcpy(iob->eod, buf, (size_t)n);
364 /* point to the end of the data */
367 /* null terminate == cheap insurance */
376 /*********************************************************************
378 * Function : decompress_iob
380 * Description : Decompress buffered page, expanding the
381 * buffer as necessary. csp->iob->cur
382 * should point to the the beginning of the
383 * compressed data block.
386 * 1 : csp = Current client state (buffers, headers, etc...)
388 * Returns : JB_ERR_OK on success,
389 * JB_ERR_MEMORY if out-of-memory limit reached, and
390 * JB_ERR_COMPRESS if error decompressing buffer.
392 *********************************************************************/
393 jb_err decompress_iob(struct client_state *csp)
395 char *buf; /* new, uncompressed buffer */
396 char *cur; /* Current iob position (to keep the original
397 * iob->cur unmodified if we return early) */
398 size_t bufsize; /* allocated size of the new buffer */
399 size_t old_size; /* Content size before decompression */
400 size_t skip_size; /* Number of bytes at the beginning of the iob
401 that we should NOT decompress. */
402 int status; /* return status of the inflate() call */
403 z_stream zstr; /* used by calls to zlib */
405 assert(csp->iob->cur - csp->iob->buf > 0);
406 assert(csp->iob->eod - csp->iob->cur > 0);
408 bufsize = csp->iob->size;
409 skip_size = (size_t)(csp->iob->cur - csp->iob->buf);
410 old_size = (size_t)(csp->iob->eod - csp->iob->cur);
414 if (bufsize < (size_t)10)
417 * This is to protect the parsing of gzipped data,
418 * but it should(?) be valid for deflated data also.
420 log_error(LOG_LEVEL_ERROR, "Buffer too small decompressing iob");
421 return JB_ERR_COMPRESS;
424 if (csp->content_type & CT_GZIP)
427 * Our task is slightly complicated by the facts that data
428 * compressed by gzip does not include a zlib header, and
429 * that there is no easily accessible interface in zlib to
430 * handle a gzip header. We strip off the gzip header by
431 * hand, and later inform zlib not to expect a header.
435 * Strip off the gzip header. Please see RFC 1952 for more
436 * explanation of the appropriate fields.
438 if (((*cur++ & 0xff) != GZIP_IDENTIFIER_1)
439 || ((*cur++ & 0xff) != GZIP_IDENTIFIER_2)
440 || (*cur++ != Z_DEFLATED))
442 log_error(LOG_LEVEL_ERROR, "Invalid gzip header when decompressing");
443 return JB_ERR_COMPRESS;
448 if (flags & GZIP_FLAG_RESERVED_BITS)
450 /* The gzip header has reserved bits set; bail out. */
451 log_error(LOG_LEVEL_ERROR, "Invalid gzip header flags when decompressing");
452 return JB_ERR_COMPRESS;
456 * Skip mtime (4 bytes), extra flags (1 byte)
457 * and OS type (1 byte).
461 /* Skip extra fields if necessary. */
462 if (flags & GZIP_FLAG_EXTRA_FIELDS)
465 * Skip a given number of bytes, specified
466 * as a 16-bit little-endian value.
468 * XXX: this code is untested and should probably be removed.
472 skip_bytes += *cur++ << 8;
475 * The number of bytes to skip should be positive
476 * and we'd like to stay in the buffer.
478 if ((skip_bytes < 0) || (skip_bytes >= (csp->iob->eod - cur)))
480 log_error(LOG_LEVEL_ERROR,
481 "Unreasonable amount of bytes to skip (%d). Stopping decompression",
483 return JB_ERR_COMPRESS;
485 log_error(LOG_LEVEL_INFO,
486 "Skipping %d bytes for gzip compression. Does this sound right?",
491 /* Skip the filename if necessary. */
492 if (flags & GZIP_FLAG_FILE_NAME)
494 /* A null-terminated string is supposed to follow. */
495 while (*cur++ && (cur < csp->iob->eod));
498 /* Skip the comment if necessary. */
499 if (flags & GZIP_FLAG_COMMENT)
501 /* A null-terminated string is supposed to follow. */
502 while (*cur++ && (cur < csp->iob->eod));
505 /* Skip the CRC if necessary. */
506 if (flags & GZIP_FLAG_CHECKSUM)
511 if (cur >= csp->iob->eod)
514 * If the current position pointer reached or passed
515 * the buffer end, we were obviously tricked to skip
518 log_error(LOG_LEVEL_ERROR,
519 "Malformed gzip header detected. Aborting decompression.");
520 return JB_ERR_COMPRESS;
524 else if (csp->content_type & CT_DEFLATE)
527 * XXX: The debug level should be lowered
528 * before the next stable release.
530 log_error(LOG_LEVEL_INFO, "Decompressing deflated iob: %d", *cur);
532 * In theory (that is, according to RFC 1950), deflate-compressed
533 * data should begin with a two-byte zlib header and have an
534 * adler32 checksum at the end. It seems that in practice only
535 * the raw compressed data is sent. Note that this means that
536 * we are not RFC 1950-compliant here, but the advantage is that
537 * this actually works. :)
539 * We add a dummy null byte to tell zlib where the data ends,
540 * and later inform it not to expect a header.
542 * Fortunately, add_to_iob() has thoughtfully null-terminated
543 * the buffer; we can just increment the end pointer to include
550 log_error(LOG_LEVEL_ERROR,
551 "Unable to determine compression format for decompression");
552 return JB_ERR_COMPRESS;
555 /* Set up the fields required by zlib. */
556 zstr.next_in = (Bytef *)cur;
557 zstr.avail_in = (unsigned int)(csp->iob->eod - cur);
558 zstr.zalloc = Z_NULL;
560 zstr.opaque = Z_NULL;
563 * Passing -MAX_WBITS to inflateInit2 tells the library
564 * that there is no zlib header.
566 if (inflateInit2(&zstr, -MAX_WBITS) != Z_OK)
568 log_error(LOG_LEVEL_ERROR, "Error initializing decompression");
569 return JB_ERR_COMPRESS;
573 * Next, we allocate new storage for the inflated data.
574 * We don't modify the existing iob yet, so in case there
575 * is error in decompression we can recover gracefully.
577 buf = zalloc(bufsize);
580 log_error(LOG_LEVEL_ERROR, "Out of memory decompressing iob");
581 return JB_ERR_MEMORY;
584 assert(bufsize >= skip_size);
585 memcpy(buf, csp->iob->buf, skip_size);
586 zstr.avail_out = (uInt)(bufsize - skip_size);
587 zstr.next_out = (Bytef *)buf + skip_size;
589 /* Try to decompress the whole stream in one shot. */
590 while (Z_BUF_ERROR == (status = inflate(&zstr, Z_FINISH)))
592 /* We need to allocate more memory for the output buffer. */
594 char *tmpbuf; /* used for realloc'ing the buffer */
595 size_t oldbufsize = bufsize; /* keep track of the old bufsize */
597 if (0 == zstr.avail_in)
600 * If zlib wants more data then there's a problem, because
601 * the complete compressed file should have been buffered.
603 log_error(LOG_LEVEL_ERROR,
604 "Unexpected end of compressed iob. Using what we got so far.");
609 * If we reached the buffer limit and still didn't have enough
610 * memory, just give up. Due to the ceiling enforced by the next
611 * if block we could actually check for equality here, but as it
612 * can be easily mistaken for a bug we don't.
614 if (bufsize >= csp->config->buffer_limit)
616 log_error(LOG_LEVEL_ERROR, "Buffer limit reached while decompressing iob");
617 return JB_ERR_MEMORY;
620 /* Try doubling the buffer size each time. */
623 /* Don't exceed the buffer limit. */
624 if (bufsize > csp->config->buffer_limit)
626 bufsize = csp->config->buffer_limit;
629 /* Try to allocate the new buffer. */
630 tmpbuf = realloc(buf, bufsize);
633 log_error(LOG_LEVEL_ERROR, "Out of memory decompressing iob");
635 return JB_ERR_MEMORY;
639 char *oldnext_out = (char *)zstr.next_out;
642 * Update the fields for inflate() to use the new
643 * buffer, which may be in a location different from
646 zstr.avail_out += (uInt)(bufsize - oldbufsize);
647 zstr.next_out = (Bytef *)tmpbuf + bufsize - zstr.avail_out;
650 * Compare with an uglier method of calculating these values
651 * that doesn't require the extra oldbufsize variable.
653 assert(zstr.avail_out == tmpbuf + bufsize - (char *)zstr.next_out);
654 assert((char *)zstr.next_out == tmpbuf + ((char *)oldnext_out - buf));
660 if (Z_STREAM_ERROR == inflateEnd(&zstr))
662 log_error(LOG_LEVEL_ERROR,
663 "Inconsistent stream state after decompression: %s", zstr.msg);
665 * XXX: Intentionally no return.
667 * According to zlib.h, Z_STREAM_ERROR is returned
668 * "if the stream state was inconsistent".
670 * I assume in this case inflate()'s status
671 * would also be something different than Z_STREAM_END
672 * so this check should be redundant, but lets see.
676 if ((status != Z_STREAM_END) && (0 != zstr.avail_in))
679 * We failed to decompress the stream and it's
680 * not simply because of missing data.
682 log_error(LOG_LEVEL_ERROR,
683 "Unexpected error while decompressing to the buffer (iob): %s",
685 return JB_ERR_COMPRESS;
689 * Finally, we can actually update the iob, since the
690 * decompression was successful. First, free the old
693 freez(csp->iob->buf);
695 /* Now, update the iob to use the new buffer. */
697 csp->iob->cur = csp->iob->buf + skip_size;
698 csp->iob->eod = (char *)zstr.next_out;
699 csp->iob->size = bufsize;
702 * Make sure the new uncompressed iob obeys some minimal
703 * consistency conditions.
705 if ((csp->iob->buf < csp->iob->cur)
706 && (csp->iob->cur <= csp->iob->eod)
707 && (csp->iob->eod <= csp->iob->buf + csp->iob->size))
709 const size_t new_size = (size_t)(csp->iob->eod - csp->iob->cur);
710 if (new_size > (size_t)0)
712 log_error(LOG_LEVEL_RE_FILTER,
713 "Decompression successful. Old size: %d, new size: %d.",
718 /* zlib thinks this is OK, so lets do the same. */
719 log_error(LOG_LEVEL_INFO, "Decompression didn't result in any content.");
724 /* It seems that zlib did something weird. */
725 log_error(LOG_LEVEL_ERROR,
726 "Unexpected error decompressing the buffer (iob): %d==%d, %d>%d, %d<%d",
727 csp->iob->cur, csp->iob->buf + skip_size, csp->iob->eod, csp->iob->buf,
728 csp->iob->eod, csp->iob->buf + csp->iob->size);
729 return JB_ERR_COMPRESS;
735 #endif /* defined(FEATURE_ZLIB) */
738 /*********************************************************************
740 * Function : string_move
742 * Description : memmove wrapper to move the last part of a string
743 * towards the beginning, overwriting the part in
744 * the middle. strlcpy() can't be used here as the
748 * 1 : dst = Destination to overwrite
749 * 2 : src = Source to move.
753 *********************************************************************/
754 static void string_move(char *dst, char *src)
758 /* +1 to copy the terminating nul as well. */
759 memmove(dst, src, strlen(src)+1);
763 /*********************************************************************
765 * Function : normalize_lws
767 * Description : Reduces unquoted linear white space in headers
768 * to a single space in accordance with RFC 2616 2.2.
769 * This simplifies parsing and filtering later on.
771 * XXX: Remove log messages before
772 * the next stable release?
775 * 1 : header = A header with linear white space to reduce.
779 *********************************************************************/
780 static void normalize_lws(char *header)
786 if (ijb_isspace(*p) && ijb_isspace(*(p+1)))
790 while (ijb_isspace(*q))
794 log_error(LOG_LEVEL_HEADER, "Reducing white space in '%s'", header);
800 log_error(LOG_LEVEL_HEADER,
801 "Converting tab to space in '%s'", header);
806 char *end_of_token = strstr(p+1, "\"");
808 if (NULL != end_of_token)
810 /* Don't mess with quoted text. */
815 log_error(LOG_LEVEL_HEADER,
816 "Ignoring single quote in '%s'", header);
822 p = strchr(header, ':');
823 if ((p != NULL) && (p != header) && ijb_isspace(*(p-1)))
826 * There's still space before the colon.
834 /*********************************************************************
836 * Function : get_header
838 * Description : This (odd) routine will parse the csp->iob
839 * to get the next complete header.
842 * 1 : iob = The I/O buffer to parse, usually csp->iob.
844 * Returns : Any one of the following:
846 * 1) a pointer to a dynamically allocated string that contains a header line
847 * 2) NULL indicating that the end of the header was reached
848 * 3) "" indicating that the end of the iob was reached before finding
849 * a complete header line.
851 *********************************************************************/
852 char *get_header(struct iob *iob)
856 header = get_header_line(iob);
858 if ((header == NULL) || (*header == '\0'))
861 * No complete header read yet, tell the client.
866 while ((iob->cur[0] == ' ') || (iob->cur[0] == '\t'))
869 * Header spans multiple lines, append the next one.
871 char *continued_header;
873 continued_header = get_header_line(iob);
874 if ((continued_header == NULL) || (*continued_header == '\0'))
877 * No complete header read yet, return what we got.
878 * XXX: Should "unread" header instead.
880 log_error(LOG_LEVEL_INFO,
881 "Failed to read a multi-line header properly: '%s'",
886 if (JB_ERR_OK != string_join(&header, continued_header))
888 log_error(LOG_LEVEL_FATAL,
889 "Out of memory while appending multiple headers.");
893 /* XXX: remove before next stable release. */
894 log_error(LOG_LEVEL_HEADER,
895 "Merged multiple header lines to: '%s'",
900 normalize_lws(header);
907 /*********************************************************************
909 * Function : get_header_line
911 * Description : This (odd) routine will parse the csp->iob
912 * to get the next header line.
915 * 1 : iob = The I/O buffer to parse, usually csp->iob.
917 * Returns : Any one of the following:
919 * 1) a pointer to a dynamically allocated string that contains a header line
920 * 2) NULL indicating that the end of the header was reached
921 * 3) "" indicating that the end of the iob was reached before finding
922 * a complete header line.
924 *********************************************************************/
925 static char *get_header_line(struct iob *iob)
929 if ((iob->cur == NULL)
930 || ((p = strchr(iob->cur, '\n')) == NULL))
932 return(""); /* couldn't find a complete header */
937 ret = strdup(iob->cur);
940 /* FIXME No way to handle error properly */
941 log_error(LOG_LEVEL_FATAL, "Out of memory in get_header_line()");
947 if ((q = strchr(ret, '\r')) != NULL) *q = '\0';
949 /* is this a blank line (i.e. the end of the header) ? */
961 /*********************************************************************
963 * Function : get_header_value
965 * Description : Get the value of a given header from a chained list
966 * of header lines or return NULL if no such header is
967 * present in the list.
970 * 1 : header_list = pointer to list
971 * 2 : header_name = string with name of header to look for.
972 * Trailing colon required, capitalization
975 * Returns : NULL if not found, else value of header
977 *********************************************************************/
978 char *get_header_value(const struct list *header_list, const char *header_name)
980 struct list_entry *cur_entry;
986 length = strlen(header_name);
988 for (cur_entry = header_list->first; cur_entry ; cur_entry = cur_entry->next)
992 if (!strncmpic(cur_entry->str, header_name, length))
995 * Found: return pointer to start of value
997 ret = cur_entry->str + length;
998 while (*ret && ijb_isspace(*ret)) ret++;
1012 /*********************************************************************
1014 * Function : scan_headers
1016 * Description : Scans headers, applies tags and updates action bits.
1019 * 1 : csp = Current client state (buffers, headers, etc...)
1021 * Returns : JB_ERR_OK
1023 *********************************************************************/
1024 static jb_err scan_headers(struct client_state *csp)
1026 struct list_entry *h; /* Header */
1027 jb_err err = JB_ERR_OK;
1029 for (h = csp->headers->first; (err == JB_ERR_OK) && (h != NULL) ; h = h->next)
1031 /* Header crunch()ed in previous run? -> ignore */
1032 if (h->str == NULL) continue;
1033 log_error(LOG_LEVEL_HEADER, "scan: %s", h->str);
1034 err = header_tagger(csp, h->str);
1041 /*********************************************************************
1045 * Description : add, delete or modify lines in the HTTP header streams.
1046 * On entry, it receives a linked list of headers space
1047 * that was allocated dynamically (both the list nodes
1048 * and the header contents).
1050 * As a side effect it frees the space used by the original
1054 * 1 : csp = Current client state (buffers, headers, etc...)
1055 * 2 : filter_server_headers = Boolean to switch between
1056 * server and header filtering.
1058 * Returns : JB_ERR_OK in case off success, or
1059 * JB_ERR_MEMORY on out-of-memory error.
1061 *********************************************************************/
1062 jb_err sed(struct client_state *csp, int filter_server_headers)
1064 /* XXX: use more descriptive names. */
1065 struct list_entry *p;
1066 const struct parsers *v;
1067 const add_header_func_ptr *f;
1068 jb_err err = JB_ERR_OK;
1070 if (filter_server_headers)
1072 v = server_patterns;
1073 f = add_server_headers;
1077 v = client_patterns;
1078 f = add_client_headers;
1083 while ((err == JB_ERR_OK) && (v->str != NULL))
1085 for (p = csp->headers->first; (err == JB_ERR_OK) && (p != NULL); p = p->next)
1087 /* Header crunch()ed in previous run? -> ignore */
1088 if (p->str == NULL) continue;
1090 /* Does the current parser handle this header? */
1091 if ((strncmpic(p->str, v->str, v->len) == 0) ||
1092 (v->len == CHECK_EVERY_HEADER_REMAINING))
1094 err = v->parser(csp, &(p->str));
1100 /* place additional headers on the csp->headers list */
1101 while ((err == JB_ERR_OK) && (*f))
1111 /*********************************************************************
1113 * Function : update_server_headers
1115 * Description : Updates server headers after the body has been modified.
1118 * 1 : csp = Current client state (buffers, headers, etc...)
1120 * Returns : JB_ERR_OK in case off success, or
1121 * JB_ERR_MEMORY on out-of-memory error.
1123 *********************************************************************/
1124 jb_err update_server_headers(struct client_state *csp)
1126 jb_err err = JB_ERR_OK;
1128 static const struct parsers server_patterns_light[] = {
1129 { "Content-Length:", 15, server_adjust_content_length },
1130 { "Transfer-Encoding:", 18, server_transfer_coding },
1132 { "Content-Encoding:", 17, server_adjust_content_encoding },
1133 #endif /* def FEATURE_ZLIB */
1137 if (strncmpic(csp->http->cmd, "HEAD", 4))
1139 const struct parsers *v;
1140 struct list_entry *p;
1142 for (v = server_patterns_light; (err == JB_ERR_OK) && (v->str != NULL); v++)
1144 for (p = csp->headers->first; (err == JB_ERR_OK) && (p != NULL); p = p->next)
1146 /* Header crunch()ed in previous run? -> ignore */
1147 if (p->str == NULL) continue;
1149 /* Does the current parser handle this header? */
1150 if (strncmpic(p->str, v->str, v->len) == 0)
1152 err = v->parser(csp, (char **)&(p->str));
1158 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
1159 if ((JB_ERR_OK == err)
1160 && (csp->flags & CSP_FLAG_MODIFIED)
1161 && (csp->flags & CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE)
1162 && !(csp->flags & CSP_FLAG_SERVER_CONTENT_LENGTH_SET))
1166 create_content_length_header(csp->content_length, header, sizeof(header));
1167 err = enlist(csp->headers, header);
1168 if (JB_ERR_OK == err)
1170 log_error(LOG_LEVEL_HEADER,
1171 "Content modified with no Content-Length header set. "
1172 "Created: %s.", header);
1175 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
1177 #ifdef FEATURE_COMPRESSION
1178 if ((JB_ERR_OK == err)
1179 && (csp->flags & CSP_FLAG_BUFFERED_CONTENT_DEFLATED))
1181 err = enlist_unique_header(csp->headers, "Content-Encoding", "deflate");
1182 if (JB_ERR_OK == err)
1184 log_error(LOG_LEVEL_HEADER, "Added header: Content-Encoding: deflate");
1193 /*********************************************************************
1195 * Function : header_tagger
1197 * Description : Executes all text substitutions from applying
1198 * tag actions and saves the result as tag.
1200 * XXX: Shares enough code with filter_header() and
1201 * pcrs_filter_response() to warrant some helper functions.
1204 * 1 : csp = Current client state (buffers, headers, etc...)
1205 * 2 : header = Header that is used as tagger input
1207 * Returns : JB_ERR_OK on success and always succeeds
1209 *********************************************************************/
1210 static jb_err header_tagger(struct client_state *csp, char *header)
1212 int wanted_filter_type;
1213 int multi_action_index;
1217 struct file_list *fl;
1218 struct re_filterfile_spec *b;
1219 struct list_entry *tag_name;
1221 const size_t header_length = strlen(header);
1223 if (csp->flags & CSP_FLAG_CLIENT_HEADER_PARSING_DONE)
1225 wanted_filter_type = FT_SERVER_HEADER_TAGGER;
1226 multi_action_index = ACTION_MULTI_SERVER_HEADER_TAGGER;
1230 wanted_filter_type = FT_CLIENT_HEADER_TAGGER;
1231 multi_action_index = ACTION_MULTI_CLIENT_HEADER_TAGGER;
1234 if (filters_available(csp) == FALSE)
1236 log_error(LOG_LEVEL_ERROR, "Inconsistent configuration: "
1237 "tagging enabled, but no taggers available.");
1241 for (i = 0; i < MAX_AF_FILES; i++)
1244 if ((NULL == fl) || (NULL == fl->f))
1247 * Either there are no filter files
1248 * left, or this filter file just
1249 * contains no valid filters.
1251 * Continue to be sure we don't miss
1252 * valid filter files that are chained
1253 * after empty or invalid ones.
1258 /* For all filters, */
1259 for (b = fl->f; b; b = b->next)
1261 if (b->type != wanted_filter_type)
1263 /* skip the ones we don't care about, */
1266 /* leaving only taggers that could apply, of which we use the ones, */
1267 for (tag_name = csp->action->multi[multi_action_index]->first;
1268 NULL != tag_name; tag_name = tag_name->next)
1270 /* that do apply, and */
1271 if (strcmp(b->name, tag_name->str) == 0)
1273 char *modified_tag = NULL;
1275 size_t size = header_length;
1276 pcrs_job *joblist = b->joblist;
1278 if (b->dynamic) joblist = compile_dynamic_pcrs_job_list(csp, b);
1280 if (NULL == joblist)
1282 log_error(LOG_LEVEL_RE_FILTER,
1283 "Tagger %s has empty joblist. Nothing to do.", b->name);
1287 /* execute their pcrs_joblist on the header. */
1288 for (job = joblist; NULL != job; job = job->next)
1290 const int hits = pcrs_execute(job, tag, size, &modified_tag, &size);
1294 /* Success, continue with the modified version. */
1303 /* Tagger doesn't match */
1306 /* Regex failure, log it but continue anyway. */
1307 assert(NULL != header);
1308 log_error(LOG_LEVEL_ERROR,
1309 "Problems with tagger \'%s\' and header \'%s\': %s",
1310 b->name, *header, pcrs_strerror(hits));
1312 freez(modified_tag);
1316 if (b->dynamic) pcrs_free_joblist(joblist);
1318 /* If this tagger matched */
1324 * There is to technical limitation which makes
1325 * it impossible to use empty tags, but I assume
1326 * no one would do it intentionally.
1329 log_error(LOG_LEVEL_INFO,
1330 "Tagger \'%s\' created an empty tag. Ignored.",
1335 if (!list_contains_item(csp->tags, tag))
1337 if (JB_ERR_OK != enlist(csp->tags, tag))
1339 log_error(LOG_LEVEL_ERROR,
1340 "Insufficient memory to add tag \'%s\', "
1341 "based on tagger \'%s\' and header \'%s\'",
1342 tag, b->name, *header);
1346 char *action_message;
1348 * update the action bits right away, to make
1349 * tagging based on tags set by earlier taggers
1350 * of the same kind possible.
1352 if (update_action_bits_for_tag(csp, tag))
1354 action_message = "Action bits updated accordingly.";
1358 action_message = "No action bits update necessary.";
1361 log_error(LOG_LEVEL_HEADER,
1362 "Tagger \'%s\' added tag \'%s\'. %s",
1363 b->name, tag, action_message);
1368 /* XXX: Is this log-worthy? */
1369 log_error(LOG_LEVEL_HEADER,
1370 "Tagger \'%s\' didn't add tag \'%s\'. "
1371 "Tag already present", b->name, tag);
1374 } /* if the tagger matched */
1375 } /* if the tagger applies */
1376 } /* for every tagger that could apply */
1377 } /* for all filters */
1378 } /* for all filter files */
1383 /* here begins the family of parser functions that reformat header lines */
1385 /*********************************************************************
1387 * Function : filter_header
1389 * Description : Executes all text substitutions from all applying
1390 * +(server|client)-header-filter actions on the header.
1391 * Most of the code was copied from pcrs_filter_response,
1392 * including the rather short variable names
1395 * 1 : csp = Current client state (buffers, headers, etc...)
1396 * 2 : header = On input, pointer to header to modify.
1397 * On output, pointer to the modified header, or NULL
1398 * to remove the header. This function frees the
1399 * original string if necessary.
1401 * Returns : JB_ERR_OK on success and always succeeds
1403 *********************************************************************/
1404 static jb_err filter_header(struct client_state *csp, char **header)
1408 size_t size = strlen(*header);
1410 char *newheader = NULL;
1413 struct file_list *fl;
1414 struct re_filterfile_spec *b;
1415 struct list_entry *filtername;
1418 int wanted_filter_type;
1419 int multi_action_index;
1421 if (csp->flags & CSP_FLAG_NO_FILTERING)
1426 if (csp->flags & CSP_FLAG_CLIENT_HEADER_PARSING_DONE)
1428 wanted_filter_type = FT_SERVER_HEADER_FILTER;
1429 multi_action_index = ACTION_MULTI_SERVER_HEADER_FILTER;
1433 wanted_filter_type = FT_CLIENT_HEADER_FILTER;
1434 multi_action_index = ACTION_MULTI_CLIENT_HEADER_FILTER;
1437 if (filters_available(csp) == FALSE)
1439 log_error(LOG_LEVEL_ERROR, "Inconsistent configuration: "
1440 "header filtering enabled, but no matching filters available.");
1444 for (i = 0; i < MAX_AF_FILES; i++)
1447 if ((NULL == fl) || (NULL == fl->f))
1450 * Either there are no filter files
1451 * left, or this filter file just
1452 * contains no valid filters.
1454 * Continue to be sure we don't miss
1455 * valid filter files that are chained
1456 * after empty or invalid ones.
1461 * For all applying +filter actions, look if a filter by that
1462 * name exists and if yes, execute its pcrs_joblist on the
1465 for (b = fl->f; b; b = b->next)
1467 if (b->type != wanted_filter_type)
1469 /* Skip other filter types */
1473 for (filtername = csp->action->multi[multi_action_index]->first;
1474 filtername ; filtername = filtername->next)
1476 if (strcmp(b->name, filtername->str) == 0)
1478 int current_hits = 0;
1479 pcrs_job *joblist = b->joblist;
1481 if (b->dynamic) joblist = compile_dynamic_pcrs_job_list(csp, b);
1483 if (NULL == joblist)
1485 log_error(LOG_LEVEL_RE_FILTER, "Filter %s has empty joblist. Nothing to do.", b->name);
1489 log_error(LOG_LEVEL_RE_FILTER, "filtering \'%s\' (size %d) with \'%s\' ...",
1490 *header, size, b->name);
1492 /* Apply all jobs from the joblist */
1493 for (job = joblist; NULL != job; job = job->next)
1495 matches = pcrs_execute(job, *header, size, &newheader, &size);
1498 current_hits += matches;
1499 log_error(LOG_LEVEL_HEADER, "Transforming \"%s\" to \"%s\"", *header, newheader);
1501 *header = newheader;
1503 else if ( 0 == matches )
1505 /* Filter doesn't change header */
1511 log_error(LOG_LEVEL_ERROR, "Filtering \'%s\' with \'%s\' didn't work out: %s",
1512 *header, b->name, pcrs_strerror(matches));
1513 if (newheader != NULL)
1515 log_error(LOG_LEVEL_ERROR, "Freeing what's left: %s", newheader);
1521 if (b->dynamic) pcrs_free_joblist(joblist);
1523 log_error(LOG_LEVEL_RE_FILTER, "... produced %d hits (new size %d).", current_hits, size);
1524 hits += current_hits;
1531 * Additionally checking for hits is important because if
1532 * the continue hack is triggered, server headers can
1533 * arrive empty to separate multiple heads from each other.
1535 if ((0 == size) && hits)
1537 log_error(LOG_LEVEL_HEADER, "Removing empty header %s", *header);
1545 /*********************************************************************
1547 * Function : server_connection
1549 * Description : Makes sure a proper "Connection:" header is
1550 * set and signals connection_header_adder to
1554 * 1 : csp = Current client state (buffers, headers, etc...)
1555 * 2 : header = On input, pointer to header to modify.
1556 * On output, pointer to the modified header, or NULL
1557 * to remove the header. This function frees the
1558 * original string if necessary.
1560 * Returns : JB_ERR_OK on success, or
1561 * JB_ERR_MEMORY on out-of-memory error.
1563 *********************************************************************/
1564 static jb_err server_connection(struct client_state *csp, char **header)
1566 if (!strcmpic(*header, "Connection: keep-alive")
1567 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
1568 && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED)
1572 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
1573 if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE))
1575 csp->flags |= CSP_FLAG_SERVER_CONNECTION_KEEP_ALIVE;
1578 if ((csp->flags & CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE))
1580 log_error(LOG_LEVEL_HEADER,
1581 "Keeping the server header '%s' around.", *header);
1584 #endif /* FEATURE_CONNECTION_KEEP_ALIVE */
1586 char *old_header = *header;
1588 *header = strdup("Connection: close");
1591 return JB_ERR_MEMORY;
1593 log_error(LOG_LEVEL_HEADER, "Replaced: \'%s\' with \'%s\'", old_header, *header);
1598 /* Signal server_connection_adder() to return early. */
1599 csp->flags |= CSP_FLAG_SERVER_CONNECTION_HEADER_SET;
1605 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
1606 /*********************************************************************
1608 * Function : server_keep_alive
1610 * Description : Stores the server's keep alive timeout.
1613 * 1 : csp = Current client state (buffers, headers, etc...)
1614 * 2 : header = On input, pointer to header to modify.
1615 * On output, pointer to the modified header, or NULL
1616 * to remove the header. This function frees the
1617 * original string if necessary.
1619 * Returns : JB_ERR_OK.
1621 *********************************************************************/
1622 static jb_err server_keep_alive(struct client_state *csp, char **header)
1624 unsigned int keep_alive_timeout;
1625 const char *timeout_position = strstr(*header, "timeout=");
1627 if ((NULL == timeout_position)
1628 || (1 != sscanf(timeout_position, "timeout=%u", &keep_alive_timeout)))
1630 log_error(LOG_LEVEL_ERROR, "Couldn't parse: %s", *header);
1634 if (keep_alive_timeout < csp->server_connection.keep_alive_timeout)
1636 log_error(LOG_LEVEL_HEADER,
1637 "Reducing keep-alive timeout from %u to %u.",
1638 csp->server_connection.keep_alive_timeout, keep_alive_timeout);
1639 csp->server_connection.keep_alive_timeout = keep_alive_timeout;
1643 /* XXX: Is this log worthy? */
1644 log_error(LOG_LEVEL_HEADER,
1645 "Server keep-alive timeout is %u. Sticking with %u.",
1646 keep_alive_timeout, csp->server_connection.keep_alive_timeout);
1648 csp->flags |= CSP_FLAG_SERVER_KEEP_ALIVE_TIMEOUT_SET;
1655 /*********************************************************************
1657 * Function : server_proxy_connection
1659 * Description : Figures out whether or not we should add a
1660 * Proxy-Connection header.
1663 * 1 : csp = Current client state (buffers, headers, etc...)
1664 * 2 : header = On input, pointer to header to modify.
1665 * On output, pointer to the modified header, or NULL
1666 * to remove the header. This function frees the
1667 * original string if necessary.
1669 * Returns : JB_ERR_OK.
1671 *********************************************************************/
1672 static jb_err server_proxy_connection(struct client_state *csp, char **header)
1674 csp->flags |= CSP_FLAG_SERVER_PROXY_CONNECTION_HEADER_SET;
1679 /*********************************************************************
1681 * Function : client_keep_alive
1683 * Description : Stores the client's keep alive timeout.
1686 * 1 : csp = Current client state (buffers, headers, etc...)
1687 * 2 : header = On input, pointer to header to modify.
1688 * On output, pointer to the modified header, or NULL
1689 * to remove the header. This function frees the
1690 * original string if necessary.
1692 * Returns : JB_ERR_OK.
1694 *********************************************************************/
1695 static jb_err client_keep_alive(struct client_state *csp, char **header)
1697 unsigned int keep_alive_timeout;
1698 const char *timeout_position = strstr(*header, ": ");
1700 if (!(csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE))
1702 log_error(LOG_LEVEL_HEADER,
1703 "keep-alive support is disabled. Crunching: %s.", *header);
1708 if ((NULL == timeout_position)
1709 || (1 != sscanf(timeout_position, ": %u", &keep_alive_timeout)))
1711 log_error(LOG_LEVEL_ERROR, "Couldn't parse: %s", *header);
1715 if (keep_alive_timeout < csp->config->keep_alive_timeout)
1717 log_error(LOG_LEVEL_HEADER,
1718 "Reducing keep-alive timeout from %u to %u.",
1719 csp->config->keep_alive_timeout, keep_alive_timeout);
1720 csp->server_connection.keep_alive_timeout = keep_alive_timeout;
1724 /* XXX: Is this log worthy? */
1725 log_error(LOG_LEVEL_HEADER,
1726 "Client keep-alive timeout is %u. Sticking with %u.",
1727 keep_alive_timeout, csp->config->keep_alive_timeout);
1735 /*********************************************************************
1737 * Function : get_content_length
1739 * Description : Gets the content length specified in a
1740 * Content-Length header.
1743 * 1 : header = The Content-Length header.
1744 * 2 : length = Storage to return the value.
1746 * Returns : JB_ERR_OK on success, or
1747 * JB_ERR_PARSE if no value is recognized.
1749 *********************************************************************/
1750 static jb_err get_content_length(const char *header, unsigned long long *length)
1752 assert(header[14] == ':');
1755 assert(sizeof(unsigned long long) > 4);
1756 if (1 != sscanf(header+14, ": %I64u", length))
1758 if (1 != sscanf(header+14, ": %llu", length))
1761 return JB_ERR_PARSE;
1768 /*********************************************************************
1770 * Function : client_save_content_length
1772 * Description : Save the Content-Length sent by the client.
1775 * 1 : csp = Current client state (buffers, headers, etc...)
1776 * 2 : header = On input, pointer to header to modify.
1777 * On output, pointer to the modified header, or NULL
1778 * to remove the header. This function frees the
1779 * original string if necessary.
1781 * Returns : JB_ERR_OK on success, or
1782 * JB_ERR_MEMORY on out-of-memory error.
1784 *********************************************************************/
1785 static jb_err client_save_content_length(struct client_state *csp, char **header)
1787 unsigned long long content_length = 0;
1789 assert(*(*header+14) == ':');
1791 if (JB_ERR_OK != get_content_length(*header, &content_length))
1793 log_error(LOG_LEVEL_ERROR, "Crunching invalid header: %s", *header);
1798 csp->expected_client_content_length = content_length;
1803 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
1807 /*********************************************************************
1809 * Function : client_connection
1811 * Description : Makes sure a proper "Connection:" header is
1812 * set and signals connection_header_adder
1816 * 1 : csp = Current client state (buffers, headers, etc...)
1817 * 2 : header = On input, pointer to header to modify.
1818 * On output, pointer to the modified header, or NULL
1819 * to remove the header. This function frees the
1820 * original string if necessary.
1822 * Returns : JB_ERR_OK on success, or
1823 * JB_ERR_MEMORY on out-of-memory error.
1825 *********************************************************************/
1826 static jb_err client_connection(struct client_state *csp, char **header)
1828 static const char connection_close[] = "Connection: close";
1830 if (!strcmpic(*header, connection_close))
1832 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
1833 if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_SHARING))
1835 if (!strcmpic(csp->http->ver, "HTTP/1.1"))
1837 log_error(LOG_LEVEL_HEADER,
1838 "Removing \'%s\' to imply keep-alive.", *header);
1841 * While we imply keep-alive to the server,
1842 * we have to remember that the client didn't.
1844 csp->flags &= ~CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE;
1848 char *old_header = *header;
1850 *header = strdup("Connection: keep-alive");
1853 return JB_ERR_MEMORY;
1855 log_error(LOG_LEVEL_HEADER,
1856 "Replaced: \'%s\' with \'%s\'", old_header, *header);
1862 log_error(LOG_LEVEL_HEADER,
1863 "Keeping the client header '%s' around. "
1864 "The connection will not be kept alive.",
1866 csp->flags &= ~CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE;
1869 else if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE))
1871 log_error(LOG_LEVEL_HEADER,
1872 "Keeping the client header '%s' around. "
1873 "The server connection will be kept alive if possible.",
1875 csp->flags |= CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE;
1876 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
1880 char *old_header = *header;
1882 *header = strdup(connection_close);
1885 return JB_ERR_MEMORY;
1887 log_error(LOG_LEVEL_HEADER,
1888 "Replaced: \'%s\' with \'%s\'", old_header, *header);
1892 /* Signal client_connection_adder() to return early. */
1893 csp->flags |= CSP_FLAG_CLIENT_CONNECTION_HEADER_SET;
1899 /*********************************************************************
1901 * Function : crumble
1903 * Description : This is called if a header matches a pattern to "crunch"
1906 * 1 : csp = Current client state (buffers, headers, etc...)
1907 * 2 : header = On input, pointer to header to modify.
1908 * On output, pointer to the modified header, or NULL
1909 * to remove the header. This function frees the
1910 * original string if necessary.
1912 * Returns : JB_ERR_OK on success, or
1913 * JB_ERR_MEMORY on out-of-memory error.
1915 *********************************************************************/
1916 static jb_err crumble(struct client_state *csp, char **header)
1919 log_error(LOG_LEVEL_HEADER, "crumble crunched: %s!", *header);
1925 /*********************************************************************
1927 * Function : crunch_server_header
1929 * Description : Crunch server header if it matches a string supplied by the
1930 * user. Called from `sed'.
1933 * 1 : csp = Current client state (buffers, headers, etc...)
1934 * 2 : header = On input, pointer to header to modify.
1935 * On output, pointer to the modified header, or NULL
1936 * to remove the header. This function frees the
1937 * original string if necessary.
1939 * Returns : JB_ERR_OK on success and always succeeds
1941 *********************************************************************/
1942 static jb_err crunch_server_header(struct client_state *csp, char **header)
1944 const char *crunch_pattern;
1946 /* Do we feel like crunching? */
1947 if ((csp->action->flags & ACTION_CRUNCH_SERVER_HEADER))
1949 crunch_pattern = csp->action->string[ACTION_STRING_SERVER_HEADER];
1951 /* Is the current header the lucky one? */
1952 if (strstr(*header, crunch_pattern))
1954 log_error(LOG_LEVEL_HEADER, "Crunching server header: %s (contains: %s)", *header, crunch_pattern);
1963 /*********************************************************************
1965 * Function : server_content_type
1967 * Description : Set the content-type for filterable types (text/.*,
1968 * .*xml.*, .*script.* and image/gif) unless filtering has been
1969 * forbidden (CT_TABOO) while parsing earlier headers.
1970 * NOTE: Since text/plain is commonly used by web servers
1971 * for files whose correct type is unknown, we don't
1972 * set CT_TEXT for it.
1975 * 1 : csp = Current client state (buffers, headers, etc...)
1976 * 2 : header = On input, pointer to header to modify.
1977 * On output, pointer to the modified header, or NULL
1978 * to remove the header. This function frees the
1979 * original string if necessary.
1981 * Returns : JB_ERR_OK on success, or
1982 * JB_ERR_MEMORY on out-of-memory error.
1984 *********************************************************************/
1985 static jb_err server_content_type(struct client_state *csp, char **header)
1987 /* Remove header if it isn't the first Content-Type header */
1988 if ((csp->content_type & CT_DECLARED))
1991 * Another, slightly slower, way to see if
1992 * we already parsed another Content-Type header.
1994 assert(NULL != get_header_value(csp->headers, "Content-Type:"));
1996 log_error(LOG_LEVEL_ERROR,
1997 "Multiple Content-Type headers. Removing and ignoring: \'%s\'",
2005 * Signal that the Content-Type has been set.
2007 csp->content_type |= CT_DECLARED;
2009 if (!(csp->content_type & CT_TABOO))
2012 * XXX: The assumption that text/plain is a sign of
2013 * binary data seems to be somewhat unreasonable nowadays
2014 * and should be dropped after 3.0.8 is out.
2016 if ((strstr(*header, "text/") && !strstr(*header, "plain"))
2017 || strstr(*header, "xml")
2018 || strstr(*header, "script"))
2020 csp->content_type |= CT_TEXT;
2022 else if (strstr(*header, "image/gif"))
2024 csp->content_type |= CT_GIF;
2029 * Are we messing with the content type?
2031 if (csp->action->flags & ACTION_CONTENT_TYPE_OVERWRITE)
2034 * Make sure the user doesn't accidentally
2035 * change the content type of binary documents.
2037 if ((csp->content_type & CT_TEXT) || (csp->action->flags & ACTION_FORCE_TEXT_MODE))
2040 *header = strdup("Content-Type: ");
2041 string_append(header, csp->action->string[ACTION_STRING_CONTENT_TYPE]);
2045 log_error(LOG_LEVEL_HEADER, "Insufficient memory to replace Content-Type!");
2046 return JB_ERR_MEMORY;
2048 log_error(LOG_LEVEL_HEADER, "Modified: %s!", *header);
2052 log_error(LOG_LEVEL_HEADER, "%s not replaced. "
2053 "It doesn't look like a content type that should be filtered. "
2054 "Enable force-text-mode if you know what you're doing.", *header);
2062 /*********************************************************************
2064 * Function : server_transfer_coding
2066 * Description : - Prohibit filtering (CT_TABOO) if transfer coding compresses
2067 * - Raise the CSP_FLAG_CHUNKED flag if coding is "chunked"
2068 * - Remove header if body was chunked but has been
2069 * de-chunked for filtering.
2072 * 1 : csp = Current client state (buffers, headers, etc...)
2073 * 2 : header = On input, pointer to header to modify.
2074 * On output, pointer to the modified header, or NULL
2075 * to remove the header. This function frees the
2076 * original string if necessary.
2078 * Returns : JB_ERR_OK on success, or
2079 * JB_ERR_MEMORY on out-of-memory error.
2081 *********************************************************************/
2082 static jb_err server_transfer_coding(struct client_state *csp, char **header)
2085 * Turn off pcrs and gif filtering if body compressed
2087 if (strstr(*header, "gzip") || strstr(*header, "compress") || strstr(*header, "deflate"))
2091 * XXX: Added to test if we could use CT_GZIP and CT_DEFLATE here.
2093 log_error(LOG_LEVEL_INFO, "Marking content type for %s as CT_TABOO because of %s.",
2094 csp->http->cmd, *header);
2095 #endif /* def FEATURE_ZLIB */
2096 csp->content_type = CT_TABOO;
2100 * Raise flag if body chunked
2102 if (strstr(*header, "chunked"))
2104 csp->flags |= CSP_FLAG_CHUNKED;
2107 * If the body was modified, it has been de-chunked first
2108 * and the header must be removed.
2110 * FIXME: If there is more than one transfer encoding,
2111 * only the "chunked" part should be removed here.
2113 if (csp->flags & CSP_FLAG_MODIFIED)
2115 log_error(LOG_LEVEL_HEADER, "Removing: %s", *header);
2124 /*********************************************************************
2126 * Function : server_content_encoding
2128 * Description : Used to check if the content is compressed, and if
2129 * FEATURE_ZLIB is disabled, filtering is disabled as
2132 * If FEATURE_ZLIB is enabled and the compression type
2133 * supported, the content is marked for decompression.
2135 * XXX: Doesn't properly deal with multiple or with
2136 * unsupported but unknown encodings.
2137 * Is case-sensitive but shouldn't be.
2140 * 1 : csp = Current client state (buffers, headers, etc...)
2141 * 2 : header = On input, pointer to header to modify.
2142 * On output, pointer to the modified header, or NULL
2143 * to remove the header. This function frees the
2144 * original string if necessary.
2146 * Returns : JB_ERR_OK on success, or
2147 * JB_ERR_MEMORY on out-of-memory error.
2149 *********************************************************************/
2150 static jb_err server_content_encoding(struct client_state *csp, char **header)
2153 if (strstr(*header, "sdch"))
2156 * Shared Dictionary Compression over HTTP isn't supported,
2157 * filtering it anyway is pretty much guaranteed to mess up
2160 csp->content_type |= CT_TABOO;
2163 * Log a warning if the user expects the content to be filtered.
2165 if ((csp->rlist != NULL) &&
2166 (!list_is_empty(csp->action->multi[ACTION_MULTI_FILTER])))
2168 log_error(LOG_LEVEL_INFO,
2169 "SDCH-compressed content detected, content filtering disabled. "
2170 "Consider suppressing SDCH offers made by the client.");
2173 else if (strstr(*header, "gzip"))
2175 /* Mark for gzip decompression */
2176 csp->content_type |= CT_GZIP;
2178 else if (strstr(*header, "deflate"))
2180 /* Mark for zlib decompression */
2181 csp->content_type |= CT_DEFLATE;
2183 else if (strstr(*header, "compress"))
2186 * We can't decompress this; therefore we can't filter
2189 csp->content_type |= CT_TABOO;
2191 #else /* !defined(FEATURE_ZLIB) */
2193 * XXX: Using a black list here isn't the right approach.
2195 * In case of SDCH, building with zlib support isn't
2198 if (strstr(*header, "gzip") ||
2199 strstr(*header, "compress") ||
2200 strstr(*header, "deflate") ||
2201 strstr(*header, "sdch"))
2204 * Body is compressed, turn off pcrs and gif filtering.
2206 csp->content_type |= CT_TABOO;
2209 * Log a warning if the user expects the content to be filtered.
2211 if ((csp->rlist != NULL) &&
2212 (!list_is_empty(csp->action->multi[ACTION_MULTI_FILTER])))
2214 log_error(LOG_LEVEL_INFO,
2215 "Compressed content detected, content filtering disabled. "
2216 "Consider recompiling Privoxy with zlib support or "
2217 "enable the prevent-compression action.");
2220 #endif /* defined(FEATURE_ZLIB) */
2228 /*********************************************************************
2230 * Function : server_adjust_content_encoding
2232 * Description : Remove the Content-Encoding header if the
2233 * decompression was successful and the content
2237 * 1 : csp = Current client state (buffers, headers, etc...)
2238 * 2 : header = On input, pointer to header to modify.
2239 * On output, pointer to the modified header, or NULL
2240 * to remove the header. This function frees the
2241 * original string if necessary.
2243 * Returns : JB_ERR_OK on success, or
2244 * JB_ERR_MEMORY on out-of-memory error.
2246 *********************************************************************/
2247 static jb_err server_adjust_content_encoding(struct client_state *csp, char **header)
2249 if ((csp->flags & CSP_FLAG_MODIFIED)
2250 && (csp->content_type & (CT_GZIP | CT_DEFLATE)))
2253 * We successfully decompressed the content,
2254 * and have to clean the header now, so the
2255 * client no longer expects compressed data.
2257 * XXX: There is a difference between cleaning
2258 * and removing it completely.
2260 log_error(LOG_LEVEL_HEADER, "Crunching: %s", *header);
2267 #endif /* defined(FEATURE_ZLIB) */
2270 /*********************************************************************
2272 * Function : server_adjust_content_length
2274 * Description : Adjust Content-Length header if we modified
2278 * 1 : csp = Current client state (buffers, headers, etc...)
2279 * 2 : header = On input, pointer to header to modify.
2280 * On output, pointer to the modified header, or NULL
2281 * to remove the header. This function frees the
2282 * original string if necessary.
2284 * Returns : JB_ERR_OK on success, or
2285 * JB_ERR_MEMORY on out-of-memory error.
2287 *********************************************************************/
2288 static jb_err server_adjust_content_length(struct client_state *csp, char **header)
2290 /* Regenerate header if the content was modified. */
2291 if (csp->flags & CSP_FLAG_MODIFIED)
2293 const size_t header_length = 50;
2295 *header = malloc(header_length);
2296 if (*header == NULL)
2298 return JB_ERR_MEMORY;
2300 create_content_length_header(csp->content_length, *header, header_length);
2301 log_error(LOG_LEVEL_HEADER,
2302 "Adjusted Content-Length to %llu", csp->content_length);
2309 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
2310 /*********************************************************************
2312 * Function : server_save_content_length
2314 * Description : Save the Content-Length sent by the server.
2317 * 1 : csp = Current client state (buffers, headers, etc...)
2318 * 2 : header = On input, pointer to header to modify.
2319 * On output, pointer to the modified header, or NULL
2320 * to remove the header. This function frees the
2321 * original string if necessary.
2323 * Returns : JB_ERR_OK on success, or
2324 * JB_ERR_MEMORY on out-of-memory error.
2326 *********************************************************************/
2327 static jb_err server_save_content_length(struct client_state *csp, char **header)
2329 unsigned long long content_length = 0;
2331 assert(*(*header+14) == ':');
2333 if (JB_ERR_OK != get_content_length(*header, &content_length))
2335 log_error(LOG_LEVEL_ERROR, "Crunching invalid header: %s", *header);
2340 csp->expected_content_length = content_length;
2341 csp->flags |= CSP_FLAG_SERVER_CONTENT_LENGTH_SET;
2342 csp->flags |= CSP_FLAG_CONTENT_LENGTH_SET;
2347 #endif /* def FEATURE_CONNECTION_KEEP_ALIVE */
2350 /*********************************************************************
2352 * Function : server_content_md5
2354 * Description : Crumble any Content-MD5 headers if the document was
2355 * modified. FIXME: Should we re-compute instead?
2358 * 1 : csp = Current client state (buffers, headers, etc...)
2359 * 2 : header = On input, pointer to header to modify.
2360 * On output, pointer to the modified header, or NULL
2361 * to remove the header. This function frees the
2362 * original string if necessary.
2364 * Returns : JB_ERR_OK on success, or
2365 * JB_ERR_MEMORY on out-of-memory error.
2367 *********************************************************************/
2368 static jb_err server_content_md5(struct client_state *csp, char **header)
2370 if (csp->flags & CSP_FLAG_MODIFIED)
2372 log_error(LOG_LEVEL_HEADER, "Crunching Content-MD5");
2380 /*********************************************************************
2382 * Function : server_content_disposition
2384 * Description : If enabled, blocks or modifies the "Content-Disposition" header.
2385 * Called from `sed'.
2388 * 1 : csp = Current client state (buffers, headers, etc...)
2389 * 2 : header = On input, pointer to header to modify.
2390 * On output, pointer to the modified header, or NULL
2391 * to remove the header. This function frees the
2392 * original string if necessary.
2394 * Returns : JB_ERR_OK on success, or
2395 * JB_ERR_MEMORY on out-of-memory error.
2397 *********************************************************************/
2398 static jb_err server_content_disposition(struct client_state *csp, char **header)
2403 * Are we messing with the Content-Disposition header?
2405 if ((csp->action->flags & ACTION_HIDE_CONTENT_DISPOSITION) == 0)
2411 newval = csp->action->string[ACTION_STRING_CONTENT_DISPOSITION];
2413 if ((newval == NULL) || (0 == strcmpic(newval, "block")))
2416 * Blocking content-disposition header
2418 log_error(LOG_LEVEL_HEADER, "Crunching %s!", *header);
2425 * Replacing Content-Disposition header
2428 *header = strdup("Content-Disposition: ");
2429 string_append(header, newval);
2431 if (*header != NULL)
2433 log_error(LOG_LEVEL_HEADER,
2434 "Content-Disposition header crunched and replaced with: %s", *header);
2437 return (*header == NULL) ? JB_ERR_MEMORY : JB_ERR_OK;
2441 /*********************************************************************
2443 * Function : server_last_modified
2445 * Description : Changes Last-Modified header to the actual date
2446 * to help hide-if-modified-since.
2447 * Called from `sed'.
2450 * 1 : csp = Current client state (buffers, headers, etc...)
2451 * 2 : header = On input, pointer to header to modify.
2452 * On output, pointer to the modified header, or NULL
2453 * to remove the header. This function frees the
2454 * original string if necessary.
2456 * Returns : JB_ERR_OK on success, or
2457 * JB_ERR_MEMORY on out-of-memory error.
2459 *********************************************************************/
2460 static jb_err server_last_modified(struct client_state *csp, char **header)
2463 time_t last_modified;
2467 * Are we messing with the Last-Modified header?
2469 if ((csp->action->flags & ACTION_OVERWRITE_LAST_MODIFIED) == 0)
2475 newval = csp->action->string[ACTION_STRING_LAST_MODIFIED];
2477 if (0 == strcmpic(newval, "block") )
2480 * Blocking Last-Modified header. Useless but why not.
2482 log_error(LOG_LEVEL_HEADER, "Crunching %s!", *header);
2486 else if (0 == strcmpic(newval, "reset-to-request-time"))
2489 * Setting Last-Modified Header to now.
2492 get_http_time(0, buf, sizeof(buf));
2494 *header = strdup("Last-Modified: ");
2495 string_append(header, buf);
2497 if (*header == NULL)
2499 log_error(LOG_LEVEL_HEADER, "Insufficient memory. Last-Modified header got lost, boohoo.");
2503 log_error(LOG_LEVEL_HEADER, "Reset to present time: %s", *header);
2506 else if (0 == strcmpic(newval, "randomize"))
2508 const char *header_time = *header + sizeof("Last-Modified:");
2510 log_error(LOG_LEVEL_HEADER, "Randomizing: %s", *header);
2512 if (JB_ERR_OK != parse_header_time(header_time, &last_modified))
2514 log_error(LOG_LEVEL_HEADER, "Couldn't parse: %s in %s (crunching!)", header_time, *header);
2520 struct tm *timeptr = NULL;
2522 #ifdef HAVE_GMTIME_R
2526 rtime = (long int)difftime(now, last_modified);
2529 long int days, hours, minutes, seconds;
2530 const int negative_delta = (rtime < 0);
2535 log_error(LOG_LEVEL_HEADER, "Server time in the future.");
2537 rtime = pick_from_range(rtime);
2542 last_modified += rtime;
2543 #ifdef HAVE_GMTIME_R
2544 timeptr = gmtime_r(&last_modified, &gmt);
2545 #elif defined(MUTEX_LOCKS_AVAILABLE)
2546 privoxy_mutex_lock(&gmtime_mutex);
2547 timeptr = gmtime(&last_modified);
2548 privoxy_mutex_unlock(&gmtime_mutex);
2550 timeptr = gmtime(&last_modified);
2552 if ((NULL == timeptr) || !strftime(newheader,
2553 sizeof(newheader), "%a, %d %b %Y %H:%M:%S GMT", timeptr))
2555 log_error(LOG_LEVEL_ERROR,
2556 "Randomizing '%s' failed. Crunching the header without replacement.",
2563 *header = strdup("Last-Modified: ");
2564 string_append(header, newheader);
2566 if (*header == NULL)
2568 log_error(LOG_LEVEL_ERROR, "Insufficient memory, header crunched without replacement.");
2569 return JB_ERR_MEMORY;
2572 days = rtime / (3600 * 24);
2573 hours = rtime / 3600 % 24;
2574 minutes = rtime / 60 % 60;
2575 seconds = rtime % 60;
2577 log_error(LOG_LEVEL_HEADER,
2578 "Randomized: %s (added %d da%s %d hou%s %d minut%s %d second%s",
2579 *header, days, (days == 1) ? "y" : "ys", hours, (hours == 1) ? "r" : "rs",
2580 minutes, (minutes == 1) ? "e" : "es", seconds, (seconds == 1) ? ")" : "s)");
2584 log_error(LOG_LEVEL_HEADER, "Randomized ... or not. No time difference to work with.");
2593 /*********************************************************************
2595 * Function : client_accept_encoding
2597 * Description : Rewrite the client's Accept-Encoding header so that
2598 * if doesn't allow compression, if the action applies.
2599 * Note: For HTTP/1.0 the absence of the header is enough.
2602 * 1 : csp = Current client state (buffers, headers, etc...)
2603 * 2 : header = On input, pointer to header to modify.
2604 * On output, pointer to the modified header, or NULL
2605 * to remove the header. This function frees the
2606 * original string if necessary.
2608 * Returns : JB_ERR_OK on success, or
2609 * JB_ERR_MEMORY on out-of-memory error.
2611 *********************************************************************/
2612 static jb_err client_accept_encoding(struct client_state *csp, char **header)
2614 #ifdef FEATURE_COMPRESSION
2615 if ((csp->config->feature_flags & RUNTIME_FEATURE_COMPRESSION)
2616 && strstr(*header, "deflate"))
2618 csp->flags |= CSP_FLAG_CLIENT_SUPPORTS_DEFLATE;
2621 if ((csp->action->flags & ACTION_NO_COMPRESSION) != 0)
2623 log_error(LOG_LEVEL_HEADER, "Suppressed offer to compress content");
2631 /*********************************************************************
2633 * Function : client_te
2635 * Description : Rewrite the client's TE header so that
2636 * if doesn't allow compression, if the action applies.
2639 * 1 : csp = Current client state (buffers, headers, etc...)
2640 * 2 : header = On input, pointer to header to modify.
2641 * On output, pointer to the modified header, or NULL
2642 * to remove the header. This function frees the
2643 * original string if necessary.
2645 * Returns : JB_ERR_OK on success, or
2646 * JB_ERR_MEMORY on out-of-memory error.
2648 *********************************************************************/
2649 static jb_err client_te(struct client_state *csp, char **header)
2651 if ((csp->action->flags & ACTION_NO_COMPRESSION) != 0)
2654 log_error(LOG_LEVEL_HEADER, "Suppressed offer to compress transfer");
2661 /*********************************************************************
2663 * Function : client_referrer
2665 * Description : Handle the "referer" config setting properly.
2666 * Called from `sed'.
2669 * 1 : csp = Current client state (buffers, headers, etc...)
2670 * 2 : header = On input, pointer to header to modify.
2671 * On output, pointer to the modified header, or NULL
2672 * to remove the header. This function frees the
2673 * original string if necessary.
2675 * Returns : JB_ERR_OK on success, or
2676 * JB_ERR_MEMORY on out-of-memory error.
2678 *********************************************************************/
2679 static jb_err client_referrer(struct client_state *csp, char **header)
2681 const char *parameter;
2682 /* booleans for parameters we have to check multiple times */
2683 int parameter_conditional_block;
2684 int parameter_conditional_forge;
2686 #ifdef FEATURE_FORCE_LOAD
2688 * Since the referrer can include the prefix even
2689 * if the request itself is non-forced, we must
2690 * clean it unconditionally.
2692 * XXX: strclean is too broad
2694 strclean(*header, FORCE_PREFIX);
2695 #endif /* def FEATURE_FORCE_LOAD */
2697 if ((csp->action->flags & ACTION_HIDE_REFERER) == 0)
2699 /* Nothing left to do */
2703 parameter = csp->action->string[ACTION_STRING_REFERER];
2704 assert(parameter != NULL);
2705 parameter_conditional_block = (0 == strcmpic(parameter, "conditional-block"));
2706 parameter_conditional_forge = (0 == strcmpic(parameter, "conditional-forge"));
2708 if (!parameter_conditional_block && !parameter_conditional_forge)
2711 * As conditional-block and conditional-forge are the only
2712 * parameters that rely on the original referrer, we can
2713 * remove it now for all the others.
2718 if (0 == strcmpic(parameter, "block"))
2720 log_error(LOG_LEVEL_HEADER, "Referer crunched!");
2723 else if (parameter_conditional_block || parameter_conditional_forge)
2725 return handle_conditional_hide_referrer_parameter(header,
2726 csp->http->hostport, parameter_conditional_block);
2728 else if (0 == strcmpic(parameter, "forge"))
2730 return create_forged_referrer(header, csp->http->hostport);
2734 /* interpret parameter as user-supplied referer to fake */
2735 return create_fake_referrer(header, parameter);
2740 /*********************************************************************
2742 * Function : client_accept_language
2744 * Description : Handle the "Accept-Language" config setting properly.
2745 * Called from `sed'.
2748 * 1 : csp = Current client state (buffers, headers, etc...)
2749 * 2 : header = On input, pointer to header to modify.
2750 * On output, pointer to the modified header, or NULL
2751 * to remove the header. This function frees the
2752 * original string if necessary.
2754 * Returns : JB_ERR_OK on success, or
2755 * JB_ERR_MEMORY on out-of-memory error.
2757 *********************************************************************/
2758 static jb_err client_accept_language(struct client_state *csp, char **header)
2763 * Are we messing with the Accept-Language?
2765 if ((csp->action->flags & ACTION_HIDE_ACCEPT_LANGUAGE) == 0)
2767 /*I don't think so*/
2771 newval = csp->action->string[ACTION_STRING_LANGUAGE];
2773 if ((newval == NULL) || (0 == strcmpic(newval, "block")) )
2776 * Blocking Accept-Language header
2778 log_error(LOG_LEVEL_HEADER, "Crunching Accept-Language!");
2785 * Replacing Accept-Language header
2788 *header = strdup("Accept-Language: ");
2789 string_append(header, newval);
2791 if (*header == NULL)
2793 log_error(LOG_LEVEL_ERROR,
2794 "Insufficient memory. Accept-Language header crunched without replacement.");
2798 log_error(LOG_LEVEL_HEADER,
2799 "Accept-Language header crunched and replaced with: %s", *header);
2802 return (*header == NULL) ? JB_ERR_MEMORY : JB_ERR_OK;
2806 /*********************************************************************
2808 * Function : crunch_client_header
2810 * Description : Crunch client header if it matches a string supplied by the
2811 * user. Called from `sed'.
2814 * 1 : csp = Current client state (buffers, headers, etc...)
2815 * 2 : header = On input, pointer to header to modify.
2816 * On output, pointer to the modified header, or NULL
2817 * to remove the header. This function frees the
2818 * original string if necessary.
2820 * Returns : JB_ERR_OK on success and always succeeds
2822 *********************************************************************/
2823 static jb_err crunch_client_header(struct client_state *csp, char **header)
2825 const char *crunch_pattern;
2827 /* Do we feel like crunching? */
2828 if ((csp->action->flags & ACTION_CRUNCH_CLIENT_HEADER))
2830 crunch_pattern = csp->action->string[ACTION_STRING_CLIENT_HEADER];
2832 /* Is the current header the lucky one? */
2833 if (strstr(*header, crunch_pattern))
2835 log_error(LOG_LEVEL_HEADER, "Crunching client header: %s (contains: %s)", *header, crunch_pattern);
2843 /*********************************************************************
2845 * Function : client_uagent
2847 * Description : Handle the "user-agent" config setting properly
2848 * and remember its original value to enable browser
2849 * bug workarounds. Called from `sed'.
2852 * 1 : csp = Current client state (buffers, headers, etc...)
2853 * 2 : header = On input, pointer to header to modify.
2854 * On output, pointer to the modified header, or NULL
2855 * to remove the header. This function frees the
2856 * original string if necessary.
2858 * Returns : JB_ERR_OK on success, or
2859 * JB_ERR_MEMORY on out-of-memory error.
2861 *********************************************************************/
2862 static jb_err client_uagent(struct client_state *csp, char **header)
2866 if ((csp->action->flags & ACTION_HIDE_USER_AGENT) == 0)
2871 newval = csp->action->string[ACTION_STRING_USER_AGENT];
2878 *header = strdup("User-Agent: ");
2879 string_append(header, newval);
2881 log_error(LOG_LEVEL_HEADER, "Modified: %s", *header);
2883 return (*header == NULL) ? JB_ERR_MEMORY : JB_ERR_OK;
2887 /*********************************************************************
2889 * Function : client_ua
2891 * Description : Handle "ua-" headers properly. Called from `sed'.
2894 * 1 : csp = Current client state (buffers, headers, etc...)
2895 * 2 : header = On input, pointer to header to modify.
2896 * On output, pointer to the modified header, or NULL
2897 * to remove the header. This function frees the
2898 * original string if necessary.
2900 * Returns : JB_ERR_OK on success, or
2901 * JB_ERR_MEMORY on out-of-memory error.
2903 *********************************************************************/
2904 static jb_err client_ua(struct client_state *csp, char **header)
2906 if ((csp->action->flags & ACTION_HIDE_USER_AGENT) != 0)
2908 log_error(LOG_LEVEL_HEADER, "crunched User-Agent!");
2916 /*********************************************************************
2918 * Function : client_from
2920 * Description : Handle the "from" config setting properly.
2921 * Called from `sed'.
2924 * 1 : csp = Current client state (buffers, headers, etc...)
2925 * 2 : header = On input, pointer to header to modify.
2926 * On output, pointer to the modified header, or NULL
2927 * to remove the header. This function frees the
2928 * original string if necessary.
2930 * Returns : JB_ERR_OK on success, or
2931 * JB_ERR_MEMORY on out-of-memory error.
2933 *********************************************************************/
2934 static jb_err client_from(struct client_state *csp, char **header)
2938 if ((csp->action->flags & ACTION_HIDE_FROM) == 0)
2945 newval = csp->action->string[ACTION_STRING_FROM];
2948 * Are we blocking the e-mail address?
2950 if ((newval == NULL) || (0 == strcmpic(newval, "block")) )
2952 log_error(LOG_LEVEL_HEADER, "crunched From!");
2956 log_error(LOG_LEVEL_HEADER, " modified");
2958 *header = strdup("From: ");
2959 string_append(header, newval);
2961 return (*header == NULL) ? JB_ERR_MEMORY : JB_ERR_OK;
2965 /*********************************************************************
2967 * Function : client_send_cookie
2969 * Description : Crunches the "cookie" header if necessary.
2970 * Called from `sed'.
2972 * XXX: Stupid name, doesn't send squat.
2975 * 1 : csp = Current client state (buffers, headers, etc...)
2976 * 2 : header = On input, pointer to header to modify.
2977 * On output, pointer to the modified header, or NULL
2978 * to remove the header. This function frees the
2979 * original string if necessary.
2981 * Returns : JB_ERR_OK on success, or
2982 * JB_ERR_MEMORY on out-of-memory error.
2984 *********************************************************************/
2985 static jb_err client_send_cookie(struct client_state *csp, char **header)
2987 if (csp->action->flags & ACTION_NO_COOKIE_READ)
2989 log_error(LOG_LEVEL_HEADER, "Crunched outgoing cookie: %s", *header);
2997 /*********************************************************************
2999 * Function : client_x_forwarded
3001 * Description : Handle the "x-forwarded-for" config setting properly,
3002 * also used in the add_client_headers list. Called from `sed'.
3005 * 1 : csp = Current client state (buffers, headers, etc...)
3006 * 2 : header = On input, pointer to header to modify.
3007 * On output, pointer to the modified header, or NULL
3008 * to remove the header. This function frees the
3009 * original string if necessary.
3011 * Returns : JB_ERR_OK on success, or
3012 * JB_ERR_MEMORY on out-of-memory error.
3014 *********************************************************************/
3015 jb_err client_x_forwarded(struct client_state *csp, char **header)
3017 if (0 != (csp->action->flags & ACTION_CHANGE_X_FORWARDED_FOR))
3019 const char *parameter = csp->action->string[ACTION_STRING_CHANGE_X_FORWARDED_FOR];
3021 if (0 == strcmpic(parameter, "block"))
3024 log_error(LOG_LEVEL_HEADER, "crunched x-forwarded-for!");
3026 else if (0 == strcmpic(parameter, "add"))
3028 string_append(header, ", ");
3029 string_append(header, csp->ip_addr_str);
3031 if (*header == NULL)
3033 return JB_ERR_MEMORY;
3035 log_error(LOG_LEVEL_HEADER,
3036 "Appended client IP address to %s", *header);
3037 csp->flags |= CSP_FLAG_X_FORWARDED_FOR_APPENDED;
3041 log_error(LOG_LEVEL_FATAL,
3042 "Invalid change-x-forwarded-for parameter: '%s'", parameter);
3050 /*********************************************************************
3052 * Function : client_max_forwards
3054 * Description : If the HTTP method is OPTIONS or TRACE, subtract one
3055 * from the value of the Max-Forwards header field.
3058 * 1 : csp = Current client state (buffers, headers, etc...)
3059 * 2 : header = On input, pointer to header to modify.
3060 * On output, pointer to the modified header, or NULL
3061 * to remove the header. This function frees the
3062 * original string if necessary.
3064 * Returns : JB_ERR_OK on success, or
3065 * JB_ERR_MEMORY on out-of-memory error.
3067 *********************************************************************/
3068 static jb_err client_max_forwards(struct client_state *csp, char **header)
3072 if ((0 == strcmpic(csp->http->gpc, "trace")) ||
3073 (0 == strcmpic(csp->http->gpc, "options")))
3075 assert(*(*header+12) == ':');
3076 if (1 == sscanf(*header+12, ": %d", &max_forwards))
3078 if (max_forwards > 0)
3080 snprintf(*header, strlen(*header)+1, "Max-Forwards: %d", --max_forwards);
3081 log_error(LOG_LEVEL_HEADER,
3082 "Max-Forwards value for %s request reduced to %d.",
3083 csp->http->gpc, max_forwards);
3085 else if (max_forwards < 0)
3087 log_error(LOG_LEVEL_ERROR, "Crunching invalid header: %s", *header);
3093 log_error(LOG_LEVEL_ERROR, "Crunching invalid header: %s", *header);
3102 /*********************************************************************
3104 * Function : client_host
3106 * Description : If the request URI did not contain host and
3107 * port information, parse and evaluate the Host
3110 * Also, kill ill-formed HOST: headers as sent by
3111 * Apple's iTunes software when used with a proxy.
3114 * 1 : csp = Current client state (buffers, headers, etc...)
3115 * 2 : header = On input, pointer to header to modify.
3116 * On output, pointer to the modified header, or NULL
3117 * to remove the header. This function frees the
3118 * original string if necessary.
3120 * Returns : JB_ERR_OK on success, or
3121 * JB_ERR_MEMORY on out-of-memory error.
3123 *********************************************************************/
3124 static jb_err client_host(struct client_state *csp, char **header)
3129 * If the header field name is all upper-case, chances are that it's
3130 * an ill-formed one from iTunes. BTW, killing innocent headers here is
3131 * not a problem -- they are regenerated later.
3133 if ((*header)[1] == 'O')
3135 log_error(LOG_LEVEL_HEADER, "Killed all-caps Host header line: %s", *header);
3140 if (!csp->http->hostport || (*csp->http->hostport == '*') ||
3141 *csp->http->hostport == ' ' || *csp->http->hostport == '\0')
3144 if (NULL == (p = strdup((*header)+6)))
3146 return JB_ERR_MEMORY;
3149 if (NULL == (q = strdup(p)))
3152 return JB_ERR_MEMORY;
3155 freez(csp->http->hostport);
3156 csp->http->hostport = p;
3157 freez(csp->http->host);
3158 csp->http->host = q;
3159 q = strchr(csp->http->host, ':');
3162 /* Terminate hostname and evaluate port string */
3164 csp->http->port = atoi(q);
3168 csp->http->port = csp->http->ssl ? 443 : 80;
3171 log_error(LOG_LEVEL_HEADER, "New host and port from Host field: %s = %s:%d",
3172 csp->http->hostport, csp->http->host, csp->http->port);
3175 /* Signal client_host_adder() to return right away */
3176 csp->flags |= CSP_FLAG_HOST_HEADER_IS_SET;
3182 /*********************************************************************
3184 * Function : client_if_modified_since
3186 * Description : Remove or modify the If-Modified-Since header.
3189 * 1 : csp = Current client state (buffers, headers, etc...)
3190 * 2 : header = On input, pointer to header to modify.
3191 * On output, pointer to the modified header, or NULL
3192 * to remove the header. This function frees the
3193 * original string if necessary.
3195 * Returns : JB_ERR_OK on success, or
3196 * JB_ERR_MEMORY on out-of-memory error.
3198 *********************************************************************/
3199 static jb_err client_if_modified_since(struct client_state *csp, char **header)
3202 #ifdef HAVE_GMTIME_R
3205 struct tm *timeptr = NULL;
3210 if ( 0 == strcmpic(*header, "If-Modified-Since: Wed, 08 Jun 1955 12:00:00 GMT"))
3213 * The client got an error message because of a temporary problem,
3214 * the problem is gone and the client now tries to revalidate our
3215 * error message on the real server. The revalidation would always
3216 * end with the transmission of the whole document and there is
3217 * no need to expose the bogus If-Modified-Since header.
3219 log_error(LOG_LEVEL_HEADER, "Crunching useless If-Modified-Since header.");
3222 else if (csp->action->flags & ACTION_HIDE_IF_MODIFIED_SINCE)
3224 newval = csp->action->string[ACTION_STRING_IF_MODIFIED_SINCE];
3226 if ((0 == strcmpic(newval, "block")))
3228 log_error(LOG_LEVEL_HEADER, "Crunching %s", *header);
3231 else /* add random value */
3233 const char *header_time = *header + sizeof("If-Modified-Since:");
3235 if (JB_ERR_OK != parse_header_time(header_time, &tm))
3237 log_error(LOG_LEVEL_HEADER, "Couldn't parse: %s in %s (crunching!)", header_time, *header);
3242 long int hours, minutes, seconds;
3243 long int rtime = strtol(newval, &endptr, 0);
3244 const int negative_range = (rtime < 0);
3248 log_error(LOG_LEVEL_HEADER, "Randomizing: %s (random range: %d minut%s)",
3249 *header, rtime, (rtime == 1 || rtime == -1) ? "e": "es");
3255 rtime = pick_from_range(rtime);
3259 log_error(LOG_LEVEL_ERROR, "Random range is 0. Assuming time transformation test.",
3262 tm += rtime * (negative_range ? -1 : 1);
3263 #ifdef HAVE_GMTIME_R
3264 timeptr = gmtime_r(&tm, &gmt);
3265 #elif defined(MUTEX_LOCKS_AVAILABLE)
3266 privoxy_mutex_lock(&gmtime_mutex);
3267 timeptr = gmtime(&tm);
3268 privoxy_mutex_unlock(&gmtime_mutex);
3270 timeptr = gmtime(&tm);
3272 if ((NULL == timeptr) || !strftime(newheader,
3273 sizeof(newheader), "%a, %d %b %Y %H:%M:%S GMT", timeptr))
3275 log_error(LOG_LEVEL_ERROR,
3276 "Randomizing '%s' failed. Crunching the header without replacement.",
3283 *header = strdup("If-Modified-Since: ");
3284 string_append(header, newheader);
3286 if (*header == NULL)
3288 log_error(LOG_LEVEL_HEADER, "Insufficient memory, header crunched without replacement.");
3289 return JB_ERR_MEMORY;
3292 hours = rtime / 3600;
3293 minutes = rtime / 60 % 60;
3294 seconds = rtime % 60;
3296 log_error(LOG_LEVEL_HEADER,
3297 "Randomized: %s (%s %d hou%s %d minut%s %d second%s",
3298 *header, (negative_range) ? "subtracted" : "added", hours,
3299 (hours == 1) ? "r" : "rs", minutes, (minutes == 1) ? "e" : "es",
3300 seconds, (seconds == 1) ? ")" : "s)");
3309 /*********************************************************************
3311 * Function : client_if_none_match
3313 * Description : Remove the If-None-Match header.
3316 * 1 : csp = Current client state (buffers, headers, etc...)
3317 * 2 : header = On input, pointer to header to modify.
3318 * On output, pointer to the modified header, or NULL
3319 * to remove the header. This function frees the
3320 * original string if necessary.
3322 * Returns : JB_ERR_OK on success, or
3323 * JB_ERR_MEMORY on out-of-memory error.
3325 *********************************************************************/
3326 static jb_err client_if_none_match(struct client_state *csp, char **header)
3328 if (csp->action->flags & ACTION_CRUNCH_IF_NONE_MATCH)
3330 log_error(LOG_LEVEL_HEADER, "Crunching %s", *header);
3338 /*********************************************************************
3340 * Function : client_x_filter
3342 * Description : Disables filtering if the client set "X-Filter: No".
3343 * Called from `sed'.
3346 * 1 : csp = Current client state (buffers, headers, etc...)
3347 * 2 : header = On input, pointer to header to modify.
3348 * On output, pointer to the modified header, or NULL
3349 * to remove the header. This function frees the
3350 * original string if necessary.
3352 * Returns : JB_ERR_OK on success
3354 *********************************************************************/
3355 jb_err client_x_filter(struct client_state *csp, char **header)
3357 if ( 0 == strcmpic(*header, "X-Filter: No"))
3359 if (!(csp->config->feature_flags & RUNTIME_FEATURE_HTTP_TOGGLE))
3361 log_error(LOG_LEVEL_INFO, "Ignored the client's request to fetch without filtering.");
3365 if (csp->action->flags & ACTION_FORCE_TEXT_MODE)
3367 log_error(LOG_LEVEL_HEADER,
3368 "force-text-mode overruled the client's request to fetch without filtering!");
3372 csp->content_type = CT_TABOO; /* XXX: This hack shouldn't be necessary */
3373 csp->flags |= CSP_FLAG_NO_FILTERING;
3374 log_error(LOG_LEVEL_HEADER, "Accepted the client's request to fetch without filtering.");
3376 log_error(LOG_LEVEL_HEADER, "Crunching %s", *header);
3384 /*********************************************************************
3386 * Function : client_range
3388 * Description : Removes Range, Request-Range and If-Range headers if
3389 * content filtering is enabled. If the client's version
3390 * of the document has been altered by Privoxy, the server
3391 * could interpret the range differently than the client
3392 * intended in which case the user could end up with
3393 * corrupted content.
3396 * 1 : csp = Current client state (buffers, headers, etc...)
3397 * 2 : header = On input, pointer to header to modify.
3398 * On output, pointer to the modified header, or NULL
3399 * to remove the header. This function frees the
3400 * original string if necessary.
3402 * Returns : JB_ERR_OK
3404 *********************************************************************/
3405 static jb_err client_range(struct client_state *csp, char **header)
3407 if (content_filters_enabled(csp->action))
3409 log_error(LOG_LEVEL_HEADER, "Content filtering is enabled."
3410 " Crunching: \'%s\' to prevent range-mismatch problems.", *header);
3417 /* the following functions add headers directly to the header list */
3419 /*********************************************************************
3421 * Function : client_host_adder
3423 * Description : Adds the Host: header field if it is missing.
3424 * Called from `sed'.
3427 * 1 : csp = Current client state (buffers, headers, etc...)
3429 * Returns : JB_ERR_OK on success, or
3430 * JB_ERR_MEMORY on out-of-memory error.
3432 *********************************************************************/
3433 static jb_err client_host_adder(struct client_state *csp)
3438 if (csp->flags & CSP_FLAG_HOST_HEADER_IS_SET)
3440 /* Header already set by the client, nothing to do. */
3444 if ( !csp->http->hostport || !*(csp->http->hostport))
3446 /* XXX: When does this happen and why is it OK? */
3447 log_error(LOG_LEVEL_INFO, "Weirdness in client_host_adder detected and ignored.");
3452 * remove 'user:pass@' from 'proto://user:pass@host'
3454 if ( (p = strchr( csp->http->hostport, '@')) != NULL )
3460 p = csp->http->hostport;
3463 /* XXX: Just add it, we already made sure that it will be unique */
3464 log_error(LOG_LEVEL_HEADER, "addh-unique: Host: %s", p);
3465 err = enlist_unique_header(csp->headers, "Host", p);
3471 /*********************************************************************
3473 * Function : client_xtra_adder
3475 * Description : Used in the add_client_headers list. Called from `sed'.
3478 * 1 : csp = Current client state (buffers, headers, etc...)
3480 * Returns : JB_ERR_OK on success, or
3481 * JB_ERR_MEMORY on out-of-memory error.
3483 *********************************************************************/
3484 static jb_err client_xtra_adder(struct client_state *csp)
3486 struct list_entry *lst;
3489 for (lst = csp->action->multi[ACTION_MULTI_ADD_HEADER]->first;
3490 lst ; lst = lst->next)
3492 log_error(LOG_LEVEL_HEADER, "addh: %s", lst->str);
3493 err = enlist(csp->headers, lst->str);
3505 /*********************************************************************
3507 * Function : client_x_forwarded_for_adder
3509 * Description : Used in the add_client_headers list. Called from `sed'.
3512 * 1 : csp = Current client state (buffers, headers, etc...)
3514 * Returns : JB_ERR_OK on success, or
3515 * JB_ERR_MEMORY on out-of-memory error.
3517 *********************************************************************/
3518 static jb_err client_x_forwarded_for_adder(struct client_state *csp)
3520 char *header = NULL;
3523 if (!((csp->action->flags & ACTION_CHANGE_X_FORWARDED_FOR)
3524 && (0 == strcmpic(csp->action->string[ACTION_STRING_CHANGE_X_FORWARDED_FOR], "add")))
3525 || (csp->flags & CSP_FLAG_X_FORWARDED_FOR_APPENDED))
3528 * If we aren't adding X-Forwarded-For headers,
3529 * or we already appended an existing X-Forwarded-For
3530 * header, there's nothing left to do here.
3535 header = strdup("X-Forwarded-For: ");
3536 string_append(&header, csp->ip_addr_str);
3540 return JB_ERR_MEMORY;
3543 log_error(LOG_LEVEL_HEADER, "addh: %s", header);
3544 err = enlist(csp->headers, header);
3551 /*********************************************************************
3553 * Function : server_connection_adder
3555 * Description : Adds an appropriate "Connection:" header to csp->headers
3556 * unless the header was already present. Called from `sed'.
3559 * 1 : csp = Current client state (buffers, headers, etc...)
3561 * Returns : JB_ERR_OK on success, or
3562 * JB_ERR_MEMORY on out-of-memory error.
3564 *********************************************************************/
3565 static jb_err server_connection_adder(struct client_state *csp)
3567 const unsigned int flags = csp->flags;
3568 const char *response_status_line = csp->headers->first->str;
3569 static const char connection_close[] = "Connection: close";
3571 if ((flags & CSP_FLAG_CLIENT_HEADER_PARSING_DONE)
3572 && (flags & CSP_FLAG_SERVER_CONNECTION_HEADER_SET))
3578 * XXX: if we downgraded the response, this check will fail.
3580 if ((csp->config->feature_flags &
3581 RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE)
3582 && (NULL != response_status_line)
3583 && !strncmpic(response_status_line, "HTTP/1.1", 8)
3584 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
3585 && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED)
3589 log_error(LOG_LEVEL_HEADER, "A HTTP/1.1 response "
3590 "without Connection header implies keep-alive.");
3591 csp->flags |= CSP_FLAG_SERVER_CONNECTION_KEEP_ALIVE;
3595 log_error(LOG_LEVEL_HEADER, "Adding: %s", connection_close);
3597 return enlist(csp->headers, connection_close);
3601 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
3602 /*********************************************************************
3604 * Function : server_proxy_connection_adder
3606 * Description : Adds a "Proxy-Connection: keep-alive" header to
3607 * csp->headers if the client asked for keep-alive.
3608 * XXX: We should reuse existent ones.
3611 * 1 : csp = Current client state (buffers, headers, etc...)
3613 * Returns : JB_ERR_OK on success, or
3614 * JB_ERR_MEMORY on out-of-memory error.
3616 *********************************************************************/
3617 static jb_err server_proxy_connection_adder(struct client_state *csp)
3619 static const char proxy_connection_header[] = "Proxy-Connection: keep-alive";
3620 jb_err err = JB_ERR_OK;
3622 if ((csp->flags & CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE)
3623 && !(csp->flags & CSP_FLAG_SERVER_SOCKET_TAINTED)
3624 && !(csp->flags & CSP_FLAG_SERVER_PROXY_CONNECTION_HEADER_SET))
3626 log_error(LOG_LEVEL_HEADER, "Adding: %s", proxy_connection_header);
3627 err = enlist(csp->headers, proxy_connection_header);
3632 #endif /* FEATURE_CONNECTION_KEEP_ALIVE */
3635 /*********************************************************************
3637 * Function : client_connection_header_adder
3639 * Description : Adds a proper "Connection:" header to csp->headers
3640 * unless the header was already present. Called from `sed'.
3643 * 1 : csp = Current client state (buffers, headers, etc...)
3645 * Returns : JB_ERR_OK on success, or
3646 * JB_ERR_MEMORY on out-of-memory error.
3648 *********************************************************************/
3649 static jb_err client_connection_header_adder(struct client_state *csp)
3651 static const char connection_close[] = "Connection: close";
3653 if (!(csp->flags & CSP_FLAG_CLIENT_HEADER_PARSING_DONE)
3654 && (csp->flags & CSP_FLAG_CLIENT_CONNECTION_HEADER_SET))
3659 #ifdef FEATURE_CONNECTION_KEEP_ALIVE
3660 if ((csp->config->feature_flags & RUNTIME_FEATURE_CONNECTION_KEEP_ALIVE)
3661 && (csp->http->ssl == 0)
3662 && !strcmpic(csp->http->ver, "HTTP/1.1"))
3664 csp->flags |= CSP_FLAG_CLIENT_CONNECTION_KEEP_ALIVE;
3667 #endif /* FEATURE_CONNECTION_KEEP_ALIVE */
3669 log_error(LOG_LEVEL_HEADER, "Adding: %s", connection_close);
3671 return enlist(csp->headers, connection_close);
3675 /*********************************************************************
3677 * Function : server_http
3679 * Description : - Save the HTTP Status into csp->http->status
3680 * - Set CT_TABOO to prevent filtering if the answer
3681 * is a partial range (HTTP status 206)
3682 * - Rewrite HTTP/1.1 answers to HTTP/1.0 if +downgrade
3686 * 1 : csp = Current client state (buffers, headers, etc...)
3687 * 2 : header = On input, pointer to header to modify.
3688 * On output, pointer to the modified header, or NULL
3689 * to remove the header. This function frees the
3690 * original string if necessary.
3692 * Returns : JB_ERR_OK on success, or
3693 * JB_ERR_MEMORY on out-of-memory error.
3695 *********************************************************************/
3696 static jb_err server_http(struct client_state *csp, char **header)
3698 sscanf(*header, "HTTP/%*d.%*d %d", &(csp->http->status));
3699 if (csp->http->status == 206)
3701 csp->content_type = CT_TABOO;
3704 if ((csp->action->flags & ACTION_DOWNGRADE) != 0)
3706 /* XXX: Should we do a real validity check here? */
3707 if (strlen(*header) > 8)
3710 log_error(LOG_LEVEL_HEADER, "Downgraded answer to HTTP/1.0");
3715 * XXX: Should we block the request or
3716 * enlist a valid status code line here?
3718 log_error(LOG_LEVEL_INFO, "Malformed server response detected. "
3719 "Downgrading to HTTP/1.0 impossible.");
3727 /*********************************************************************
3729 * Function : server_set_cookie
3731 * Description : Handle the server "cookie" header properly.
3732 * Crunch, accept or rewrite it to a session cookie.
3733 * Called from `sed'.
3735 * TODO: Allow the user to specify a new expiration
3736 * time to cause the cookie to expire even before the
3737 * browser is closed.
3740 * 1 : csp = Current client state (buffers, headers, etc...)
3741 * 2 : header = On input, pointer to header to modify.
3742 * On output, pointer to the modified header, or NULL
3743 * to remove the header. This function frees the
3744 * original string if necessary.
3746 * Returns : JB_ERR_OK on success, or
3747 * JB_ERR_MEMORY on out-of-memory error.
3749 *********************************************************************/
3750 static jb_err server_set_cookie(struct client_state *csp, char **header)
3757 if ((csp->action->flags & ACTION_NO_COOKIE_SET) != 0)
3759 log_error(LOG_LEVEL_HEADER, "Crunching incoming cookie: %s", *header);
3762 else if ((csp->action->flags & ACTION_NO_COOKIE_KEEP) != 0)
3764 /* Flag whether or not to log a message */
3767 /* A variable to store the tag we're working on */
3770 /* Skip "Set-Cookie:" (11 characters) in header */
3771 cur_tag = *header + 11;
3773 /* skip whitespace between "Set-Cookie:" and value */
3774 while (*cur_tag && ijb_isspace(*cur_tag))
3779 /* Loop through each tag in the cookie */
3783 char *next_tag = strchr(cur_tag, ';');
3784 if (next_tag != NULL)
3786 /* Skip the ';' character itself */
3789 /* skip whitespace ";" and start of tag */
3790 while (*next_tag && ijb_isspace(*next_tag))
3797 /* "Next tag" is the end of the string */
3798 next_tag = cur_tag + strlen(cur_tag);
3802 * Check the expiration date to see
3803 * if the cookie is still valid, if yes,
3804 * rewrite it to a session cookie.
3806 if ((strncmpic(cur_tag, "expires=", 8) == 0) && *(cur_tag + 8))
3808 char *expiration_date = cur_tag + 8; /* Skip "[Ee]xpires=" */
3810 if ((expiration_date[0] == '"')
3811 && (expiration_date[1] != '\0'))
3814 * Skip quotation mark. RFC 2109 10.1.2 seems to hint
3815 * that the expiration date isn't supposed to be quoted,
3816 * but some servers do it anyway.
3821 /* Did we detect the date properly? */
3822 if (JB_ERR_OK != parse_header_time(expiration_date, &cookie_time))
3825 * Nope, treat it as if it was still valid.
3827 * XXX: Should we remove the whole cookie instead?
3829 log_error(LOG_LEVEL_ERROR,
3830 "Can't parse \'%s\', send by %s. Unsupported time format?", cur_tag, csp->http->url);
3831 string_move(cur_tag, next_tag);
3837 * Yes. Check if the cookie is still valid.
3839 * If the cookie is already expired it's probably
3840 * a delete cookie and even if it isn't, the browser
3841 * will discard it anyway.
3845 * XXX: timegm() isn't available on some AmigaOS
3846 * versions and our replacement doesn't work.
3848 * Our options are to either:
3850 * - disable session-cookies-only completely if timegm
3853 * - to simply remove all expired tags, like it has
3854 * been done until Privoxy 3.0.6 and to live with
3855 * the consequence that it can cause login/logout
3856 * problems on servers that don't validate their
3857 * input properly, or
3859 * - to replace it with mktime in which
3860 * case there is a slight chance of valid cookies
3861 * passing as already expired.
3863 * This is the way it's currently done and it's not
3864 * as bad as it sounds. If the missing GMT offset is
3865 * enough to change the result of the expiration check
3866 * the cookie will be only valid for a few hours
3867 * anyway, which in many cases will be shorter
3868 * than a browser session.
3870 if (cookie_time - now < 0)
3872 log_error(LOG_LEVEL_HEADER,
3873 "Cookie \'%s\' is already expired and can pass unmodified.", *header);
3874 /* Just in case some clown sets more then one expiration date */
3880 * Still valid, delete expiration date by copying
3881 * the rest of the string over it.
3883 string_move(cur_tag, next_tag);
3885 /* That changed the header, need to issue a log message */
3889 * Note that the next tag has now been moved to *cur_tag,
3890 * so we do not need to update the cur_tag pointer.
3898 /* Move on to next cookie tag */
3905 assert(NULL != *header);
3906 log_error(LOG_LEVEL_HEADER, "Cookie rewritten to a temporary one: %s",
3915 #ifdef FEATURE_FORCE_LOAD
3916 /*********************************************************************
3918 * Function : strclean
3920 * Description : In-Situ-Eliminate all occurrences of substring in
3924 * 1 : string = string to clean
3925 * 2 : substring = substring to eliminate
3927 * Returns : Number of eliminations
3929 *********************************************************************/
3930 int strclean(char *string, const char *substring)
3936 len = strlen(substring);
3938 while((pos = strstr(string, substring)) != NULL)
3945 while (*p++ != '\0');
3952 #endif /* def FEATURE_FORCE_LOAD */
3955 /*********************************************************************
3957 * Function : parse_header_time
3959 * Description : Parses time formats used in HTTP header strings
3960 * to get the numerical respresentation.
3963 * 1 : header_time = HTTP header time as string.
3964 * 2 : result = storage for header_time in seconds
3966 * Returns : JB_ERR_OK if the time format was recognized, or
3967 * JB_ERR_PARSE otherwise.
3969 *********************************************************************/
3970 static jb_err parse_header_time(const char *header_time, time_t *result)
3974 * Checking for two-digit years first in an
3975 * attempt to work around GNU libc's strptime()
3976 * reporting negative year values when using %Y.
3978 static const char * const time_formats[] = {
3979 /* Tue, 02-Jun-37 20:00:00 */
3980 "%a, %d-%b-%y %H:%M:%S",
3981 /* Tue, 02 Jun 2037 20:00:00 */
3982 "%a, %d %b %Y %H:%M:%S",
3983 /* Tue, 02-Jun-2037 20:00:00 */
3984 "%a, %d-%b-%Y %H:%M:%S",
3985 /* Tuesday, 02-Jun-2037 20:00:00 */
3986 "%A, %d-%b-%Y %H:%M:%S",
3987 /* Tuesday Jun 02 20:00:00 2037 */
3988 "%A %b %d %H:%M:%S %Y"
3992 for (i = 0; i < SZ(time_formats); i++)
3995 * Zero out gmt to prevent time zone offsets.
3996 * Documented to be required for GNU libc.
3998 memset(&gmt, 0, sizeof(gmt));
4000 if (NULL != strptime(header_time, time_formats[i], &gmt))
4002 /* Sanity check for GNU libc. */
4003 if (gmt.tm_year < 0)
4005 log_error(LOG_LEVEL_HEADER,
4006 "Failed to parse '%s' using '%s'. Moving on.",
4007 header_time, time_formats[i]);
4010 *result = timegm(&gmt);
4015 return JB_ERR_PARSE;
4020 /*********************************************************************
4022 * Function : get_destination_from_headers
4024 * Description : Parse the "Host:" header to get the request's destination.
4025 * Only needed if the client's request was forcefully
4026 * redirected into Privoxy.
4028 * Code mainly copied from client_host() which is currently
4029 * run too late for this purpose.
4032 * 1 : headers = List of headers (one of them hopefully being
4033 * the "Host:" header)
4034 * 2 : http = storage for the result (host, port and hostport).
4036 * Returns : JB_ERR_MEMORY in case of memory problems,
4037 * JB_ERR_PARSE if the host header couldn't be found,
4038 * JB_ERR_OK otherwise.
4040 *********************************************************************/
4041 jb_err get_destination_from_headers(const struct list *headers, struct http_request *http)
4047 host = get_header_value(headers, "Host:");
4051 log_error(LOG_LEVEL_ERROR, "No \"Host:\" header found.");
4052 return JB_ERR_PARSE;
4058 log_error(LOG_LEVEL_ERROR, "Out of memory while parsing \"Host:\" header");
4059 return JB_ERR_MEMORY;
4062 if (NULL == (q = strdup(p)))
4065 log_error(LOG_LEVEL_ERROR, "Out of memory while parsing \"Host:\" header");
4066 return JB_ERR_MEMORY;
4069 freez(http->hostport);
4073 q = strchr(http->host, ':');
4076 /* Terminate hostname and evaluate port string */
4078 http->port = atoi(q);
4082 http->port = http->ssl ? 443 : 80;
4085 /* Rebuild request URL */
4087 http->url = strdup(http->ssl ? "https://" : "http://");
4088 string_append(&http->url, http->hostport);
4089 string_append(&http->url, http->path);
4090 if (http->url == NULL)
4092 return JB_ERR_MEMORY;
4095 log_error(LOG_LEVEL_HEADER, "Destination extracted from \"Host:\" header. New request URL: %s",
4103 /*********************************************************************
4105 * Function : create_forged_referrer
4107 * Description : Helper for client_referrer to forge a referer as
4108 * 'http://[hostname:port/' to fool stupid
4109 * checks for in-site links
4112 * 1 : header = Pointer to header pointer
4113 * 2 : hostport = Host and optionally port as string
4115 * Returns : JB_ERR_OK in case of success, or
4116 * JB_ERR_MEMORY in case of memory problems.
4118 *********************************************************************/
4119 static jb_err create_forged_referrer(char **header, const char *hostport)
4121 assert(NULL == *header);
4123 *header = strdup("Referer: http://");
4124 string_append(header, hostport);
4125 string_append(header, "/");
4127 if (NULL == *header)
4129 return JB_ERR_MEMORY;
4132 log_error(LOG_LEVEL_HEADER, "Referer forged to: %s", *header);
4139 /*********************************************************************
4141 * Function : create_fake_referrer
4143 * Description : Helper for client_referrer to create a fake referrer
4144 * based on a string supplied by the user.
4147 * 1 : header = Pointer to header pointer
4148 * 2 : hosthost = Referrer to fake
4150 * Returns : JB_ERR_OK in case of success, or
4151 * JB_ERR_MEMORY in case of memory problems.
4153 *********************************************************************/
4154 static jb_err create_fake_referrer(char **header, const char *fake_referrer)
4156 assert(NULL == *header);
4158 if ((0 != strncmpic(fake_referrer, "http://", 7)) && (0 != strncmpic(fake_referrer, "https://", 8)))
4160 log_error(LOG_LEVEL_HEADER,
4161 "Parameter: +hide-referrer{%s} is a bad idea, but I don't care.", fake_referrer);
4163 *header = strdup("Referer: ");
4164 string_append(header, fake_referrer);
4166 if (NULL == *header)
4168 return JB_ERR_MEMORY;
4171 log_error(LOG_LEVEL_HEADER, "Referer replaced with: %s", *header);
4178 /*********************************************************************
4180 * Function : handle_conditional_hide_referrer_parameter
4182 * Description : Helper for client_referrer to crunch or forge
4183 * the referrer header if the host has changed.
4186 * 1 : header = Pointer to header pointer
4187 * 2 : host = The target host (may include the port)
4188 * 3 : parameter_conditional_block = Boolean to signal
4189 * if we're in conditional-block mode. If not set,
4190 * we're in conditional-forge mode.
4192 * Returns : JB_ERR_OK in case of success, or
4193 * JB_ERR_MEMORY in case of memory problems.
4195 *********************************************************************/
4196 static jb_err handle_conditional_hide_referrer_parameter(char **header,
4197 const char *host, const int parameter_conditional_block)
4199 char *referer = strdup(*header);
4200 const size_t hostlength = strlen(host);
4201 const char *referer_url = NULL;
4203 if (NULL == referer)
4206 return JB_ERR_MEMORY;
4209 /* referer begins with 'Referer: http[s]://' */
4210 if ((hostlength+17) < strlen(referer))
4213 * Shorten referer to make sure the referer is blocked
4214 * if www.example.org/www.example.com-shall-see-the-referer/
4215 * links to www.example.com/
4217 referer[hostlength+17] = '\0';
4219 referer_url = strstr(referer, "http://");
4220 if ((NULL == referer_url) || (NULL == strstr(referer_url, host)))
4222 /* Host has changed, Referer is invalid or a https URL. */
4223 if (parameter_conditional_block)
4225 log_error(LOG_LEVEL_HEADER, "New host is: %s. Crunching %s!", host, *header);
4232 return create_forged_referrer(header, host);
4242 /*********************************************************************
4244 * Function : create_content_length_header
4246 * Description : Creates a Content-Length header.
4249 * 1 : content_length = The content length to be used in the header.
4250 * 2 : header = Allocated space to safe the header.
4251 * 3 : buffer_length = The length of the allocated space.
4255 *********************************************************************/
4256 static void create_content_length_header(unsigned long long content_length,
4257 char *header, size_t buffer_length)
4259 snprintf(header, buffer_length, "Content-Length: %llu", content_length);