3 ################################################################################
7 # Generates an exclude file that can be passed to runtests.pl to skip certain
8 # tests that aren't expected to work when run through Privoxy.
10 # Copyright (c) 2012-2022 Fabian Keil <fk@fabiankeil.de>
12 # Permission to use, copy, modify, and distribute this software for any
13 # purpose with or without fee is hereby granted, provided that the above
14 # copyright notice and this permission notice appear in all copies.
16 # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
17 # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
18 # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
19 # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
20 # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
21 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
22 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
23 ################################################################################
31 for my $testnum (1..3500) {
34 $why = "Expected to fail. Tab in cookie that Privoxy converts to a space which the test does not expect.";
35 } elsif ($testnum eq 16 or
38 $why = "Not supposed to work with Privoxy. Expected forwarding failure. Workaround probably possible.";
39 } elsif ($testnum eq 17) {
40 $why = "Not supposed to work with Privoxy. Invalid METHODs get rejected.";
41 } elsif ($testnum eq 19 or
43 $why = "Not supposed to work with Privoxy. Tests behaviour with non-existing server and doesn't deal with error messages coming from a proxy.";
44 } elsif ($testnum eq 30 or
48 $why = "Expected to fail with Privoxy. In case of missing server headers Privoxy generates an error message the test doesn't expect.";
49 } elsif ($testnum eq 31 or $testnum eq 1105 or $testnum eq 1160) {
50 $why = "Expected to fail. Privoxy normalizes white-space in a cookie.";
51 } elsif ($testnum eq 46) {
52 $why = "Invalid URL and use of --resolv.";
53 } elsif ($testnum eq 59) {
54 $why = "Invalid URL gets rejected by Privoxy which the test can't handle.";
55 } elsif ($testnum eq 129) {
56 $why = "Invalid HTTP version. Privoxy downgrades it to 1.1.";
57 } elsif ($testnum eq 187) {
58 $why = "Expected to fail. Test doesn't deal with Privoxy's error message in case of invalid URLs.";
59 } elsif ($testnum eq 207) {
60 $why = "Expected to fail. Test doesn't handle Privoxy's error message. Privoxy doesn't behave correctly, though.";
61 } elsif ($testnum eq 260) {
62 $why = "Known to fail. Looks like a curl bug. The URL passed to Privoxy is invalid but the test expect a valid one when not using a proxy";
63 } elsif ($testnum eq 262) {
64 $why = "Not supposed to work with Privoxy. Privoxy doesn't support nul bytes in headers and neither does the spec.";
65 } elsif ($testnum eq 266 or $testnum eq 1116 or $testnum eq 1540) {
66 $why = "Known to fail. Uses chunk trailers which Privoxy currently doesn't support.";
67 } elsif ($testnum eq 309) {
68 $why = "Known to fail. Uses https and test does not expect the 'Connection established' response from Privoxy";
69 } elsif ($testnum eq 339 or $testnum eq 347 or $testnum eq 1591) {
70 $why = "Chunked transfer with trailers which Privoxy does not understand. Needs investigating.";
71 } elsif ($testnum eq 1052) {
72 $why = "Expected to fail. Connection header expected in one response but not in the other. Not yet covered by runtests.pl's proxy mode.";
73 } elsif ($testnum eq 1118) {
74 $why = "Expected to fail. Looks like a curl bug although Privoxy's behaviour seems subobtimal as well.";
75 } elsif ($testnum eq 1310) {
76 $why = "Known to fail. NTLM-related. Cause not properly diagnosed yet. Privoxy's behaviour seems reasonable.";
77 } elsif ($testnum eq 155) {
78 $why = "Known to fail. Not yet analyzed.";
79 } elsif ($testnum eq 158 or $testnum eq 246 or $testnum eq 565 or $testnum eq 579) {
80 $why = "Known to fail. Not properly analyzed. Looks like Privoxy's continue hack is insufficient.";
81 } elsif ($testnum eq 412 or $testnum eq 413) {
82 $why = "Known to fail as curl is tunneling the request even though it's vanilla HTTP.";
83 } elsif ($testnum eq 415) {
84 $why = "Known to fail. Control code in Content-Length header.";
85 } elsif ($testnum eq 435) {
86 $why = "Expected to fail. Uses %{remote_port} and expects the port of the server and not the one from Privoxy.";
87 } elsif ($testnum eq 507) {
88 $why = "Expected to fail. DNS failures cause a Privoxy error message the test doesn't handle.";
89 } elsif ($testnum eq 501) {
90 $why = "Not relevant for a proxy.";
91 } elsif ($testnum eq 530 or
93 $why = "Known to fail. Test server expects pipelined requests and doesn't respond otherwise.";
94 } elsif ($testnum eq 556) {
95 $why = "Expected to fail. Uses HTTP/1.2 which Privoxy rejects as invalid.";
96 } elsif ($testnum eq 581) {
97 $why = "Expected to fail. Privoxy removes second Content-Type header.";
98 } elsif ($testnum eq 587 or $testnum eq 644) {
99 $why = "Expected to fail. POST request doesn't make it to the server. Needs investigating.";
100 } elsif ($testnum eq 655) {
101 $why = "Expected to fail. Uses tool. Failure reason not yet analyzed";
102 } elsif ($testnum eq 970 or $testnum eq 972) {
103 $why = "Expected to fail. Privoxy adds a Proxy-Connection header which results in a modified num_headers value in the JSON output";
104 } elsif ($testnum eq 1074) {
105 $why = "Expected to fail. Privoxy doesn't downgrade the forwarded request and doesn't have ".
106 "to as long as the client is treated like a HTTP/1.0 client. Needs double-checking.";
107 } elsif ($testnum eq 1144) {
108 $why = "Expected to fail. Server response is invalid and results in 502 message from Privoxy";
109 } elsif ($testnum eq 1147) {
110 $why = "Expected to fail. Privoxy merges a two-line cookie into a one line cookie.";
111 } elsif ($testnum eq 1151) {
112 $why = "Expected to fail. Large cookies that don't make it to the cookie file. Needs investigating.";
113 } elsif ($testnum eq 1188) {
114 $why = "Expected to fail. Relies on a connection failure which results in a Privoxy error message the test does not expect";
115 } elsif ($testnum eq 1223) {
116 $why = "Expected to fail. Tests remote address which doesn't work with proxies.";
117 } elsif ($testnum eq 1274) {
118 $why = "Expected to fail. Privoxy unfolds the folded headers which the test does not expect.";
119 } elsif ($testnum eq 1433) {
120 $why = "Expected to fail. Privoxy will enforce a valid HTTP version number";
121 } elsif ($testnum eq 1506 or $testnum eq 1510) {
122 $why = "Expected to fail when using a proxy. Hardcoded addresses in expected output.";
123 } elsif ($testnum eq 1156) {
124 $why = "Expected to fail as it relies on Range requests making it to the server.";
125 } elsif ($testnum eq 1164 or $testnum eq 1172 or $testnum eq 1174) {
126 $why = "Expected to fail as Privoxy does not support HTTP/0.9.";
127 } elsif ($testnum eq 1292) {
128 $why = "Expected to fail as Privoxy replaces the empty Host header.";
129 } elsif ($testnum eq 1533) {
130 $why = "Sends an invalid method. Needs investigating.";
131 } elsif ($testnum eq 1543) {
132 $why = "Expected to fail as the URL contains spaces. XXX: Looks like a curl bug that should be investigated.";
133 } elsif ($testnum eq 1556) {
134 $why = "Known to fail. Body value changes from 100008 to 100009. Needs investigating.";
135 } elsif ($testnum eq 1671) {
136 $why = "Known to fail as curl adds a Proxy-Connection header to the JSON output which the test doesn't expect.";
137 } elsif ($testnum eq 1915) {
138 $why = "Known to fail. Uses tool that doesn't expect a proxy.";
139 } elsif ($testnum eq 1933) {
140 $why = "Known to fail. Modified signature in Authorization header. Needs investigating";
141 } elsif ($testnum eq 2032 or $testnum eq 2033) {
142 $why = "Known to fail due to a limitation of the test which doesn't properly deal with interleaved output from two parallel connections";
143 } elsif ($testnum eq 2049 or $testnum eq 2052 or $testnum eq 2053 or $testnum eq 2054) {
144 $why = "Uses --connect-to. Need investigating.";
145 } elsif ($testnum eq 2082 or $testnum eq 2084 or $testnum eq 2085) {
146 $why = "Known to fail. Uses %HTTPPORT and does not expect Privoxy's port but the remote one.";
147 } elsif ($testnum eq 96) {
148 $why = "Test 96 is incomplete";
149 } elsif ($testnum eq 1901 or $testnum eq 1902 or $testnum eq 1903) {
150 $why = "Known to fail due to different response orders.";
151 } elsif ($testnum eq 2100) {
152 $why = "Known to fail. Use DNS-over-HTTP.";
153 } elsif ($testnum eq 3014 or $testnum eq 3015) {
154 $why = "Known to fail. Curl adds a Proxy-Connection header while test expect a certain number of headers.";
157 next unless defined $why;
159 if (exists $related_tests{$why}) {
160 $related_tests{$why} = $related_tests{$why} . ", $testnum";
162 $related_tests{$why} = "$testnum";
166 foreach my $why (keys %related_tests) {
167 printf("test:%s: %s\n", $related_tests{$why}, $why);
170 foreach my $protocol ('FTP', 'POP3', 'IMAP', 'SMTP', 'GOPHER', 'TELNET', 'FILE', 'RTSP') {
171 # Curl's behaviour when combining --proxy, -H and telnet:// seems strange and may be a bug."
172 printf("keyword:%s: Protocol %s is not supported by Privoxy.\n", $protocol, $protocol);
175 foreach my $protocol ('SOCKS4', 'SOCKS5') {
176 # Curl's behaviour when combining --proxy, -H and telnet:// seems strange and may be a bug."
177 printf("keyword:%s: Protocol %s is supported by Privoxy but the tests need additional magic before they can be used.\n", $protocol, $protocol);
180 foreach my $misc ('proxy', '--resolve', '--libcurl', 'CURLOPT_RESOLVE') {
181 printf("keyword:%s: Tests with keyword '%s' currently don't work with Privoxy as they need additional runtests.pl changes.\n", $misc, $misc);
183 printf("keyword:%s: Tests with keyword '%s' aren't expected to work with Privoxy running in a jail without IPv6 connectivity.\n", 'IPv6', 'IPv6');
185 printf("keyword:%s: Tests with keyword '%s' don't work with Privoxy as they use the OPTIONS method which is currently not properly supported (TODO #186).\n", '--request-target', '--request-target');
187 printf("keyword:%s: Tests with keyword '%s' obviously should be skipped. Check the full keyword for details.\n", 'skip', 'skip');
189 foreach my $keyword ('FAILURE', 'unsupported', 'curl-config') {
190 printf("keyword:%s: Tests with keyword '%s' do not reach the proxy. Or do they?\n", $keyword, $keyword);
193 foreach my $tool ('lib517', 'lib543', 'lib543', '--manual', '--help', 'symbols-in-versions', 'memory-includes', 'unittest') {
194 printf("tool:%s: Tests with tool '%s' are not relevant for proxies.\n", $tool, $tool);