1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
|
From b30d694a027eb771c02a3db0dee0ca03ccab7377 Mon Sep 17 00:00:00 2001
From: Stefan Eissing <stefan@eissing.org>
Date: Thu, 28 Mar 2024 11:08:15 +0100
Subject: [PATCH] content_encoding: brotli and others, pass through 0-length
writes
- curl's transfer handling may write 0-length chunks at the end of the
download with an EOS flag. (HTTP/2 does this commonly)
- content encoders need to pass-through such a write and not count this
as error in case they are finished decoding
Fixes #13209
Fixes #13212
Closes #13219
---
lib/content_encoding.c | 10 +++++-----
tests/http/test_02_download.py | 13 +++++++++++++
tests/http/testenv/env.py | 7 ++++++-
tests/http/testenv/httpd.py | 20 ++++++++++++++++++++
4 files changed, 44 insertions(+), 6 deletions(-)
diff --git a/lib/content_encoding.c b/lib/content_encoding.c
index c1abf24e8c027c..8e926dd2ecd5ad 100644
--- a/lib/content_encoding.c
+++ b/lib/content_encoding.c
@@ -300,7 +300,7 @@ static CURLcode deflate_do_write(struct Curl_easy *data,
struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
/* Set the compressed input when this function is called */
@@ -457,7 +457,7 @@ static CURLcode gzip_do_write(struct Curl_easy *data,
struct zlib_writer *zp = (struct zlib_writer *) writer;
z_stream *z = &zp->z; /* zlib state structure */
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(zp->zlib_init == ZLIB_INIT_GZIP) {
@@ -669,7 +669,7 @@ static CURLcode brotli_do_write(struct Curl_easy *data,
CURLcode result = CURLE_OK;
BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(!bp->br)
@@ -762,7 +762,7 @@ static CURLcode zstd_do_write(struct Curl_easy *data,
ZSTD_outBuffer out;
size_t errorCode;
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
if(!zp->decomp) {
@@ -916,7 +916,7 @@ static CURLcode error_do_write(struct Curl_easy *data,
(void) buf;
(void) nbytes;
- if(!(type & CLIENTWRITE_BODY))
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
failf(data, "Unrecognized content encoding type. "
diff --git a/tests/http/test_02_download.py b/tests/http/test_02_download.py
index 4db9c9d36e9ed5..395fc862f2f839 100644
--- a/tests/http/test_02_download.py
+++ b/tests/http/test_02_download.py
@@ -394,6 +394,19 @@ def test_02_27_paused_no_cl(self, env: Env, httpd, nghttpx, repeat):
r = client.run(args=[url])
r.check_exit_code(0)
+ @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3'])
+ def test_02_28_get_compressed(self, env: Env, httpd, nghttpx, repeat, proto):
+ if proto == 'h3' and not env.have_h3():
+ pytest.skip("h3 not supported")
+ count = 1
+ urln = f'https://{env.authority_for(env.domain1brotli, proto)}/data-100k?[0-{count-1}]'
+ curl = CurlClient(env=env)
+ r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
+ '--compressed'
+ ])
+ r.check_exit_code(code=0)
+ r.check_response(count=count, http_status=200)
+
def check_downloads(self, client, srcfile: str, count: int,
complete: bool = True):
for i in range(count):
diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py
index a207059dcd57c5..13c5d6bd46ee57 100644
--- a/tests/http/testenv/env.py
+++ b/tests/http/testenv/env.py
@@ -129,10 +129,11 @@ def __init__(self):
self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs')
self.tld = 'http.curl.se'
self.domain1 = f"one.{self.tld}"
+ self.domain1brotli = f"brotli.one.{self.tld}"
self.domain2 = f"two.{self.tld}"
self.proxy_domain = f"proxy.{self.tld}"
self.cert_specs = [
- CertificateSpec(domains=[self.domain1, 'localhost'], key_type='rsa2048'),
+ CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost'], key_type='rsa2048'),
CertificateSpec(domains=[self.domain2], key_type='rsa2048'),
CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'),
CertificateSpec(name="clientsX", sub_specs=[
@@ -376,6 +377,10 @@ def htdocs_dir(self) -> str:
def domain1(self) -> str:
return self.CONFIG.domain1
+ @property
+ def domain1brotli(self) -> str:
+ return self.CONFIG.domain1brotli
+
@property
def domain2(self) -> str:
return self.CONFIG.domain2
diff --git a/tests/http/testenv/httpd.py b/tests/http/testenv/httpd.py
index c04c22699a62c4..b8615875a9a558 100644
--- a/tests/http/testenv/httpd.py
+++ b/tests/http/testenv/httpd.py
@@ -50,6 +50,7 @@ class Httpd:
'alias', 'env', 'filter', 'headers', 'mime', 'setenvif',
'socache_shmcb',
'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect',
+ 'brotli',
'mpm_event',
]
COMMON_MODULES_DIRS = [
@@ -203,6 +204,7 @@ def _mkpath(self, path):
def _write_config(self):
domain1 = self.env.domain1
+ domain1brotli = self.env.domain1brotli
creds1 = self.env.get_credentials(domain1)
domain2 = self.env.domain2
creds2 = self.env.get_credentials(domain2)
@@ -285,6 +287,24 @@ def _write_config(self):
f'</VirtualHost>',
f'',
])
+ # Alternate to domain1 with BROTLI compression
+ conf.extend([ # https host for domain1, h1 + h2
+ f'<VirtualHost *:{self.env.https_port}>',
+ f' ServerName {domain1brotli}',
+ f' Protocols h2 http/1.1',
+ f' SSLEngine on',
+ f' SSLCertificateFile {creds1.cert_file}',
+ f' SSLCertificateKeyFile {creds1.pkey_file}',
+ f' DocumentRoot "{self._docs_dir}"',
+ f' SetOutputFilter BROTLI_COMPRESS',
+ ])
+ conf.extend(self._curltest_conf(domain1))
+ if domain1 in self._extra_configs:
+ conf.extend(self._extra_configs[domain1])
+ conf.extend([
+ f'</VirtualHost>',
+ f'',
+ ])
conf.extend([ # https host for domain2, no h2
f'<VirtualHost *:{self.env.https_port}>',
f' ServerName {domain2}',
|
