@@ -14,6 +14,11 @@ namespace mcpp::pack::compress::codec {
1414
1515namespace {
1616
17+ // zlib's counters are `uInt` (32 bits) and `uLong` (32 bits on Windows): the
18+ // input is handed over and the output taken in pieces of at most `kZlibPiece`,
19+ // positions are kept here, so a tree over 4 GiB is one gzip member as any other.
20+ constexpr std::size_t kZlibPiece = std::size_t {1 } << 30 ;
21+
1722// zlib's deflate with `windowBits`: 15 + 16 is a gzip member, -15 is raw
1823// DEFLATE. A gzip member's header is stated rather than left to zlib, whose own
1924// header names the system zlib was built for (3 on Unix, 10 or 11 on Windows):
@@ -32,30 +37,39 @@ std::expected<std::string, std::string> zlib_deflate(std::string_view in, int le
3237 return std::unexpected (std::string (" zlib: deflateSetHeader failed" ));
3338 }
3439 std::string out;
35- out.resize (deflateBound (&s, static_cast <uLong>(in.size ())) + 64 );
36- s.next_in = reinterpret_cast <Bytef*>(const_cast <char *>(in.data ()));
37- s.avail_in = static_cast <uInt>(in.size ());
38- s.next_out = reinterpret_cast <Bytef*>(out.data ());
39- s.avail_out = static_cast <uInt>(out.size ());
40- const int rc = ::deflate (&s, Z_FINISH );
41- const auto written = s.total_out ;
40+ out.resize (std::min (in.size (), kZlibPiece ) + in.size () / 1000 + 1024 );
41+ std::size_t fed = 0 , written = 0 ;
42+ int rc = Z_OK ;
43+ while (rc != Z_STREAM_END ) {
44+ if (s.avail_in == 0 && fed < in.size ()) {
45+ const auto piece = std::min (in.size () - fed, kZlibPiece );
46+ s.next_in = reinterpret_cast <Bytef*>(const_cast <char *>(in.data () + fed));
47+ s.avail_in = static_cast <uInt>(piece);
48+ fed += piece;
49+ }
50+ if (written == out.size ()) out.resize (out.size () + std::max (out.size () / 2 , std::size_t {1024 }));
51+ const auto room = std::min (out.size () - written, kZlibPiece );
52+ s.next_out = reinterpret_cast <Bytef*>(out.data () + written);
53+ s.avail_out = static_cast <uInt>(room);
54+ rc = ::deflate (&s, fed == in.size () ? Z_FINISH : Z_NO_FLUSH );
55+ written += room - s.avail_out ;
56+ if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR ) {
57+ deflateEnd (&s);
58+ return std::unexpected (std::format (" zlib: deflate returned {}" , rc));
59+ }
60+ }
4261 deflateEnd (&s);
43- if (rc != Z_STREAM_END ) return std::unexpected (std::format (" zlib: deflate returned {}" , rc));
4462 out.resize (written);
4563 return out;
4664}
4765
4866} // namespace
4967
5068std::expected<std::string, std::string> gzip (std::string_view in, int level) {
51- if (in.size () > std::numeric_limits<uInt>::max ())
52- return std::unexpected (std::string (" gzip: an input over 4 GiB" ));
5369 return zlib_deflate (in, level, 15 + 16 );
5470}
5571
5672std::expected<std::string, std::string> deflate (std::string_view in, int level) {
57- if (in.size () > std::numeric_limits<uInt>::max ())
58- return std::unexpected (std::string (" deflate: an input over 4 GiB" ));
5973 return zlib_deflate (in, level, -15 );
6074}
6175
@@ -104,11 +118,10 @@ namespace {
104118
105119// zlib's inflate until the stream ends: a gzip member (windowBits 15 + 32
106120// detects the header), its output growing, or raw DEFLATE (-15) of a stated
107- // `size`, its output never more than that.
121+ // `size`, its output never more than that. Input and output pass in pieces
122+ // (`kZlibPiece`), as for deflate.
108123std::expected<std::string, std::string> zlib_inflate (std::string_view in, int windowBits,
109124 std::optional<std::size_t > size) {
110- if (in.size () > std::numeric_limits<uInt>::max ())
111- return std::unexpected (std::string (" zlib: an input over 4 GiB" ));
112125 // DEFLATE expands at most 1032 times (a run of one byte): a stated size
113126 // beyond that is a lie, refused before anything is allocated, and a stream
114127 // that grows past it is refused as it grows.
@@ -122,11 +135,18 @@ std::expected<std::string, std::string> zlib_inflate(std::string_view in, int wi
122135 // One byte past a stated size: a stream that fills it holds more than it
123136 // said.
124137 out.resize (size ? *size + 1 : in.size () * 4 + 1024 );
125- s.next_in = reinterpret_cast <Bytef*>(const_cast <char *>(in.data ()));
126- s.avail_in = static_cast <uInt>(in.size ());
138+ std::size_t fed = 0 , written = 0 ;
127139 int rc = Z_OK ;
128- while (rc == Z_OK ) {
129- if (s.total_out >= out.size ()) {
140+ while (rc == Z_OK || rc == Z_BUF_ERROR ) {
141+ if (s.avail_in == 0 && fed < in.size ()) {
142+ const auto piece = std::min (in.size () - fed, kZlibPiece );
143+ s.next_in = reinterpret_cast <Bytef*>(const_cast <char *>(in.data () + fed));
144+ s.avail_in = static_cast <uInt>(piece);
145+ fed += piece;
146+ } else if (rc == Z_BUF_ERROR && s.avail_in == 0 && written < out.size ()) {
147+ break ; // the input ended before the stream did
148+ }
149+ if (written >= out.size ()) {
130150 if (size) {
131151 inflateEnd (&s);
132152 return std::unexpected (std::format (" zlib: the stream holds more than its {} bytes" ,
@@ -138,12 +158,12 @@ std::expected<std::string, std::string> zlib_inflate(std::string_view in, int wi
138158 }
139159 out.resize (std::min (out.size () * 2 , most + 1 ));
140160 }
141- s. next_out = reinterpret_cast <Bytef*> (out.data () + s. total_out );
142- s.avail_out = static_cast <uInt>(std::min<std:: size_t >( out.size () - s. total_out ,
143- std::numeric_limits <uInt>:: max ()) );
161+ const auto room = std::min (out.size () - written, kZlibPiece );
162+ s.next_out = reinterpret_cast <Bytef*>( out.data () + written);
163+ s. avail_out = static_cast <uInt>(room );
144164 rc = ::inflate (&s, Z_NO_FLUSH );
165+ written += room - s.avail_out ;
145166 }
146- const auto written = s.total_out ;
147167 inflateEnd (&s);
148168 if (rc != Z_STREAM_END ) return std::unexpected (std::format (" zlib: inflate returned {}" , rc));
149169 out.resize (written);
0 commit comments