1 : /******************************************************************************
2 : * $Id: cpl_vsil_gzip.cpp 20028 2010-07-11 18:20:55Z rouault $
3 : *
4 : * Project: CPL - Common Portability Library
5 : * Purpose: Implement VSI large file api for gz/zip files (.gz and .zip).
6 : * Author: Even Rouault, even.rouault at mines-paris.org
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2007, Even Rouault
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : /* gzio.c -- IO on .gz files
31 : Copyright (C) 1995-2005 Jean-loup Gailly.
32 :
33 : This software is provided 'as-is', without any express or implied
34 : warranty. In no event will the authors be held liable for any damages
35 : arising from the use of this software.
36 :
37 : Permission is granted to anyone to use this software for any purpose,
38 : including commercial applications, and to alter it and redistribute it
39 : freely, subject to the following restrictions:
40 :
41 : 1. The origin of this software must not be misrepresented; you must not
42 : claim that you wrote the original software. If you use this software
43 : in a product, an acknowledgment in the product documentation would be
44 : appreciated but is not required.
45 : 2. Altered source versions must be plainly marked as such, and must not be
46 : misrepresented as being the original software.
47 : 3. This notice may not be removed or altered from any source distribution.
48 :
49 : Jean-loup Gailly Mark Adler
50 : jloup@gzip.org madler@alumni.caltech.edu
51 :
52 :
53 : The data format used by the zlib library is described by RFCs (Request for
54 : Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
55 : (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
56 : */
57 :
58 :
59 : /* This file contains a refactoring of gzio.c from zlib project.
60 :
61 : It replaces classical calls operating on FILE* by calls to the VSI large file
62 : API. It also adds the capability to seek at the end of the file, which is not
63 : implemented in original gzSeek. It also implements a concept of in-memory "snapshots",
64 : that are a way of improving efficiency while seeking GZip files. Snapshots are
65 : ceated regularly when decompressing the data a snapshot of the gzip state.
66 : Later we can seek directly in the compressed data to the closest snapshot in order to
67 : reduce the amount of data to uncompress again.
68 :
69 : For .gz files, an effort is done to cache the size of the uncompressed data in
70 : a .gz.properties file, so that we don't need to seek at the end of the file
71 : each time a Stat() is done.
72 :
73 : For .zip, only reading is supported, for .gz read and write is supported.
74 : */
75 :
76 :
77 : #include "cpl_vsi_virtual.h"
78 : #include "cpl_string.h"
79 : #include "cpl_multiproc.h"
80 : #include <map>
81 :
82 : #include <zlib.h>
83 : #include "cpl_minizip_unzip.h"
84 : #include "cpl_time.h"
85 :
86 : CPL_CVSID("$Id: cpl_vsil_gzip.cpp 20028 2010-07-11 18:20:55Z rouault $");
87 :
88 : #define Z_BUFSIZE 65536 /* original size is 16384 */
89 : static int const gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */
90 :
91 : /* gzip flag byte */
92 : #define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */
93 : #define HEAD_CRC 0x02 /* bit 1 set: header CRC present */
94 : #define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */
95 : #define ORIG_NAME 0x08 /* bit 3 set: original file name present */
96 : #define COMMENT 0x10 /* bit 4 set: file comment present */
97 : #define RESERVED 0xE0 /* bits 5..7: reserved */
98 :
99 : #define ALLOC(size) malloc(size)
100 : #define TRYFREE(p) {if (p) free(p);}
101 :
102 : #define CPL_VSIL_GZ_RETURN_MINUS_ONE() \
103 : CPLError(CE_Failure, CPLE_AppDefined, "In file %s, at line %d, return -1", __FILE__, __LINE__)
104 :
105 : #define ENABLE_DEBUG 0
106 :
107 : /************************************************************************/
108 : /* ==================================================================== */
109 : /* VSIGZipHandle */
110 : /* ==================================================================== */
111 : /************************************************************************/
112 :
113 : typedef struct
114 : {
115 : vsi_l_offset uncompressed_pos;
116 : z_stream stream;
117 : uLong crc;
118 : int transparent;
119 : vsi_l_offset in;
120 : vsi_l_offset out;
121 : } GZipSnapshot;
122 :
123 : class VSIGZipHandle : public VSIVirtualHandle
124 : {
125 : VSIVirtualHandle* poBaseHandle;
126 : vsi_l_offset offset;
127 : vsi_l_offset compressed_size;
128 : vsi_l_offset uncompressed_size;
129 : vsi_l_offset offsetEndCompressedData;
130 : unsigned int expected_crc;
131 : char *pszBaseFileName; /* optional */
132 :
133 : /* Fields from gz_stream structure */
134 : z_stream stream;
135 : int z_err; /* error code for last stream operation */
136 : int z_eof; /* set if end of input file */
137 : Byte *inbuf; /* input buffer */
138 : Byte *outbuf; /* output buffer */
139 : uLong crc; /* crc32 of uncompressed data */
140 : int transparent; /* 1 if input file is not a .gz file */
141 : vsi_l_offset startOff; /* startOff of compressed data in file (header skipped) */
142 : vsi_l_offset in; /* bytes into deflate or inflate */
143 : vsi_l_offset out; /* bytes out of deflate or inflate */
144 : vsi_l_offset nLastReadOffset;
145 :
146 : GZipSnapshot* snapshots;
147 : vsi_l_offset snapshot_byte_interval; /* number of compressed bytes at which we create a "snapshot" */
148 :
149 : void check_header();
150 : int get_byte();
151 : int gzseek( vsi_l_offset nOffset, int nWhence );
152 : int gzrewind ();
153 : uLong getLong ();
154 :
155 : public:
156 :
157 : VSIGZipHandle(VSIVirtualHandle* poBaseHandle,
158 : const char* pszBaseFileName,
159 : vsi_l_offset offset = 0,
160 : vsi_l_offset compressed_size = 0,
161 : vsi_l_offset uncompressed_size = 0,
162 : unsigned int expected_crc = 0,
163 : int transparent = 0);
164 : ~VSIGZipHandle();
165 :
166 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
167 : virtual vsi_l_offset Tell();
168 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
169 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
170 : virtual int Eof();
171 : virtual int Flush();
172 : virtual int Close();
173 :
174 : VSIGZipHandle* Duplicate();
175 : void CloseBaseHandle();
176 :
177 106 : vsi_l_offset GetLastReadOffset() { return nLastReadOffset; }
178 303 : const char* GetBaseFileName() { return pszBaseFileName; }
179 :
180 2 : void SetUncompressedSize(vsi_l_offset nUncompressedSize) { uncompressed_size = nUncompressedSize; }
181 8 : vsi_l_offset GetUncompressedSize() { return uncompressed_size; }
182 : };
183 :
184 :
185 : class VSIGZipFilesystemHandler : public VSIFilesystemHandler
186 : {
187 : void* hMutex;
188 : VSIGZipHandle* poHandleLastGZipFile;
189 :
190 : public:
191 : VSIGZipFilesystemHandler();
192 : ~VSIGZipFilesystemHandler();
193 :
194 : virtual VSIVirtualHandle *Open( const char *pszFilename,
195 : const char *pszAccess);
196 : VSIGZipHandle *OpenGZipReadOnly( const char *pszFilename,
197 : const char *pszAccess);
198 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf );
199 : virtual int Unlink( const char *pszFilename );
200 : virtual int Rename( const char *oldpath, const char *newpath );
201 : virtual int Mkdir( const char *pszDirname, long nMode );
202 : virtual int Rmdir( const char *pszDirname );
203 : virtual char **ReadDir( const char *pszDirname );
204 :
205 : void SaveInfo( VSIGZipHandle* poHandle );
206 : };
207 :
208 :
209 : /************************************************************************/
210 : /* Duplicate() */
211 : /************************************************************************/
212 :
213 52 : VSIGZipHandle* VSIGZipHandle::Duplicate()
214 : {
215 52 : CPLAssert (offset == 0);
216 52 : CPLAssert (compressed_size != 0);
217 52 : CPLAssert (pszBaseFileName != NULL);
218 :
219 : VSIFilesystemHandler *poFSHandler =
220 52 : VSIFileManager::GetHandler( pszBaseFileName );
221 :
222 : VSIVirtualHandle* poNewBaseHandle =
223 52 : poFSHandler->Open( pszBaseFileName, "rb" );
224 :
225 52 : if (poNewBaseHandle == NULL)
226 0 : return NULL;
227 :
228 : VSIGZipHandle* poHandle = new VSIGZipHandle(poNewBaseHandle,
229 : pszBaseFileName,
230 : 0,
231 : compressed_size,
232 52 : uncompressed_size);
233 :
234 52 : poHandle->nLastReadOffset = nLastReadOffset;
235 :
236 : /* Most important : duplicate the snapshots ! */
237 :
238 : unsigned int i;
239 54 : for(i=0;i<compressed_size / snapshot_byte_interval + 1;i++)
240 : {
241 52 : if (snapshots[i].uncompressed_pos == 0)
242 50 : break;
243 :
244 2 : poHandle->snapshots[i].uncompressed_pos = snapshots[i].uncompressed_pos;
245 2 : inflateCopy( &poHandle->snapshots[i].stream, &snapshots[i].stream);
246 2 : poHandle->snapshots[i].crc = snapshots[i].crc;
247 2 : poHandle->snapshots[i].transparent = snapshots[i].transparent;
248 2 : poHandle->snapshots[i].in = snapshots[i].in;
249 2 : poHandle->snapshots[i].out = snapshots[i].out;
250 : }
251 :
252 52 : return poHandle;
253 : }
254 :
255 : /************************************************************************/
256 : /* CloseBaseHandle() */
257 : /************************************************************************/
258 :
259 9 : void VSIGZipHandle::CloseBaseHandle()
260 : {
261 9 : if (poBaseHandle)
262 9 : VSIFCloseL((FILE*)poBaseHandle);
263 9 : poBaseHandle = NULL;
264 9 : }
265 :
266 : /************************************************************************/
267 : /* VSIGZipHandle() */
268 : /************************************************************************/
269 :
270 : VSIGZipHandle::VSIGZipHandle(VSIVirtualHandle* poBaseHandle,
271 : const char* pszBaseFileName,
272 : vsi_l_offset offset,
273 : vsi_l_offset compressed_size,
274 : vsi_l_offset uncompressed_size,
275 : unsigned int expected_crc,
276 83 : int transparent)
277 : {
278 83 : this->poBaseHandle = poBaseHandle;
279 83 : this->expected_crc = expected_crc;
280 83 : this->pszBaseFileName = (pszBaseFileName) ? CPLStrdup(pszBaseFileName) : NULL;
281 83 : this->offset = offset;
282 83 : if (compressed_size)
283 : {
284 74 : this->compressed_size = compressed_size;
285 : }
286 : else
287 : {
288 9 : VSIFSeekL((FILE*)poBaseHandle, 0, SEEK_END);
289 9 : this->compressed_size = VSIFTellL((FILE*)poBaseHandle) - offset;
290 9 : compressed_size = this->compressed_size;
291 : }
292 83 : this->uncompressed_size = uncompressed_size;
293 83 : offsetEndCompressedData = offset + compressed_size;
294 :
295 83 : VSIFSeekL((FILE*)poBaseHandle, offset, SEEK_SET);
296 :
297 83 : nLastReadOffset = 0;
298 83 : stream.zalloc = (alloc_func)0;
299 83 : stream.zfree = (free_func)0;
300 83 : stream.opaque = (voidpf)0;
301 83 : stream.next_in = inbuf = Z_NULL;
302 83 : stream.next_out = outbuf = Z_NULL;
303 83 : stream.avail_in = stream.avail_out = 0;
304 83 : z_err = Z_OK;
305 83 : z_eof = 0;
306 83 : in = 0;
307 83 : out = 0;
308 83 : crc = crc32(0L, Z_NULL, 0);
309 83 : this->transparent = transparent;
310 :
311 83 : stream.next_in = inbuf = (Byte*)ALLOC(Z_BUFSIZE);
312 :
313 83 : int err = inflateInit2(&(stream), -MAX_WBITS);
314 : /* windowBits is passed < 0 to tell that there is no zlib header.
315 : * Note that in this case inflate *requires* an extra "dummy" byte
316 : * after the compressed stream in order to complete decompression and
317 : * return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are
318 : * present after the compressed stream.
319 : */
320 83 : if (err != Z_OK || inbuf == Z_NULL) {
321 0 : CPLError(CE_Failure, CPLE_NotSupported, "inflateInit2 init failed");
322 : }
323 83 : stream.avail_out = Z_BUFSIZE;
324 :
325 83 : if (offset == 0) check_header(); /* skip the .gz header */
326 83 : startOff = VSIFTellL((FILE*)poBaseHandle) - stream.avail_in;
327 :
328 83 : if (transparent == 0)
329 : {
330 83 : snapshot_byte_interval = MAX(Z_BUFSIZE, compressed_size / 100);
331 83 : snapshots = (GZipSnapshot*)CPLCalloc(sizeof(GZipSnapshot), (size_t) (compressed_size / snapshot_byte_interval + 1));
332 : }
333 : else
334 : {
335 0 : snapshots = NULL;
336 : }
337 83 : }
338 :
339 : /************************************************************************/
340 : /* ~VSIGZipHandle() */
341 : /************************************************************************/
342 :
343 82 : VSIGZipHandle::~VSIGZipHandle()
344 : {
345 :
346 82 : if (pszBaseFileName)
347 : {
348 : VSIFilesystemHandler *poFSHandler =
349 60 : VSIFileManager::GetHandler( "/vsigzip/" );
350 60 : ((VSIGZipFilesystemHandler*)poFSHandler)->SaveInfo(this);
351 : }
352 :
353 82 : if (stream.state != NULL) {
354 82 : inflateEnd(&(stream));
355 : }
356 :
357 82 : TRYFREE(inbuf);
358 82 : TRYFREE(outbuf);
359 :
360 82 : if (snapshots != NULL)
361 : {
362 : unsigned int i;
363 164 : for(i=0;i<compressed_size / snapshot_byte_interval + 1;i++)
364 : {
365 82 : if (snapshots[i].uncompressed_pos)
366 : {
367 32 : inflateEnd(&(snapshots[i].stream));
368 : }
369 : }
370 82 : CPLFree(snapshots);
371 : }
372 82 : CPLFree(pszBaseFileName);
373 :
374 82 : if (poBaseHandle)
375 74 : VSIFCloseL((FILE*)poBaseHandle);
376 82 : }
377 :
378 : /************************************************************************/
379 : /* check_header() */
380 : /************************************************************************/
381 :
382 76 : void VSIGZipHandle::check_header()
383 : {
384 : int method; /* method byte */
385 : int flags; /* flags byte */
386 : uInt len;
387 : int c;
388 :
389 : /* Assure two bytes in the buffer so we can peek ahead -- handle case
390 : where first byte of header is at the end of the buffer after the last
391 : gzip segment */
392 76 : len = stream.avail_in;
393 76 : if (len < 2) {
394 76 : if (len) inbuf[0] = stream.next_in[0];
395 76 : errno = 0;
396 76 : len = (uInt)VSIFReadL(inbuf + len, 1, Z_BUFSIZE >> len, (FILE*)poBaseHandle);
397 : if (ENABLE_DEBUG) CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
398 : VSIFTellL((FILE*)poBaseHandle), offsetEndCompressedData);
399 76 : if (VSIFTellL((FILE*)poBaseHandle) > offsetEndCompressedData)
400 : {
401 0 : len = len + (uInt) (offsetEndCompressedData - VSIFTellL((FILE*)poBaseHandle));
402 0 : VSIFSeekL((FILE*)poBaseHandle, offsetEndCompressedData, SEEK_SET);
403 : }
404 76 : if (len == 0 /* && ferror(file)*/)
405 : {
406 15 : if (VSIFTellL((FILE*)poBaseHandle) != offsetEndCompressedData)
407 0 : z_err = Z_ERRNO;
408 : }
409 76 : stream.avail_in += len;
410 76 : stream.next_in = inbuf;
411 76 : if (stream.avail_in < 2) {
412 15 : transparent = stream.avail_in;
413 15 : return;
414 : }
415 : }
416 :
417 : /* Peek ahead to check the gzip magic header */
418 61 : if (stream.next_in[0] != gz_magic[0] ||
419 : stream.next_in[1] != gz_magic[1]) {
420 0 : transparent = 1;
421 0 : return;
422 : }
423 61 : stream.avail_in -= 2;
424 61 : stream.next_in += 2;
425 :
426 : /* Check the rest of the gzip header */
427 61 : method = get_byte();
428 61 : flags = get_byte();
429 61 : if (method != Z_DEFLATED || (flags & RESERVED) != 0) {
430 0 : z_err = Z_DATA_ERROR;
431 0 : return;
432 : }
433 :
434 : /* Discard time, xflags and OS code: */
435 61 : for (len = 0; len < 6; len++) (void)get_byte();
436 :
437 61 : if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */
438 0 : len = (uInt)get_byte();
439 0 : len += ((uInt)get_byte())<<8;
440 : /* len is garbage if EOF but the loop below will quit anyway */
441 0 : while (len-- != 0 && get_byte() != EOF) ;
442 : }
443 61 : if ((flags & ORIG_NAME) != 0) { /* skip the original file name */
444 0 : while ((c = get_byte()) != 0 && c != EOF) ;
445 : }
446 61 : if ((flags & COMMENT) != 0) { /* skip the .gz file comment */
447 0 : while ((c = get_byte()) != 0 && c != EOF) ;
448 : }
449 61 : if ((flags & HEAD_CRC) != 0) { /* skip the header crc */
450 0 : for (len = 0; len < 2; len++) (void)get_byte();
451 : }
452 61 : z_err = z_eof ? Z_DATA_ERROR : Z_OK;
453 : }
454 :
455 : /************************************************************************/
456 : /* get_byte() */
457 : /************************************************************************/
458 :
459 608 : int VSIGZipHandle::get_byte()
460 : {
461 608 : if (z_eof) return EOF;
462 608 : if (stream.avail_in == 0) {
463 0 : errno = 0;
464 0 : stream.avail_in = (uInt)VSIFReadL(inbuf, 1, Z_BUFSIZE, (FILE*)poBaseHandle);
465 : if (ENABLE_DEBUG) CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
466 : VSIFTellL((FILE*)poBaseHandle), offsetEndCompressedData);
467 0 : if (VSIFTellL((FILE*)poBaseHandle) > offsetEndCompressedData)
468 : {
469 0 : stream.avail_in = stream.avail_in + (uInt) (offsetEndCompressedData - VSIFTellL((FILE*)poBaseHandle));
470 0 : VSIFSeekL((FILE*)poBaseHandle, offsetEndCompressedData, SEEK_SET);
471 : }
472 0 : if (stream.avail_in == 0) {
473 0 : z_eof = 1;
474 0 : if (VSIFTellL((FILE*)poBaseHandle) != offsetEndCompressedData)
475 0 : z_err = Z_ERRNO;
476 : /*if (ferror(file)) z_err = Z_ERRNO;*/
477 0 : return EOF;
478 : }
479 0 : stream.next_in = inbuf;
480 : }
481 608 : stream.avail_in--;
482 608 : return *(stream.next_in)++;
483 : }
484 :
485 : /************************************************************************/
486 : /* gzrewind() */
487 : /************************************************************************/
488 :
489 27 : int VSIGZipHandle::gzrewind ()
490 : {
491 27 : z_err = Z_OK;
492 27 : z_eof = 0;
493 27 : stream.avail_in = 0;
494 27 : stream.next_in = inbuf;
495 27 : crc = crc32(0L, Z_NULL, 0);
496 27 : if (!transparent) (void)inflateReset(&stream);
497 27 : in = 0;
498 27 : out = 0;
499 27 : return VSIFSeekL((FILE*)poBaseHandle, startOff, SEEK_SET);
500 : }
501 :
502 : /************************************************************************/
503 : /* Seek() */
504 : /************************************************************************/
505 :
506 254 : int VSIGZipHandle::Seek( vsi_l_offset nOffset, int nWhence )
507 : {
508 : /* The semantics of gzseek is different from ::Seek */
509 : /* It returns the current offset, where as ::Seek shoud return 0 */
510 : /* if successfull */
511 254 : int ret = gzseek(nOffset, nWhence);
512 254 : return (ret >= 0) ? 0 : ret;
513 : }
514 :
515 : /************************************************************************/
516 : /* gzseek() */
517 : /************************************************************************/
518 :
519 254 : int VSIGZipHandle::gzseek( vsi_l_offset offset, int whence )
520 : {
521 254 : vsi_l_offset original_offset = offset;
522 254 : int original_nWhence = whence;
523 :
524 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Seek(" CPL_FRMT_GUIB ",%d)", offset, whence);
525 :
526 254 : if (transparent)
527 : {
528 0 : stream.avail_in = 0;
529 0 : stream.next_in = inbuf;
530 0 : if (whence == SEEK_CUR)
531 : {
532 0 : if (out + offset < 0 || out + offset > compressed_size)
533 : {
534 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
535 0 : return -1L;
536 : }
537 :
538 0 : offset = startOff + out + offset;
539 : }
540 0 : else if (whence == SEEK_SET)
541 : {
542 0 : if (offset < 0 || offset > compressed_size)
543 : {
544 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
545 0 : return -1L;
546 : }
547 :
548 0 : offset = startOff + offset;
549 : }
550 0 : else if (whence == SEEK_END)
551 : {
552 : /* Commented test : because vsi_l_offset is unsigned (for the moment) */
553 : /* so no way to seek backward. See #1590 */
554 0 : if (offset > 0 /*|| -offset > compressed_size*/)
555 : {
556 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
557 0 : return -1L;
558 : }
559 :
560 0 : offset = startOff + compressed_size - offset;
561 : }
562 : else
563 : {
564 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
565 0 : return -1L;
566 : }
567 0 : if (VSIFSeekL((FILE*)poBaseHandle, offset, SEEK_SET) < 0)
568 : {
569 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
570 0 : return -1L;
571 : }
572 :
573 0 : in = out = offset - startOff;
574 : if (ENABLE_DEBUG) CPLDebug("GZIP", "return " CPL_FRMT_GUIB, offset);
575 0 : return (int) in;
576 : }
577 :
578 : /* whence == SEEK_END is unsuppored in original gzseek. */
579 254 : if (whence == SEEK_END)
580 : {
581 : /* If we known the uncompressed size, we can fake a jump to */
582 : /* the end of the stream */
583 27 : if (offset == 0 && uncompressed_size != 0)
584 : {
585 25 : out = uncompressed_size;
586 25 : return 1;
587 : }
588 :
589 : /* We don't know the uncompressed size. This is unfortunate. Let's do the slow version... */
590 : static int firstWarning = 1;
591 2 : if (compressed_size > 10 * 1024 * 1024 && firstWarning)
592 : {
593 : CPLError(CE_Warning, CPLE_AppDefined,
594 0 : "VSIFSeekL(xxx, SEEK_END) may be really slow on GZip streams.");
595 0 : firstWarning = 0;
596 : }
597 :
598 2 : whence = SEEK_CUR;
599 2 : offset = 1024 * 1024 * 1024;
600 2 : offset *= 1024 * 1024;
601 : }
602 :
603 229 : if (/*whence == SEEK_END ||*/
604 : z_err == Z_ERRNO || z_err == Z_DATA_ERROR) {
605 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
606 0 : return -1L;
607 : }
608 :
609 : /* Rest of function is for reading only */
610 :
611 : /* compute absolute position */
612 229 : if (whence == SEEK_CUR) {
613 2 : offset += out;
614 : }
615 : if (offset < 0) {
616 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
617 : return -1L;
618 : }
619 :
620 : /* For a negative seek, rewind and use positive seek */
621 229 : if (offset >= out) {
622 202 : offset -= out;
623 27 : } else if (gzrewind() < 0) {
624 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
625 0 : return -1L;
626 : }
627 :
628 : unsigned int i;
629 229 : for(i=0;i<compressed_size / snapshot_byte_interval + 1;i++)
630 : {
631 229 : if (snapshots[i].uncompressed_pos == 0)
632 170 : break;
633 59 : if (snapshots[i].out <= out + offset &&
634 : (i == compressed_size / snapshot_byte_interval || snapshots[i+1].out == 0 || snapshots[i+1].out > out+offset))
635 : {
636 59 : if (out >= snapshots[i].out)
637 59 : break;
638 :
639 : if (ENABLE_DEBUG)
640 : CPLDebug("SNAPSHOT", "using snapshot %d : uncompressed_pos(snapshot)=" CPL_FRMT_GUIB
641 : " in(snapshot)=" CPL_FRMT_GUIB
642 : " out(snapshot)=" CPL_FRMT_GUIB
643 : " out=" CPL_FRMT_GUIB
644 : " offset=" CPL_FRMT_GUIB,
645 : i, snapshots[i].uncompressed_pos, snapshots[i].in, snapshots[i].out, out, offset);
646 0 : offset = out + offset - snapshots[i].out;
647 0 : VSIFSeekL((FILE*)poBaseHandle, snapshots[i].uncompressed_pos, SEEK_SET);
648 0 : inflateEnd(&stream);
649 0 : inflateCopy(&stream, &snapshots[i].stream);
650 0 : crc = snapshots[i].crc;
651 0 : transparent = snapshots[i].transparent;
652 0 : in = snapshots[i].in;
653 0 : out = snapshots[i].out;
654 0 : break;
655 : }
656 : }
657 :
658 : /* offset is now the number of bytes to skip. */
659 :
660 229 : if (offset != 0 && outbuf == Z_NULL) {
661 38 : outbuf = (Byte*)ALLOC(Z_BUFSIZE);
662 38 : if (outbuf == Z_NULL) {
663 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
664 0 : return -1L;
665 : }
666 : }
667 :
668 229 : if (original_nWhence == SEEK_END && z_err == Z_STREAM_END)
669 : {
670 : if (ENABLE_DEBUG) CPLDebug("GZIP", "gzseek return " CPL_FRMT_GUIB, out);
671 0 : return (int) out;
672 : }
673 :
674 607 : while (offset > 0) {
675 152 : int size = Z_BUFSIZE;
676 152 : if (offset < Z_BUFSIZE) size = (int)offset;
677 :
678 152 : int read_size = Read(outbuf, 1, (uInt)size);
679 152 : if (read_size == 0) {
680 : //CPL_VSIL_GZ_RETURN_MINUS_ONE();
681 1 : return -1L;
682 : }
683 151 : if (original_nWhence == SEEK_END)
684 : {
685 2 : if (size != read_size)
686 : {
687 2 : z_err = Z_STREAM_END;
688 2 : break;
689 : }
690 : }
691 149 : offset -= read_size;
692 : }
693 : if (ENABLE_DEBUG) CPLDebug("GZIP", "gzseek return " CPL_FRMT_GUIB, out);
694 :
695 228 : if (original_offset == 0 && original_nWhence == SEEK_END)
696 : {
697 2 : uncompressed_size = out;
698 :
699 2 : if (pszBaseFileName)
700 : {
701 2 : CPLString osCacheFilename (pszBaseFileName);
702 2 : osCacheFilename += ".properties";
703 :
704 : /* Write a .properties file to avoid seeking next time */
705 2 : FILE* fpCacheLength = VSIFOpen(osCacheFilename.c_str(), "wt");
706 2 : if (fpCacheLength)
707 : {
708 : char* pszFirstNonSpace;
709 : char szBuffer[32];
710 2 : szBuffer[31] = 0;
711 :
712 2 : CPLPrintUIntBig(szBuffer, compressed_size, 31);
713 2 : pszFirstNonSpace = szBuffer;
714 2 : while (*pszFirstNonSpace == ' ') pszFirstNonSpace ++;
715 2 : VSIFPrintf(fpCacheLength, "compressed_size=%s\n", pszFirstNonSpace);
716 :
717 2 : CPLPrintUIntBig(szBuffer, uncompressed_size, 31);
718 2 : pszFirstNonSpace = szBuffer;
719 2 : while (*pszFirstNonSpace == ' ') pszFirstNonSpace ++;
720 2 : VSIFPrintf(fpCacheLength, "uncompressed_size=%s\n", pszFirstNonSpace);
721 :
722 2 : VSIFClose(fpCacheLength);
723 2 : }
724 : }
725 : }
726 :
727 228 : return (int) out;
728 : }
729 :
730 : /************************************************************************/
731 : /* Tell() */
732 : /************************************************************************/
733 :
734 81 : vsi_l_offset VSIGZipHandle::Tell()
735 : {
736 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Tell() = " CPL_FRMT_GUIB, out);
737 81 : return out;
738 : }
739 :
740 : /************************************************************************/
741 : /* Read() */
742 : /************************************************************************/
743 :
744 841 : size_t VSIGZipHandle::Read( void *buf, size_t nSize, size_t nMemb )
745 : {
746 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Read(%p, %d, %d)", buf, (int)nSize, (int)nMemb);
747 :
748 841 : unsigned len = nSize * nMemb;
749 :
750 841 : Bytef *pStart = (Bytef*)buf; /* startOffing point for crc computation */
751 : Byte *next_out; /* == stream.next_out but not forced far (for MSDOS) */
752 :
753 841 : if (z_err == Z_DATA_ERROR || z_err == Z_ERRNO)
754 : {
755 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
756 0 : return 0;
757 : }
758 841 : if (z_err == Z_STREAM_END)
759 : {
760 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Read: Eof");
761 4 : return 0; /* EOF */
762 : }
763 :
764 837 : next_out = (Byte*)buf;
765 837 : stream.next_out = (Bytef*)buf;
766 837 : stream.avail_out = len;
767 :
768 2477 : while (stream.avail_out != 0) {
769 :
770 837 : if (transparent) {
771 : /* Copy first the lookahead bytes: */
772 0 : uInt n = stream.avail_in;
773 0 : if (n > stream.avail_out) n = stream.avail_out;
774 0 : if (n > 0) {
775 0 : memcpy (stream.next_out, stream.next_in, n);
776 0 : next_out += n;
777 0 : stream.next_out = next_out;
778 0 : stream.next_in += n;
779 0 : stream.avail_out -= n;
780 0 : stream.avail_in -= n;
781 : }
782 0 : if (stream.avail_out > 0) {
783 : stream.avail_out -=
784 0 : (uInt)VSIFReadL(next_out, 1, stream.avail_out, (FILE*)poBaseHandle);
785 : }
786 0 : len -= stream.avail_out;
787 0 : in += len;
788 0 : out += len;
789 0 : if (len == 0) z_eof = 1;
790 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Read return %d", (int)(len / nSize));
791 0 : return (int)len / nSize;
792 : }
793 837 : if (stream.avail_in == 0 && !z_eof)
794 : {
795 44 : vsi_l_offset uncompressed_pos = VSIFTellL((FILE*)poBaseHandle);
796 44 : GZipSnapshot* snapshot = &snapshots[(uncompressed_pos - startOff) / snapshot_byte_interval];
797 44 : if (uncompressed_pos >= 0 && snapshot->uncompressed_pos == 0)
798 : {
799 30 : snapshot->crc = crc32 (crc, pStart, (uInt) (stream.next_out - pStart));
800 : if (ENABLE_DEBUG)
801 : CPLDebug("SNAPSHOT",
802 : "creating snapshot %d : uncompressed_pos=" CPL_FRMT_GUIB
803 : " in=" CPL_FRMT_GUIB
804 : " out=" CPL_FRMT_GUIB
805 : " crc=%X",
806 : (int)((uncompressed_pos - startOff) / snapshot_byte_interval),
807 : uncompressed_pos, in, out, (unsigned int)snapshot->crc);
808 30 : snapshot->uncompressed_pos = uncompressed_pos;
809 30 : inflateCopy(&snapshot->stream, &stream);
810 30 : snapshot->transparent = transparent;
811 30 : snapshot->in = in;
812 30 : snapshot->out = out;
813 :
814 30 : if (out > nLastReadOffset)
815 0 : nLastReadOffset = out;
816 : }
817 :
818 44 : errno = 0;
819 44 : stream.avail_in = (uInt)VSIFReadL(inbuf, 1, Z_BUFSIZE, (FILE*)poBaseHandle);
820 : if (ENABLE_DEBUG)
821 : CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
822 : VSIFTellL((FILE*)poBaseHandle), offsetEndCompressedData);
823 44 : if (VSIFTellL((FILE*)poBaseHandle) > offsetEndCompressedData)
824 : {
825 : if (ENABLE_DEBUG) CPLDebug("GZIP", "avail_in before = %d", stream.avail_in);
826 33 : stream.avail_in = stream.avail_in + (uInt) (offsetEndCompressedData - VSIFTellL((FILE*)poBaseHandle));
827 33 : VSIFSeekL((FILE*)poBaseHandle, offsetEndCompressedData, SEEK_SET);
828 : if (ENABLE_DEBUG) CPLDebug("GZIP", "avail_in after = %d", stream.avail_in);
829 : }
830 44 : if (stream.avail_in == 0) {
831 0 : z_eof = 1;
832 0 : if (VSIFTellL((FILE*)poBaseHandle) != offsetEndCompressedData)
833 : {
834 0 : z_err = Z_ERRNO;
835 0 : break;
836 : }
837 : /*if (ferror (file)) {
838 : z_err = Z_ERRNO;
839 : break;
840 : }*/
841 : }
842 44 : stream.next_in = inbuf;
843 : }
844 837 : in += stream.avail_in;
845 837 : out += stream.avail_out;
846 837 : z_err = inflate(& (stream), Z_NO_FLUSH);
847 837 : in -= stream.avail_in;
848 837 : out -= stream.avail_out;
849 :
850 837 : if (z_err == Z_STREAM_END) {
851 : /* Check CRC and original size */
852 34 : crc = crc32 (crc, pStart, (uInt) (stream.next_out - pStart));
853 34 : pStart = stream.next_out;
854 34 : if (expected_crc)
855 : {
856 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Computed CRC = %X. Expected CRC = %X", (unsigned int)crc, expected_crc);
857 : }
858 34 : if (expected_crc != 0 && expected_crc != crc)
859 : {
860 0 : CPLError(CE_Failure, CPLE_FileIO, "CRC error. Got %X instead of %X", (unsigned int)crc, expected_crc);
861 0 : z_err = Z_DATA_ERROR;
862 : }
863 34 : else if (expected_crc == 0)
864 : {
865 15 : unsigned int read_crc = getLong();
866 15 : if (read_crc != crc)
867 : {
868 0 : CPLError(CE_Failure, CPLE_FileIO, "CRC error. Got %X instead of %X", (unsigned int)crc, read_crc);
869 0 : z_err = Z_DATA_ERROR;
870 : }
871 : else
872 : {
873 15 : (void)getLong();
874 : /* The uncompressed length returned by above getlong() may be
875 : * different from out in case of concatenated .gz files.
876 : * Check for such files:
877 : */
878 15 : check_header();
879 15 : if (z_err == Z_OK) {
880 0 : inflateReset(& (stream));
881 0 : crc = crc32(0L, Z_NULL, 0);
882 : }
883 : }
884 : }
885 : }
886 837 : if (z_err != Z_OK || z_eof) break;
887 : }
888 837 : crc = crc32 (crc, pStart, (uInt) (stream.next_out - pStart));
889 :
890 837 : if (len == stream.avail_out &&
891 : (z_err == Z_DATA_ERROR || z_err == Z_ERRNO))
892 : {
893 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
894 0 : return 0;
895 : }
896 : if (ENABLE_DEBUG)
897 : CPLDebug("GZIP", "Read return %d (z_err=%d, z_eof=%d)",
898 : (int)((len - stream.avail_out) / nSize), z_err, z_eof);
899 837 : return (int)(len - stream.avail_out) / nSize;
900 : }
901 :
902 : /************************************************************************/
903 : /* getLong() */
904 : /************************************************************************/
905 :
906 30 : uLong VSIGZipHandle::getLong ()
907 : {
908 30 : uLong x = (uLong)get_byte();
909 : int c;
910 :
911 30 : x += ((uLong)get_byte())<<8;
912 30 : x += ((uLong)get_byte())<<16;
913 30 : c = get_byte();
914 30 : if (c == EOF) z_err = Z_DATA_ERROR;
915 30 : x += ((uLong)c)<<24;
916 30 : return x;
917 : }
918 :
919 : /************************************************************************/
920 : /* Write() */
921 : /************************************************************************/
922 :
923 0 : size_t VSIGZipHandle::Write( const void *pBuffer, size_t nSize, size_t nMemb )
924 : {
925 0 : CPLError(CE_Failure, CPLE_NotSupported, "VSIFWriteL is not supported on GZip streams\n");
926 0 : return 0;
927 : }
928 :
929 : /************************************************************************/
930 : /* Eof() */
931 : /************************************************************************/
932 :
933 :
934 2 : int VSIGZipHandle::Eof()
935 : {
936 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Eof()");
937 2 : if (z_eof) return 1;
938 2 : return z_err == Z_STREAM_END;
939 : }
940 :
941 : /************************************************************************/
942 : /* Flush() */
943 : /************************************************************************/
944 :
945 0 : int VSIGZipHandle::Flush()
946 : {
947 0 : return 0;
948 : }
949 :
950 : /************************************************************************/
951 : /* Close() */
952 : /************************************************************************/
953 :
954 70 : int VSIGZipHandle::Close()
955 : {
956 70 : return 0;
957 : }
958 :
959 :
960 : /************************************************************************/
961 : /* ==================================================================== */
962 : /* VSIGZipWriteHandle */
963 : /* ==================================================================== */
964 : /************************************************************************/
965 :
966 : class VSIGZipWriteHandle : public VSIVirtualHandle
967 : {
968 : VSIVirtualHandle* poBaseHandle;
969 : z_stream sStream;
970 : Byte *pabyInBuf;
971 : Byte *pabyOutBuf;
972 : bool bCompressActive;
973 : vsi_l_offset nCurOffset;
974 : GUInt32 nCRC;
975 :
976 : public:
977 :
978 : VSIGZipWriteHandle(VSIVirtualHandle* poBaseHandle);
979 :
980 : ~VSIGZipWriteHandle();
981 :
982 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
983 : virtual vsi_l_offset Tell();
984 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
985 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
986 : virtual int Eof();
987 : virtual int Flush();
988 : virtual int Close();
989 : };
990 :
991 : /************************************************************************/
992 : /* VSIGZipWriteHandle() */
993 : /************************************************************************/
994 :
995 24 : VSIGZipWriteHandle::VSIGZipWriteHandle( VSIVirtualHandle *poBaseHandle )
996 :
997 : {
998 24 : nCurOffset = 0;
999 :
1000 24 : this->poBaseHandle = poBaseHandle;
1001 :
1002 24 : nCRC = crc32(0L, Z_NULL, 0);
1003 24 : sStream.zalloc = (alloc_func)0;
1004 24 : sStream.zfree = (free_func)0;
1005 24 : sStream.opaque = (voidpf)0;
1006 24 : sStream.next_in = Z_NULL;
1007 24 : sStream.next_out = Z_NULL;
1008 24 : sStream.avail_in = sStream.avail_out = 0;
1009 :
1010 24 : pabyInBuf = (Byte *) CPLMalloc( Z_BUFSIZE );
1011 24 : sStream.next_in = pabyInBuf;
1012 :
1013 24 : pabyOutBuf = (Byte *) CPLMalloc( Z_BUFSIZE );
1014 :
1015 24 : if( deflateInit2( &sStream, Z_DEFAULT_COMPRESSION,
1016 : Z_DEFLATED, -MAX_WBITS, 8,
1017 : Z_DEFAULT_STRATEGY ) != Z_OK )
1018 0 : bCompressActive = false;
1019 : else
1020 : {
1021 : char header[11];
1022 :
1023 : /* Write a very simple .gz header:
1024 : */
1025 : sprintf( header, "%c%c%c%c%c%c%c%c%c%c", gz_magic[0], gz_magic[1],
1026 : Z_DEFLATED, 0 /*flags*/, 0,0,0,0 /*time*/, 0 /*xflags*/,
1027 24 : 0x03 );
1028 24 : poBaseHandle->Write( header, 1, 10 );
1029 :
1030 24 : bCompressActive = true;
1031 : }
1032 24 : }
1033 :
1034 : /************************************************************************/
1035 : /* ~VSIGZipWriteHandle() */
1036 : /************************************************************************/
1037 :
1038 24 : VSIGZipWriteHandle::~VSIGZipWriteHandle()
1039 :
1040 : {
1041 24 : if( bCompressActive )
1042 0 : Close();
1043 :
1044 24 : CPLFree( pabyInBuf );
1045 24 : CPLFree( pabyOutBuf );
1046 24 : }
1047 :
1048 : /************************************************************************/
1049 : /* Close() */
1050 : /************************************************************************/
1051 :
1052 24 : int VSIGZipWriteHandle::Close()
1053 :
1054 : {
1055 24 : if( bCompressActive )
1056 : {
1057 24 : sStream.next_out = pabyOutBuf;
1058 24 : sStream.avail_out = Z_BUFSIZE;
1059 :
1060 24 : deflate( &sStream, Z_FINISH );
1061 :
1062 24 : size_t nOutBytes = Z_BUFSIZE - sStream.avail_out;
1063 :
1064 24 : if( poBaseHandle->Write( pabyOutBuf, 1, nOutBytes ) < nOutBytes )
1065 0 : return EOF;
1066 :
1067 24 : deflateEnd( &sStream );
1068 :
1069 : GUInt32 anTrailer[2];
1070 :
1071 24 : anTrailer[0] = CPL_LSBWORD32( nCRC );
1072 24 : anTrailer[1] = CPL_LSBWORD32( (GUInt32) nCurOffset );
1073 :
1074 24 : poBaseHandle->Write( anTrailer, 1, 8 );
1075 24 : poBaseHandle->Close();
1076 :
1077 24 : delete poBaseHandle;
1078 :
1079 24 : bCompressActive = false;
1080 : }
1081 :
1082 24 : return 0;
1083 : }
1084 :
1085 : /************************************************************************/
1086 : /* Read() */
1087 : /************************************************************************/
1088 :
1089 0 : size_t VSIGZipWriteHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
1090 :
1091 : {
1092 0 : CPLError(CE_Failure, CPLE_NotSupported, "VSIFReadL is not supported on GZip write streams\n");
1093 0 : return 0;
1094 : }
1095 :
1096 : /************************************************************************/
1097 : /* Write() */
1098 : /************************************************************************/
1099 :
1100 : size_t VSIGZipWriteHandle::Write( const void *pBuffer,
1101 1155 : size_t nSize, size_t nMemb )
1102 :
1103 : {
1104 : int nBytesToWrite, nNextByte;
1105 :
1106 1155 : nBytesToWrite = (int) (nSize * nMemb);
1107 1155 : nNextByte = 0;
1108 :
1109 1155 : nCRC = crc32(nCRC, (const Bytef *)pBuffer, nBytesToWrite);
1110 :
1111 1155 : if( !bCompressActive )
1112 0 : return 0;
1113 :
1114 3465 : while( nNextByte < nBytesToWrite )
1115 : {
1116 1155 : sStream.next_out = pabyOutBuf;
1117 1155 : sStream.avail_out = Z_BUFSIZE;
1118 :
1119 1155 : if( sStream.avail_in > 0 )
1120 0 : memmove( pabyInBuf, sStream.next_in, sStream.avail_in );
1121 :
1122 1155 : int nNewBytesToWrite = MIN((int) (Z_BUFSIZE-sStream.avail_in),
1123 : nBytesToWrite - nNextByte);
1124 : memcpy( pabyInBuf + sStream.avail_in,
1125 : ((Byte *) pBuffer) + nNextByte,
1126 1155 : nNewBytesToWrite );
1127 :
1128 1155 : sStream.next_in = pabyInBuf;
1129 1155 : sStream.avail_in += nNewBytesToWrite;
1130 :
1131 1155 : deflate( &sStream, Z_NO_FLUSH );
1132 :
1133 1155 : size_t nOutBytes = Z_BUFSIZE - sStream.avail_out;
1134 :
1135 1155 : if( nOutBytes > 0 )
1136 : {
1137 0 : if( poBaseHandle->Write( pabyOutBuf, 1, nOutBytes ) < nOutBytes )
1138 0 : return 0;
1139 : }
1140 :
1141 1155 : nNextByte += nNewBytesToWrite;
1142 1155 : nCurOffset += nNewBytesToWrite;
1143 : }
1144 :
1145 1155 : return nMemb;
1146 : }
1147 :
1148 : /************************************************************************/
1149 : /* Flush() */
1150 : /************************************************************************/
1151 :
1152 0 : int VSIGZipWriteHandle::Flush()
1153 :
1154 : {
1155 : // we *could* do something for this but for now we choose not to.
1156 :
1157 0 : return 0;
1158 : }
1159 :
1160 : /************************************************************************/
1161 : /* Eof() */
1162 : /************************************************************************/
1163 :
1164 0 : int VSIGZipWriteHandle::Eof()
1165 :
1166 : {
1167 0 : return 1;
1168 : }
1169 :
1170 : /************************************************************************/
1171 : /* Seek() */
1172 : /************************************************************************/
1173 :
1174 0 : int VSIGZipWriteHandle::Seek( vsi_l_offset nOffset, int nWhence )
1175 :
1176 : {
1177 0 : if( nOffset == 0 && (nWhence == SEEK_END || nWhence == SEEK_CUR) )
1178 0 : return 0;
1179 0 : else if( nWhence == SEEK_SET && nOffset == nCurOffset )
1180 0 : return 0;
1181 : else
1182 : {
1183 : CPLError(CE_Failure, CPLE_NotSupported,
1184 0 : "Seeking on writable compressed data streams not supported." );
1185 :
1186 0 : return -1;
1187 : }
1188 : }
1189 :
1190 : /************************************************************************/
1191 : /* Tell() */
1192 : /************************************************************************/
1193 :
1194 0 : vsi_l_offset VSIGZipWriteHandle::Tell()
1195 :
1196 : {
1197 0 : return nCurOffset;
1198 : }
1199 :
1200 :
1201 : /************************************************************************/
1202 : /* ==================================================================== */
1203 : /* VSIGZipFilesystemHandler */
1204 : /* ==================================================================== */
1205 : /************************************************************************/
1206 :
1207 :
1208 : /************************************************************************/
1209 : /* VSIGZipFilesystemHandler() */
1210 : /************************************************************************/
1211 :
1212 447 : VSIGZipFilesystemHandler::VSIGZipFilesystemHandler()
1213 : {
1214 447 : hMutex = NULL;
1215 :
1216 447 : poHandleLastGZipFile = NULL;
1217 447 : }
1218 :
1219 : /************************************************************************/
1220 : /* ~VSIGZipFilesystemHandler() */
1221 : /************************************************************************/
1222 :
1223 432 : VSIGZipFilesystemHandler::~VSIGZipFilesystemHandler()
1224 : {
1225 432 : if (poHandleLastGZipFile)
1226 0 : delete poHandleLastGZipFile;
1227 :
1228 432 : if( hMutex != NULL )
1229 0 : CPLDestroyMutex( hMutex );
1230 432 : hMutex = NULL;
1231 432 : }
1232 :
1233 : /************************************************************************/
1234 : /* SaveInfo() */
1235 : /************************************************************************/
1236 :
1237 62 : void VSIGZipFilesystemHandler::SaveInfo( VSIGZipHandle* poHandle )
1238 : {
1239 62 : CPLMutexHolder oHolder(&hMutex);
1240 :
1241 62 : CPLAssert(poHandle->GetBaseFileName() != NULL);
1242 :
1243 62 : if (poHandleLastGZipFile &&
1244 : strcmp(poHandleLastGZipFile->GetBaseFileName(), poHandle->GetBaseFileName()) == 0)
1245 : {
1246 53 : if (poHandle->GetLastReadOffset() > poHandleLastGZipFile->GetLastReadOffset())
1247 : {
1248 0 : delete poHandleLastGZipFile;
1249 0 : poHandleLastGZipFile = poHandle->Duplicate();
1250 0 : poHandleLastGZipFile->CloseBaseHandle();
1251 : }
1252 : }
1253 : else
1254 : {
1255 9 : delete poHandleLastGZipFile;
1256 9 : poHandleLastGZipFile = poHandle->Duplicate();
1257 9 : poHandleLastGZipFile->CloseBaseHandle();
1258 62 : }
1259 62 : }
1260 :
1261 : /************************************************************************/
1262 : /* Open() */
1263 : /************************************************************************/
1264 :
1265 : VSIVirtualHandle* VSIGZipFilesystemHandler::Open( const char *pszFilename,
1266 112 : const char *pszAccess)
1267 : {
1268 : VSIFilesystemHandler *poFSHandler =
1269 112 : VSIFileManager::GetHandler( pszFilename + strlen("/vsigzip/"));
1270 :
1271 : /* -------------------------------------------------------------------- */
1272 : /* Is this an attempt to write a new file without update (w+) */
1273 : /* access? If so, create a writable handle for the underlying */
1274 : /* filename. */
1275 : /* -------------------------------------------------------------------- */
1276 112 : if (strchr(pszAccess, 'w') != NULL )
1277 : {
1278 24 : if( strchr(pszAccess, '+') != NULL )
1279 : {
1280 : CPLError(CE_Failure, CPLE_AppDefined,
1281 0 : "Write+update (w+) not supported for /vsigzip, only read-only or write-only.");
1282 0 : return NULL;
1283 : }
1284 :
1285 : VSIVirtualHandle* poVirtualHandle =
1286 24 : poFSHandler->Open( pszFilename + strlen("/vsigzip/"), "wb" );
1287 :
1288 24 : if (poVirtualHandle == NULL)
1289 0 : return NULL;
1290 :
1291 : else
1292 24 : return new VSIGZipWriteHandle( poVirtualHandle );
1293 : }
1294 :
1295 : /* -------------------------------------------------------------------- */
1296 : /* Otherwise we are in the read access case. */
1297 : /* -------------------------------------------------------------------- */
1298 :
1299 88 : VSIGZipHandle* poGZIPHandle = OpenGZipReadOnly(pszFilename, pszAccess);
1300 88 : if (poGZIPHandle)
1301 : /* Wrap the VSIGZipHandle inside a buffered reader that will */
1302 : /* improve dramatically performance when doing small backward */
1303 : /* seeks */
1304 48 : return VSICreateBufferedReaderHandle(poGZIPHandle);
1305 : else
1306 40 : return NULL;
1307 : }
1308 :
1309 : /************************************************************************/
1310 : /* OpenGZipReadOnly() */
1311 : /************************************************************************/
1312 :
1313 : VSIGZipHandle* VSIGZipFilesystemHandler::OpenGZipReadOnly( const char *pszFilename,
1314 92 : const char *pszAccess)
1315 : {
1316 : VSIFilesystemHandler *poFSHandler =
1317 92 : VSIFileManager::GetHandler( pszFilename + strlen("/vsigzip/"));
1318 :
1319 92 : CPLMutexHolder oHolder(&hMutex);
1320 :
1321 92 : if (poHandleLastGZipFile != NULL &&
1322 : strcmp(pszFilename + strlen("/vsigzip/"), poHandleLastGZipFile->GetBaseFileName()) == 0 &&
1323 : EQUAL(pszAccess, "rb"))
1324 : {
1325 43 : VSIGZipHandle* poHandle = poHandleLastGZipFile->Duplicate();
1326 43 : if (poHandle)
1327 43 : return poHandle;
1328 : }
1329 :
1330 : unsigned char signature[2];
1331 :
1332 : VSIVirtualHandle* poVirtualHandle =
1333 49 : poFSHandler->Open( pszFilename + strlen("/vsigzip/"), "rb" );
1334 :
1335 49 : if (poVirtualHandle == NULL)
1336 40 : return NULL;
1337 :
1338 9 : if (VSIFReadL(signature, 1, 2, (FILE*)poVirtualHandle) != 2)
1339 0 : return NULL;
1340 :
1341 9 : if (signature[0] != gz_magic[0] || signature[1] != gz_magic[1])
1342 0 : return NULL;
1343 :
1344 9 : if (poHandleLastGZipFile)
1345 8 : delete poHandleLastGZipFile;
1346 9 : poHandleLastGZipFile = NULL;
1347 :
1348 9 : return new VSIGZipHandle(poVirtualHandle, pszFilename + strlen("/vsigzip/"));
1349 : }
1350 :
1351 : /************************************************************************/
1352 : /* Stat() */
1353 : /************************************************************************/
1354 :
1355 44 : int VSIGZipFilesystemHandler::Stat( const char *pszFilename, VSIStatBufL *pStatBuf )
1356 : {
1357 44 : CPLMutexHolder oHolder(&hMutex);
1358 :
1359 44 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
1360 :
1361 44 : if (poHandleLastGZipFile != NULL &&
1362 : strcmp(pszFilename+strlen("/vsigzip/"), poHandleLastGZipFile->GetBaseFileName()) == 0)
1363 : {
1364 4 : if (poHandleLastGZipFile->GetUncompressedSize() != 0)
1365 : {
1366 4 : pStatBuf->st_mode = S_IFREG;
1367 4 : pStatBuf->st_size = poHandleLastGZipFile->GetUncompressedSize();
1368 4 : return 0;
1369 : }
1370 : }
1371 :
1372 : /* Begin by doing a stat on the real file */
1373 40 : int ret = VSIStatL(pszFilename+strlen("/vsigzip/"), pStatBuf);
1374 :
1375 40 : if (ret == 0)
1376 : {
1377 4 : CPLString osCacheFilename(pszFilename+strlen("/vsigzip/"));
1378 4 : osCacheFilename += ".properties";
1379 :
1380 : /* Can we save a bit of seeking by using a .properties file ? */
1381 4 : FILE* fpCacheLength = VSIFOpen(osCacheFilename.c_str(), "rt");
1382 4 : if (fpCacheLength)
1383 : {
1384 : char szBuffer[80];
1385 2 : szBuffer[79] = 0;
1386 2 : GUIntBig nCompressedSize = 0;
1387 2 : GUIntBig nUncompressedSize = 0;
1388 8 : while (CPLFGets(szBuffer, 79, fpCacheLength))
1389 : {
1390 4 : if (EQUALN(szBuffer, "compressed_size=", strlen("compressed_size=")))
1391 : {
1392 2 : char* pszBuffer = szBuffer + strlen("compressed_size=");
1393 : nCompressedSize =
1394 2 : CPLScanUIntBig(pszBuffer, strlen(pszBuffer));
1395 : }
1396 2 : else if (EQUALN(szBuffer, "uncompressed_size=", strlen("uncompressed_size=")))
1397 : {
1398 2 : char* pszBuffer = szBuffer + strlen("uncompressed_size=");
1399 : nUncompressedSize =
1400 2 : CPLScanUIntBig(pszBuffer, strlen(pszBuffer));
1401 : }
1402 : }
1403 :
1404 2 : VSIFClose(fpCacheLength);
1405 :
1406 2 : if (nCompressedSize == (GUIntBig) pStatBuf->st_size)
1407 : {
1408 : /* Patch with the uncompressed size */
1409 2 : pStatBuf->st_size = (long)nUncompressedSize;
1410 :
1411 : VSIGZipHandle* poHandle =
1412 2 : VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename, "rb");
1413 2 : if (poHandle)
1414 : {
1415 2 : poHandle->SetUncompressedSize(nUncompressedSize);
1416 2 : SaveInfo(poHandle);
1417 2 : delete poHandle;
1418 : }
1419 :
1420 2 : return ret;
1421 : }
1422 : }
1423 :
1424 : /* No, then seek at the end of the data (slow) */
1425 : VSIGZipHandle* poHandle =
1426 2 : VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename, "rb");
1427 2 : if (poHandle)
1428 : {
1429 : GUIntBig uncompressed_size;
1430 2 : poHandle->Seek(0, SEEK_END);
1431 2 : uncompressed_size = (GUIntBig) poHandle->Tell();
1432 2 : poHandle->Seek(0, SEEK_SET);
1433 :
1434 : /* Patch with the uncompressed size */
1435 2 : pStatBuf->st_size = (long)uncompressed_size;
1436 :
1437 2 : delete poHandle;
1438 : }
1439 : else
1440 : {
1441 0 : ret = -1;
1442 0 : }
1443 : }
1444 :
1445 38 : return ret;
1446 : }
1447 :
1448 : /************************************************************************/
1449 : /* Unlink() */
1450 : /************************************************************************/
1451 :
1452 0 : int VSIGZipFilesystemHandler::Unlink( const char *pszFilename )
1453 : {
1454 0 : return -1;
1455 : }
1456 :
1457 : /************************************************************************/
1458 : /* Rename() */
1459 : /************************************************************************/
1460 :
1461 0 : int VSIGZipFilesystemHandler::Rename( const char *oldpath, const char *newpath )
1462 : {
1463 0 : return -1;
1464 : }
1465 :
1466 : /************************************************************************/
1467 : /* Mkdir() */
1468 : /************************************************************************/
1469 :
1470 0 : int VSIGZipFilesystemHandler::Mkdir( const char *pszDirname, long nMode )
1471 : {
1472 0 : return -1;
1473 : }
1474 : /************************************************************************/
1475 : /* Rmdir() */
1476 : /************************************************************************/
1477 :
1478 0 : int VSIGZipFilesystemHandler::Rmdir( const char *pszDirname )
1479 : {
1480 0 : return -1;
1481 : }
1482 :
1483 : /************************************************************************/
1484 : /* ReadDir() */
1485 : /************************************************************************/
1486 :
1487 4 : char** VSIGZipFilesystemHandler::ReadDir( const char *pszDirname )
1488 : {
1489 4 : return NULL;
1490 : }
1491 :
1492 : /************************************************************************/
1493 : /* VSIInstallGZipFileHandler() */
1494 : /************************************************************************/
1495 :
1496 :
1497 : /**
1498 : * \brief Install GZip file system handler.
1499 : *
1500 : * A special file handler is installed that allows reading on-the-fly and
1501 : * writing in GZip (.gz) files.
1502 : *
1503 : * All portions of the file system underneath the base
1504 : * path "/vsigzip/" will be handled by this driver.
1505 : *
1506 : * Additional documentation is to be found at http://trac.osgeo.org/gdal/wiki/UserDocs/ReadInZip
1507 : *
1508 : */
1509 :
1510 447 : void VSIInstallGZipFileHandler(void)
1511 : {
1512 447 : VSIFileManager::InstallHandler( "/vsigzip/", new VSIGZipFilesystemHandler );
1513 447 : }
1514 :
1515 :
1516 : /************************************************************************/
1517 : /* ==================================================================== */
1518 : /* VSIZipEntryFileOffset */
1519 : /* ==================================================================== */
1520 : /************************************************************************/
1521 :
1522 : class VSIZipEntryFileOffset : public VSIArchiveEntryFileOffset
1523 0 : {
1524 : public:
1525 : unz_file_pos file_pos;
1526 :
1527 13 : VSIZipEntryFileOffset(unz_file_pos file_pos)
1528 13 : {
1529 13 : this->file_pos.pos_in_zip_directory = file_pos.pos_in_zip_directory;
1530 13 : this->file_pos.num_of_file = file_pos.num_of_file;
1531 13 : }
1532 : };
1533 :
1534 : /************************************************************************/
1535 : /* ==================================================================== */
1536 : /* VSIZipReader */
1537 : /* ==================================================================== */
1538 : /************************************************************************/
1539 :
1540 : class VSIZipReader : public VSIArchiveReader
1541 : {
1542 : private:
1543 : unzFile unzF;
1544 : unz_file_pos file_pos;
1545 : GUIntBig nNextFileSize;
1546 : CPLString osNextFileName;
1547 : GIntBig nModifiedTime;
1548 :
1549 : void SetInfo();
1550 :
1551 : public:
1552 : VSIZipReader(const char* pszZipFileName);
1553 : virtual ~VSIZipReader();
1554 :
1555 50 : int IsValid() { return unzF != NULL; }
1556 :
1557 22 : unzFile GetUnzFileHandle() { return unzF; }
1558 :
1559 : virtual int GotoFirstFile();
1560 : virtual int GotoNextFile();
1561 13 : virtual VSIArchiveEntryFileOffset* GetFileOffset() { return new VSIZipEntryFileOffset(file_pos); }
1562 15 : virtual GUIntBig GetFileSize() { return nNextFileSize; }
1563 20 : virtual CPLString GetFileName() { return osNextFileName; }
1564 15 : virtual GIntBig GetModifiedTime() { return nModifiedTime; }
1565 : virtual int GotoFileOffset(VSIArchiveEntryFileOffset* pOffset);
1566 : };
1567 :
1568 :
1569 : /************************************************************************/
1570 : /* VSIZipReader() */
1571 : /************************************************************************/
1572 :
1573 50 : VSIZipReader::VSIZipReader(const char* pszZipFileName)
1574 : {
1575 50 : unzF = cpl_unzOpen(pszZipFileName);
1576 50 : nNextFileSize = 0;
1577 50 : nModifiedTime = 0;
1578 50 : }
1579 :
1580 : /************************************************************************/
1581 : /* ~VSIZipReader() */
1582 : /************************************************************************/
1583 :
1584 50 : VSIZipReader::~VSIZipReader()
1585 : {
1586 50 : if (unzF)
1587 50 : cpl_unzClose(unzF);
1588 50 : }
1589 :
1590 : /************************************************************************/
1591 : /* SetInfo() */
1592 : /************************************************************************/
1593 :
1594 92 : void VSIZipReader::SetInfo()
1595 : {
1596 : char fileName[512];
1597 : unz_file_info file_info;
1598 92 : cpl_unzGetCurrentFileInfo (unzF, &file_info, fileName, 512, NULL, 0, NULL, 0);
1599 92 : osNextFileName = fileName;
1600 92 : nNextFileSize = file_info.uncompressed_size;
1601 : struct tm brokendowntime;
1602 92 : brokendowntime.tm_sec = file_info.tmu_date.tm_sec;
1603 92 : brokendowntime.tm_min = file_info.tmu_date.tm_min;
1604 92 : brokendowntime.tm_hour = file_info.tmu_date.tm_hour;
1605 92 : brokendowntime.tm_mday = file_info.tmu_date.tm_mday;
1606 92 : brokendowntime.tm_mon = file_info.tmu_date.tm_mon;
1607 92 : brokendowntime.tm_year = file_info.tmu_date.tm_year;
1608 92 : nModifiedTime = CPLYMDHMSToUnixTime(&brokendowntime);
1609 :
1610 92 : cpl_unzGetFilePos(unzF, &this->file_pos);
1611 92 : }
1612 :
1613 : /************************************************************************/
1614 : /* GotoNextFile() */
1615 : /************************************************************************/
1616 :
1617 23 : int VSIZipReader::GotoNextFile()
1618 : {
1619 23 : if (cpl_unzGoToNextFile(unzF) != UNZ_OK)
1620 12 : return FALSE;
1621 :
1622 11 : SetInfo();
1623 :
1624 11 : return TRUE;
1625 : }
1626 :
1627 : /************************************************************************/
1628 : /* GotoFirstFile() */
1629 : /************************************************************************/
1630 :
1631 63 : int VSIZipReader::GotoFirstFile()
1632 : {
1633 63 : if (cpl_unzGoToFirstFile(unzF) != UNZ_OK)
1634 0 : return FALSE;
1635 :
1636 63 : SetInfo();
1637 :
1638 63 : return TRUE;
1639 : }
1640 :
1641 : /************************************************************************/
1642 : /* GotoFileOffset() */
1643 : /************************************************************************/
1644 :
1645 18 : int VSIZipReader::GotoFileOffset(VSIArchiveEntryFileOffset* pOffset)
1646 : {
1647 18 : VSIZipEntryFileOffset* pZipEntryOffset = (VSIZipEntryFileOffset*)pOffset;
1648 18 : cpl_unzGoToFilePos(unzF, &(pZipEntryOffset->file_pos));
1649 :
1650 18 : SetInfo();
1651 :
1652 18 : return TRUE;
1653 : }
1654 :
1655 : /************************************************************************/
1656 : /* ==================================================================== */
1657 : /* VSIZipFilesystemHandler */
1658 : /* ==================================================================== */
1659 : /************************************************************************/
1660 :
1661 : class VSIZipFilesystemHandler : public VSIArchiveFilesystemHandler
1662 879 : {
1663 : public:
1664 178 : virtual const char* GetPrefix() { return "/vsizip"; }
1665 : virtual std::vector<CPLString> GetExtensions();
1666 : virtual VSIArchiveReader* CreateReader(const char* pszZipFileName);
1667 :
1668 : virtual VSIVirtualHandle *Open( const char *pszFilename,
1669 : const char *pszAccess);
1670 : };
1671 :
1672 : /************************************************************************/
1673 : /* GetExtensions() */
1674 : /************************************************************************/
1675 :
1676 1864 : std::vector<CPLString> VSIZipFilesystemHandler::GetExtensions()
1677 : {
1678 1864 : std::vector<CPLString> oList;
1679 3728 : oList.push_back(".zip");
1680 0 : return oList;
1681 : }
1682 :
1683 : /************************************************************************/
1684 : /* CreateReader() */
1685 : /************************************************************************/
1686 :
1687 50 : VSIArchiveReader* VSIZipFilesystemHandler::CreateReader(const char* pszZipFileName)
1688 : {
1689 50 : VSIZipReader* poReader = new VSIZipReader(pszZipFileName);
1690 :
1691 50 : if (!poReader->IsValid())
1692 : {
1693 0 : delete poReader;
1694 0 : return NULL;
1695 : }
1696 :
1697 50 : if (!poReader->GotoFirstFile())
1698 : {
1699 0 : delete poReader;
1700 0 : return NULL;
1701 : }
1702 :
1703 50 : return poReader;
1704 : }
1705 :
1706 : /************************************************************************/
1707 : /* Open() */
1708 : /************************************************************************/
1709 :
1710 : VSIVirtualHandle* VSIZipFilesystemHandler::Open( const char *pszFilename,
1711 49 : const char *pszAccess)
1712 : {
1713 : char* zipFilename;
1714 49 : CPLString osZipInFileName;
1715 :
1716 49 : if (strchr(pszAccess, 'w') != NULL ||
1717 : strchr(pszAccess, '+') != NULL)
1718 : {
1719 : CPLError(CE_Failure, CPLE_AppDefined,
1720 0 : "Only read-only mode is supported for /vsizip");
1721 0 : return NULL;
1722 : }
1723 :
1724 49 : zipFilename = SplitFilename(pszFilename, osZipInFileName);
1725 49 : if (zipFilename == NULL)
1726 8 : return NULL;
1727 :
1728 41 : VSIArchiveReader* poReader = OpenArchiveFile(zipFilename, osZipInFileName);
1729 41 : if (poReader == NULL)
1730 : {
1731 19 : CPLFree(zipFilename);
1732 19 : return NULL;
1733 : }
1734 :
1735 : VSIFilesystemHandler *poFSHandler =
1736 22 : VSIFileManager::GetHandler( zipFilename);
1737 :
1738 : VSIVirtualHandle* poVirtualHandle =
1739 22 : poFSHandler->Open( zipFilename, "rb" );
1740 :
1741 22 : CPLFree(zipFilename);
1742 22 : zipFilename = NULL;
1743 :
1744 22 : if (poVirtualHandle == NULL)
1745 : {
1746 0 : delete poReader;
1747 0 : return NULL;
1748 : }
1749 :
1750 22 : unzFile unzF = ((VSIZipReader*)poReader)->GetUnzFileHandle();
1751 :
1752 22 : cpl_unzOpenCurrentFile(unzF);
1753 :
1754 22 : uLong64 pos = cpl_unzGetCurrentFileZStreamPos(unzF);
1755 :
1756 : unz_file_info file_info;
1757 22 : cpl_unzGetCurrentFileInfo (unzF, &file_info, NULL, 0, NULL, 0, NULL, 0);
1758 :
1759 22 : cpl_unzCloseCurrentFile(unzF);
1760 :
1761 22 : delete poReader;
1762 :
1763 : return new VSIGZipHandle(poVirtualHandle,
1764 : NULL,
1765 : pos,
1766 : file_info.compressed_size,
1767 : file_info.uncompressed_size,
1768 : file_info.crc,
1769 22 : file_info.compression_method == 0);
1770 : }
1771 :
1772 : /************************************************************************/
1773 : /* VSIInstallZipFileHandler() */
1774 : /************************************************************************/
1775 :
1776 :
1777 : /**
1778 : * \brief Install ZIP file system handler.
1779 : *
1780 : * A special file handler is installed that allows reading on-the-fly in ZIP (.zip) archives.
1781 : * All portions of the file system underneath the base
1782 : * path "/vsizip/" will be handled by this driver.
1783 : *
1784 : * Additional documentation is to be found at http://trac.osgeo.org/gdal/wiki/UserDocs/ReadInZip
1785 : *
1786 : */
1787 :
1788 447 : void VSIInstallZipFileHandler(void)
1789 : {
1790 447 : VSIFileManager::InstallHandler( "/vsizip/", new VSIZipFilesystemHandler() );
1791 447 : }
1792 :
|
