1 : /******************************************************************************
2 : * $Id: cpl_vsil_gzip.cpp 24684 2012-07-20 15:49:23Z rouault $
3 : *
4 : * Project: CPL - Common Portability Library
5 : * Purpose: Implement VSI large file api for gz/zip files (.gz and .zip).
6 : * Author: Even Rouault, even.rouault at mines-paris.org
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2007, Even Rouault
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : /* gzio.c -- IO on .gz files
31 : Copyright (C) 1995-2005 Jean-loup Gailly.
32 :
33 : This software is provided 'as-is', without any express or implied
34 : warranty. In no event will the authors be held liable for any damages
35 : arising from the use of this software.
36 :
37 : Permission is granted to anyone to use this software for any purpose,
38 : including commercial applications, and to alter it and redistribute it
39 : freely, subject to the following restrictions:
40 :
41 : 1. The origin of this software must not be misrepresented; you must not
42 : claim that you wrote the original software. If you use this software
43 : in a product, an acknowledgment in the product documentation would be
44 : appreciated but is not required.
45 : 2. Altered source versions must be plainly marked as such, and must not be
46 : misrepresented as being the original software.
47 : 3. This notice may not be removed or altered from any source distribution.
48 :
49 : Jean-loup Gailly Mark Adler
50 : jloup@gzip.org madler@alumni.caltech.edu
51 :
52 :
53 : The data format used by the zlib library is described by RFCs (Request for
54 : Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
55 : (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
56 : */
57 :
58 :
59 : /* This file contains a refactoring of gzio.c from zlib project.
60 :
61 : It replaces classical calls operating on FILE* by calls to the VSI large file
62 : API. It also adds the capability to seek at the end of the file, which is not
63 : implemented in original gzSeek. It also implements a concept of in-memory "snapshots",
64 : that are a way of improving efficiency while seeking GZip files. Snapshots are
65 : ceated regularly when decompressing the data a snapshot of the gzip state.
66 : Later we can seek directly in the compressed data to the closest snapshot in order to
67 : reduce the amount of data to uncompress again.
68 :
69 : For .gz files, an effort is done to cache the size of the uncompressed data in
70 : a .gz.properties file, so that we don't need to seek at the end of the file
71 : each time a Stat() is done.
72 :
73 : For .zip and .gz, both reading and writing are supported, but just one mode at a time
74 : (read-only or write-only)
75 : */
76 :
77 :
78 : #include "cpl_vsi_virtual.h"
79 : #include "cpl_string.h"
80 : #include "cpl_multiproc.h"
81 : #include <map>
82 :
83 : #include <zlib.h>
84 : #include "cpl_minizip_unzip.h"
85 : #include "cpl_time.h"
86 :
87 : CPL_CVSID("$Id: cpl_vsil_gzip.cpp 24684 2012-07-20 15:49:23Z rouault $");
88 :
89 : #define Z_BUFSIZE 65536 /* original size is 16384 */
90 : static int const gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */
91 :
92 : /* gzip flag byte */
93 : #define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */
94 : #define HEAD_CRC 0x02 /* bit 1 set: header CRC present */
95 : #define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */
96 : #define ORIG_NAME 0x08 /* bit 3 set: original file name present */
97 : #define COMMENT 0x10 /* bit 4 set: file comment present */
98 : #define RESERVED 0xE0 /* bits 5..7: reserved */
99 :
100 : #define ALLOC(size) malloc(size)
101 : #define TRYFREE(p) {if (p) free(p);}
102 :
103 : #define CPL_VSIL_GZ_RETURN_MINUS_ONE() \
104 : CPLError(CE_Failure, CPLE_AppDefined, "In file %s, at line %d, return -1", __FILE__, __LINE__)
105 :
106 : #define ENABLE_DEBUG 0
107 :
108 : /************************************************************************/
109 : /* ==================================================================== */
110 : /* VSIGZipHandle */
111 : /* ==================================================================== */
112 : /************************************************************************/
113 :
114 : typedef struct
115 : {
116 : vsi_l_offset uncompressed_pos;
117 : z_stream stream;
118 : uLong crc;
119 : int transparent;
120 : vsi_l_offset in;
121 : vsi_l_offset out;
122 : } GZipSnapshot;
123 :
124 : class VSIGZipHandle : public VSIVirtualHandle
125 : {
126 : VSIVirtualHandle* poBaseHandle;
127 : vsi_l_offset offset;
128 : vsi_l_offset compressed_size;
129 : vsi_l_offset uncompressed_size;
130 : vsi_l_offset offsetEndCompressedData;
131 : unsigned int expected_crc;
132 : char *pszBaseFileName; /* optional */
133 :
134 : /* Fields from gz_stream structure */
135 : z_stream stream;
136 : int z_err; /* error code for last stream operation */
137 : int z_eof; /* set if end of input file */
138 : Byte *inbuf; /* input buffer */
139 : Byte *outbuf; /* output buffer */
140 : uLong crc; /* crc32 of uncompressed data */
141 : int transparent; /* 1 if input file is not a .gz file */
142 : vsi_l_offset startOff; /* startOff of compressed data in file (header skipped) */
143 : vsi_l_offset in; /* bytes into deflate or inflate */
144 : vsi_l_offset out; /* bytes out of deflate or inflate */
145 : vsi_l_offset nLastReadOffset;
146 :
147 : GZipSnapshot* snapshots;
148 : vsi_l_offset snapshot_byte_interval; /* number of compressed bytes at which we create a "snapshot" */
149 :
150 : void check_header();
151 : int get_byte();
152 : int gzseek( vsi_l_offset nOffset, int nWhence );
153 : int gzrewind ();
154 : uLong getLong ();
155 :
156 : public:
157 :
158 : VSIGZipHandle(VSIVirtualHandle* poBaseHandle,
159 : const char* pszBaseFileName,
160 : vsi_l_offset offset = 0,
161 : vsi_l_offset compressed_size = 0,
162 : vsi_l_offset uncompressed_size = 0,
163 : unsigned int expected_crc = 0,
164 : int transparent = 0);
165 : ~VSIGZipHandle();
166 :
167 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
168 : virtual vsi_l_offset Tell();
169 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
170 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
171 : virtual int Eof();
172 : virtual int Flush();
173 : virtual int Close();
174 :
175 : VSIGZipHandle* Duplicate();
176 : void CloseBaseHandle();
177 :
178 104 : vsi_l_offset GetLastReadOffset() { return nLastReadOffset; }
179 310 : const char* GetBaseFileName() { return pszBaseFileName; }
180 :
181 0 : void SetUncompressedSize(vsi_l_offset nUncompressedSize) { uncompressed_size = nUncompressedSize; }
182 5 : vsi_l_offset GetUncompressedSize() { return uncompressed_size; }
183 : };
184 :
185 :
186 : class VSIGZipFilesystemHandler : public VSIFilesystemHandler
187 : {
188 : void* hMutex;
189 : VSIGZipHandle* poHandleLastGZipFile;
190 : int bInSaveInfo;
191 :
192 : public:
193 : VSIGZipFilesystemHandler();
194 : ~VSIGZipFilesystemHandler();
195 :
196 : virtual VSIVirtualHandle *Open( const char *pszFilename,
197 : const char *pszAccess);
198 : VSIGZipHandle *OpenGZipReadOnly( const char *pszFilename,
199 : const char *pszAccess);
200 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
201 : virtual int Unlink( const char *pszFilename );
202 : virtual int Rename( const char *oldpath, const char *newpath );
203 : virtual int Mkdir( const char *pszDirname, long nMode );
204 : virtual int Rmdir( const char *pszDirname );
205 : virtual char **ReadDir( const char *pszDirname );
206 :
207 : void SaveInfo( VSIGZipHandle* poHandle );
208 : };
209 :
210 :
211 : /************************************************************************/
212 : /* Duplicate() */
213 : /************************************************************************/
214 :
215 52 : VSIGZipHandle* VSIGZipHandle::Duplicate()
216 : {
217 52 : CPLAssert (offset == 0);
218 52 : CPLAssert (compressed_size != 0);
219 52 : CPLAssert (pszBaseFileName != NULL);
220 :
221 : VSIFilesystemHandler *poFSHandler =
222 52 : VSIFileManager::GetHandler( pszBaseFileName );
223 :
224 : VSIVirtualHandle* poNewBaseHandle =
225 52 : poFSHandler->Open( pszBaseFileName, "rb" );
226 :
227 52 : if (poNewBaseHandle == NULL)
228 0 : return NULL;
229 :
230 : VSIGZipHandle* poHandle = new VSIGZipHandle(poNewBaseHandle,
231 : pszBaseFileName,
232 : 0,
233 : compressed_size,
234 52 : uncompressed_size);
235 :
236 52 : poHandle->nLastReadOffset = nLastReadOffset;
237 :
238 : /* Most important : duplicate the snapshots ! */
239 :
240 : unsigned int i;
241 55 : for(i=0;i<compressed_size / snapshot_byte_interval + 1;i++)
242 : {
243 52 : if (snapshots[i].uncompressed_pos == 0)
244 49 : break;
245 :
246 3 : poHandle->snapshots[i].uncompressed_pos = snapshots[i].uncompressed_pos;
247 3 : inflateCopy( &poHandle->snapshots[i].stream, &snapshots[i].stream);
248 3 : poHandle->snapshots[i].crc = snapshots[i].crc;
249 3 : poHandle->snapshots[i].transparent = snapshots[i].transparent;
250 3 : poHandle->snapshots[i].in = snapshots[i].in;
251 3 : poHandle->snapshots[i].out = snapshots[i].out;
252 : }
253 :
254 52 : return poHandle;
255 : }
256 :
257 : /************************************************************************/
258 : /* CloseBaseHandle() */
259 : /************************************************************************/
260 :
261 11 : void VSIGZipHandle::CloseBaseHandle()
262 : {
263 11 : if (poBaseHandle)
264 11 : VSIFCloseL((VSILFILE*)poBaseHandle);
265 11 : poBaseHandle = NULL;
266 11 : }
267 :
268 : /************************************************************************/
269 : /* VSIGZipHandle() */
270 : /************************************************************************/
271 :
272 326 : VSIGZipHandle::VSIGZipHandle(VSIVirtualHandle* poBaseHandle,
273 : const char* pszBaseFileName,
274 : vsi_l_offset offset,
275 : vsi_l_offset compressed_size,
276 : vsi_l_offset uncompressed_size,
277 : unsigned int expected_crc,
278 326 : int transparent)
279 : {
280 326 : this->poBaseHandle = poBaseHandle;
281 326 : this->expected_crc = expected_crc;
282 326 : this->pszBaseFileName = (pszBaseFileName) ? CPLStrdup(pszBaseFileName) : NULL;
283 326 : this->offset = offset;
284 641 : if (compressed_size || transparent)
285 : {
286 315 : this->compressed_size = compressed_size;
287 : }
288 : else
289 : {
290 11 : VSIFSeekL((VSILFILE*)poBaseHandle, 0, SEEK_END);
291 11 : this->compressed_size = VSIFTellL((VSILFILE*)poBaseHandle) - offset;
292 11 : compressed_size = this->compressed_size;
293 : }
294 326 : this->uncompressed_size = uncompressed_size;
295 326 : offsetEndCompressedData = offset + compressed_size;
296 :
297 326 : VSIFSeekL((VSILFILE*)poBaseHandle, offset, SEEK_SET);
298 :
299 326 : nLastReadOffset = 0;
300 326 : stream.zalloc = (alloc_func)0;
301 326 : stream.zfree = (free_func)0;
302 326 : stream.opaque = (voidpf)0;
303 326 : stream.next_in = inbuf = Z_NULL;
304 326 : stream.next_out = outbuf = Z_NULL;
305 326 : stream.avail_in = stream.avail_out = 0;
306 326 : z_err = Z_OK;
307 326 : z_eof = 0;
308 326 : in = 0;
309 326 : out = 0;
310 326 : crc = crc32(0L, Z_NULL, 0);
311 326 : this->transparent = transparent;
312 :
313 326 : stream.next_in = inbuf = (Byte*)ALLOC(Z_BUFSIZE);
314 :
315 326 : int err = inflateInit2(&(stream), -MAX_WBITS);
316 : /* windowBits is passed < 0 to tell that there is no zlib header.
317 : * Note that in this case inflate *requires* an extra "dummy" byte
318 : * after the compressed stream in order to complete decompression and
319 : * return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are
320 : * present after the compressed stream.
321 : */
322 326 : if (err != Z_OK || inbuf == Z_NULL) {
323 0 : CPLError(CE_Failure, CPLE_NotSupported, "inflateInit2 init failed");
324 : }
325 326 : stream.avail_out = Z_BUFSIZE;
326 :
327 326 : if (offset == 0) check_header(); /* skip the .gz header */
328 326 : startOff = VSIFTellL((VSILFILE*)poBaseHandle) - stream.avail_in;
329 :
330 326 : if (transparent == 0)
331 : {
332 322 : snapshot_byte_interval = MAX(Z_BUFSIZE, compressed_size / 100);
333 322 : snapshots = (GZipSnapshot*)CPLCalloc(sizeof(GZipSnapshot), (size_t) (compressed_size / snapshot_byte_interval + 1));
334 : }
335 : else
336 : {
337 4 : snapshots = NULL;
338 : }
339 326 : }
340 :
341 : /************************************************************************/
342 : /* ~VSIGZipHandle() */
343 : /************************************************************************/
344 :
345 326 : VSIGZipHandle::~VSIGZipHandle()
346 : {
347 :
348 326 : if (pszBaseFileName)
349 : {
350 : VSIFilesystemHandler *poFSHandler =
351 63 : VSIFileManager::GetHandler( "/vsigzip/" );
352 63 : ((VSIGZipFilesystemHandler*)poFSHandler)->SaveInfo(this);
353 : }
354 :
355 326 : if (stream.state != NULL) {
356 326 : inflateEnd(&(stream));
357 : }
358 :
359 326 : TRYFREE(inbuf);
360 326 : TRYFREE(outbuf);
361 :
362 326 : if (snapshots != NULL)
363 : {
364 : unsigned int i;
365 671 : for(i=0;i<compressed_size / snapshot_byte_interval + 1;i++)
366 : {
367 349 : if (snapshots[i].uncompressed_pos)
368 : {
369 279 : inflateEnd(&(snapshots[i].stream));
370 : }
371 : }
372 322 : CPLFree(snapshots);
373 : }
374 326 : CPLFree(pszBaseFileName);
375 :
376 326 : if (poBaseHandle)
377 315 : VSIFCloseL((VSILFILE*)poBaseHandle);
378 326 : }
379 :
380 : /************************************************************************/
381 : /* check_header() */
382 : /************************************************************************/
383 :
384 84 : void VSIGZipHandle::check_header()
385 : {
386 : int method; /* method byte */
387 : int flags; /* flags byte */
388 : uInt len;
389 : int c;
390 :
391 : /* Assure two bytes in the buffer so we can peek ahead -- handle case
392 : where first byte of header is at the end of the buffer after the last
393 : gzip segment */
394 84 : len = stream.avail_in;
395 84 : if (len < 2) {
396 84 : if (len) inbuf[0] = stream.next_in[0];
397 84 : errno = 0;
398 84 : len = (uInt)VSIFReadL(inbuf + len, 1, Z_BUFSIZE >> len, (VSILFILE*)poBaseHandle);
399 : if (ENABLE_DEBUG) CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
400 : VSIFTellL((VSILFILE*)poBaseHandle), offsetEndCompressedData);
401 84 : if (VSIFTellL((VSILFILE*)poBaseHandle) > offsetEndCompressedData)
402 : {
403 0 : len = len + (uInt) (offsetEndCompressedData - VSIFTellL((VSILFILE*)poBaseHandle));
404 0 : VSIFSeekL((VSILFILE*)poBaseHandle, offsetEndCompressedData, SEEK_SET);
405 : }
406 84 : if (len == 0 /* && ferror(file)*/)
407 : {
408 21 : if (VSIFTellL((VSILFILE*)poBaseHandle) != offsetEndCompressedData)
409 0 : z_err = Z_ERRNO;
410 : }
411 84 : stream.avail_in += len;
412 84 : stream.next_in = inbuf;
413 84 : if (stream.avail_in < 2) {
414 21 : transparent = stream.avail_in;
415 21 : return;
416 : }
417 : }
418 :
419 : /* Peek ahead to check the gzip magic header */
420 189 : if (stream.next_in[0] != gz_magic[0] ||
421 126 : stream.next_in[1] != gz_magic[1]) {
422 0 : transparent = 1;
423 0 : return;
424 : }
425 63 : stream.avail_in -= 2;
426 63 : stream.next_in += 2;
427 :
428 : /* Check the rest of the gzip header */
429 63 : method = get_byte();
430 63 : flags = get_byte();
431 63 : if (method != Z_DEFLATED || (flags & RESERVED) != 0) {
432 0 : z_err = Z_DATA_ERROR;
433 0 : return;
434 : }
435 :
436 : /* Discard time, xflags and OS code: */
437 63 : for (len = 0; len < 6; len++) (void)get_byte();
438 :
439 63 : if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */
440 0 : len = (uInt)get_byte();
441 0 : len += ((uInt)get_byte())<<8;
442 : /* len is garbage if EOF but the loop below will quit anyway */
443 0 : while (len-- != 0 && get_byte() != EOF) ;
444 : }
445 63 : if ((flags & ORIG_NAME) != 0) { /* skip the original file name */
446 0 : while ((c = get_byte()) != 0 && c != EOF) ;
447 : }
448 63 : if ((flags & COMMENT) != 0) { /* skip the .gz file comment */
449 0 : while ((c = get_byte()) != 0 && c != EOF) ;
450 : }
451 63 : if ((flags & HEAD_CRC) != 0) { /* skip the header crc */
452 0 : for (len = 0; len < 2; len++) (void)get_byte();
453 : }
454 63 : z_err = z_eof ? Z_DATA_ERROR : Z_OK;
455 : }
456 :
457 : /************************************************************************/
458 : /* get_byte() */
459 : /************************************************************************/
460 :
461 672 : int VSIGZipHandle::get_byte()
462 : {
463 672 : if (z_eof) return EOF;
464 672 : if (stream.avail_in == 0) {
465 0 : errno = 0;
466 0 : stream.avail_in = (uInt)VSIFReadL(inbuf, 1, Z_BUFSIZE, (VSILFILE*)poBaseHandle);
467 : if (ENABLE_DEBUG) CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
468 : VSIFTellL((VSILFILE*)poBaseHandle), offsetEndCompressedData);
469 0 : if (VSIFTellL((VSILFILE*)poBaseHandle) > offsetEndCompressedData)
470 : {
471 0 : stream.avail_in = stream.avail_in + (uInt) (offsetEndCompressedData - VSIFTellL((VSILFILE*)poBaseHandle));
472 0 : VSIFSeekL((VSILFILE*)poBaseHandle, offsetEndCompressedData, SEEK_SET);
473 : }
474 0 : if (stream.avail_in == 0) {
475 0 : z_eof = 1;
476 0 : if (VSIFTellL((VSILFILE*)poBaseHandle) != offsetEndCompressedData)
477 0 : z_err = Z_ERRNO;
478 : /*if (ferror(file)) z_err = Z_ERRNO;*/
479 0 : return EOF;
480 : }
481 0 : stream.next_in = inbuf;
482 : }
483 672 : stream.avail_in--;
484 672 : return *(stream.next_in)++;
485 : }
486 :
487 : /************************************************************************/
488 : /* gzrewind() */
489 : /************************************************************************/
490 :
491 90 : int VSIGZipHandle::gzrewind ()
492 : {
493 90 : z_err = Z_OK;
494 90 : z_eof = 0;
495 90 : stream.avail_in = 0;
496 90 : stream.next_in = inbuf;
497 90 : crc = crc32(0L, Z_NULL, 0);
498 90 : if (!transparent) (void)inflateReset(&stream);
499 90 : in = 0;
500 90 : out = 0;
501 90 : return VSIFSeekL((VSILFILE*)poBaseHandle, startOff, SEEK_SET);
502 : }
503 :
504 : /************************************************************************/
505 : /* Seek() */
506 : /************************************************************************/
507 :
508 566 : int VSIGZipHandle::Seek( vsi_l_offset nOffset, int nWhence )
509 : {
510 : /* The semantics of gzseek is different from ::Seek */
511 : /* It returns the current offset, where as ::Seek shoud return 0 */
512 : /* if successfull */
513 566 : int ret = gzseek(nOffset, nWhence);
514 566 : return (ret >= 0) ? 0 : ret;
515 : }
516 :
517 : /************************************************************************/
518 : /* gzseek() */
519 : /************************************************************************/
520 :
521 566 : int VSIGZipHandle::gzseek( vsi_l_offset offset, int whence )
522 : {
523 566 : vsi_l_offset original_offset = offset;
524 566 : int original_nWhence = whence;
525 :
526 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Seek(" CPL_FRMT_GUIB ",%d)", offset, whence);
527 :
528 566 : if (transparent)
529 : {
530 3 : stream.avail_in = 0;
531 3 : stream.next_in = inbuf;
532 3 : if (whence == SEEK_CUR)
533 : {
534 0 : if (out + offset > compressed_size)
535 : {
536 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
537 0 : return -1L;
538 : }
539 :
540 0 : offset = startOff + out + offset;
541 : }
542 3 : else if (whence == SEEK_SET)
543 : {
544 2 : if (offset > compressed_size)
545 : {
546 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
547 0 : return -1L;
548 : }
549 :
550 2 : offset = startOff + offset;
551 : }
552 1 : else if (whence == SEEK_END)
553 : {
554 : /* Commented test : because vsi_l_offset is unsigned (for the moment) */
555 : /* so no way to seek backward. See #1590 */
556 1 : if (offset > 0 /*|| -offset > compressed_size*/)
557 : {
558 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
559 0 : return -1L;
560 : }
561 :
562 1 : offset = startOff + compressed_size - offset;
563 : }
564 : else
565 : {
566 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
567 0 : return -1L;
568 : }
569 3 : if (VSIFSeekL((VSILFILE*)poBaseHandle, offset, SEEK_SET) < 0)
570 : {
571 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
572 0 : return -1L;
573 : }
574 :
575 3 : in = out = offset - startOff;
576 : if (ENABLE_DEBUG) CPLDebug("GZIP", "return " CPL_FRMT_GUIB, in);
577 3 : return (int) in;
578 : }
579 :
580 : /* whence == SEEK_END is unsuppored in original gzseek. */
581 563 : if (whence == SEEK_END)
582 : {
583 : /* If we known the uncompressed size, we can fake a jump to */
584 : /* the end of the stream */
585 63 : if (offset == 0 && uncompressed_size != 0)
586 : {
587 55 : out = uncompressed_size;
588 55 : return 1;
589 : }
590 :
591 : /* We don't know the uncompressed size. This is unfortunate. Let's do the slow version... */
592 : static int firstWarning = 1;
593 8 : if (compressed_size > 10 * 1024 * 1024 && firstWarning)
594 : {
595 : CPLError(CE_Warning, CPLE_AppDefined,
596 0 : "VSIFSeekL(xxx, SEEK_END) may be really slow on GZip streams.");
597 0 : firstWarning = 0;
598 : }
599 :
600 8 : whence = SEEK_CUR;
601 8 : offset = 1024 * 1024 * 1024;
602 8 : offset *= 1024 * 1024;
603 : }
604 :
605 508 : if (/*whence == SEEK_END ||*/
606 : z_err == Z_ERRNO || z_err == Z_DATA_ERROR) {
607 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
608 0 : return -1L;
609 : }
610 :
611 : /* Rest of function is for reading only */
612 :
613 : /* compute absolute position */
614 508 : if (whence == SEEK_CUR) {
615 8 : offset += out;
616 : }
617 :
618 : /* For a negative seek, rewind and use positive seek */
619 508 : if (offset >= out) {
620 418 : offset -= out;
621 90 : } else if (gzrewind() < 0) {
622 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
623 0 : return -1L;
624 : }
625 :
626 : unsigned int i;
627 508 : for(i=0;i<compressed_size / snapshot_byte_interval + 1;i++)
628 : {
629 508 : if (snapshots[i].uncompressed_pos == 0)
630 442 : break;
631 69 : if (snapshots[i].out <= out + offset &&
632 3 : (i == compressed_size / snapshot_byte_interval || snapshots[i+1].out == 0 || snapshots[i+1].out > out+offset))
633 : {
634 66 : if (out >= snapshots[i].out)
635 66 : break;
636 :
637 : if (ENABLE_DEBUG)
638 : CPLDebug("SNAPSHOT", "using snapshot %d : uncompressed_pos(snapshot)=" CPL_FRMT_GUIB
639 : " in(snapshot)=" CPL_FRMT_GUIB
640 : " out(snapshot)=" CPL_FRMT_GUIB
641 : " out=" CPL_FRMT_GUIB
642 : " offset=" CPL_FRMT_GUIB,
643 : i, snapshots[i].uncompressed_pos, snapshots[i].in, snapshots[i].out, out, offset);
644 0 : offset = out + offset - snapshots[i].out;
645 0 : VSIFSeekL((VSILFILE*)poBaseHandle, snapshots[i].uncompressed_pos, SEEK_SET);
646 0 : inflateEnd(&stream);
647 0 : inflateCopy(&stream, &snapshots[i].stream);
648 0 : crc = snapshots[i].crc;
649 0 : transparent = snapshots[i].transparent;
650 0 : in = snapshots[i].in;
651 0 : out = snapshots[i].out;
652 0 : break;
653 : }
654 : }
655 :
656 : /* offset is now the number of bytes to skip. */
657 :
658 508 : if (offset != 0 && outbuf == Z_NULL) {
659 50 : outbuf = (Byte*)ALLOC(Z_BUFSIZE);
660 50 : if (outbuf == Z_NULL) {
661 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
662 0 : return -1L;
663 : }
664 : }
665 :
666 508 : if (original_nWhence == SEEK_END && z_err == Z_STREAM_END)
667 : {
668 : if (ENABLE_DEBUG) CPLDebug("GZIP", "gzseek return " CPL_FRMT_GUIB, out);
669 2 : return (int) out;
670 : }
671 :
672 1213 : while (offset > 0) {
673 208 : int size = Z_BUFSIZE;
674 208 : if (offset < Z_BUFSIZE) size = (int)offset;
675 :
676 208 : int read_size = Read(outbuf, 1, (uInt)size);
677 208 : if (read_size == 0) {
678 : //CPL_VSIL_GZ_RETURN_MINUS_ONE();
679 1 : return -1L;
680 : }
681 207 : if (original_nWhence == SEEK_END)
682 : {
683 6 : if (size != read_size)
684 : {
685 6 : z_err = Z_STREAM_END;
686 6 : break;
687 : }
688 : }
689 201 : offset -= read_size;
690 : }
691 : if (ENABLE_DEBUG) CPLDebug("GZIP", "gzseek return " CPL_FRMT_GUIB, out);
692 :
693 505 : if (original_offset == 0 && original_nWhence == SEEK_END)
694 : {
695 6 : uncompressed_size = out;
696 :
697 6 : if (pszBaseFileName)
698 : {
699 6 : CPLString osCacheFilename (pszBaseFileName);
700 6 : osCacheFilename += ".properties";
701 :
702 : /* Write a .properties file to avoid seeking next time */
703 6 : VSILFILE* fpCacheLength = VSIFOpenL(osCacheFilename.c_str(), "wb");
704 6 : if (fpCacheLength)
705 : {
706 : char* pszFirstNonSpace;
707 : char szBuffer[32];
708 6 : szBuffer[31] = 0;
709 :
710 6 : CPLPrintUIntBig(szBuffer, compressed_size, 31);
711 6 : pszFirstNonSpace = szBuffer;
712 6 : while (*pszFirstNonSpace == ' ') pszFirstNonSpace ++;
713 6 : VSIFPrintfL(fpCacheLength, "compressed_size=%s\n", pszFirstNonSpace);
714 :
715 6 : CPLPrintUIntBig(szBuffer, uncompressed_size, 31);
716 6 : pszFirstNonSpace = szBuffer;
717 6 : while (*pszFirstNonSpace == ' ') pszFirstNonSpace ++;
718 6 : VSIFPrintfL(fpCacheLength, "uncompressed_size=%s\n", pszFirstNonSpace);
719 :
720 6 : VSIFCloseL(fpCacheLength);
721 6 : }
722 : }
723 : }
724 :
725 505 : return (int) out;
726 : }
727 :
728 : /************************************************************************/
729 : /* Tell() */
730 : /************************************************************************/
731 :
732 64 : vsi_l_offset VSIGZipHandle::Tell()
733 : {
734 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Tell() = " CPL_FRMT_GUIB, out);
735 64 : return out;
736 : }
737 :
738 : /************************************************************************/
739 : /* Read() */
740 : /************************************************************************/
741 :
742 2130 : size_t VSIGZipHandle::Read( void *buf, size_t nSize, size_t nMemb )
743 : {
744 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Read(%p, %d, %d)", buf, (int)nSize, (int)nMemb);
745 :
746 2130 : unsigned len = nSize * nMemb;
747 :
748 2130 : Bytef *pStart = (Bytef*)buf; /* startOffing point for crc computation */
749 : Byte *next_out; /* == stream.next_out but not forced far (for MSDOS) */
750 :
751 2130 : if (z_err == Z_DATA_ERROR || z_err == Z_ERRNO)
752 : {
753 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
754 0 : return 0;
755 : }
756 2130 : if (z_eof || z_err == Z_STREAM_END)
757 : {
758 119 : z_eof = 1;
759 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Read: Eof");
760 119 : return 0; /* EOF */
761 : }
762 :
763 2011 : next_out = (Byte*)buf;
764 2011 : stream.next_out = (Bytef*)buf;
765 2011 : stream.avail_out = len;
766 :
767 5777 : while (stream.avail_out != 0) {
768 :
769 2014 : if (transparent) {
770 : /* Copy first the lookahead bytes: */
771 2 : uInt nRead = 0;
772 2 : uInt n = stream.avail_in;
773 2 : if (n > stream.avail_out) n = stream.avail_out;
774 2 : if (n > 0) {
775 0 : memcpy (stream.next_out, stream.next_in, n);
776 0 : next_out += n;
777 0 : stream.next_out = next_out;
778 0 : stream.next_in += n;
779 0 : stream.avail_out -= n;
780 0 : stream.avail_in -= n;
781 0 : nRead += n;
782 : }
783 2 : if (stream.avail_out > 0) {
784 2 : uInt nToRead = (uInt) MIN(compressed_size - (in + nRead), stream.avail_out);
785 : uInt nReadFromFile =
786 2 : (uInt)VSIFReadL(next_out, 1, nToRead, (VSILFILE*)poBaseHandle);
787 2 : stream.avail_out -= nReadFromFile;
788 2 : nRead += nReadFromFile;
789 : }
790 2 : in += nRead;
791 2 : out += nRead;
792 2 : if (nRead < len) z_eof = 1;
793 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Read return %d", (int)(nRead / nSize));
794 2 : return (int)nRead / nSize;
795 : }
796 2012 : if (stream.avail_in == 0 && !z_eof)
797 : {
798 314 : vsi_l_offset uncompressed_pos = VSIFTellL((VSILFILE*)poBaseHandle);
799 314 : GZipSnapshot* snapshot = &snapshots[(uncompressed_pos - startOff) / snapshot_byte_interval];
800 314 : if (snapshot->uncompressed_pos == 0)
801 : {
802 276 : snapshot->crc = crc32 (crc, pStart, (uInt) (stream.next_out - pStart));
803 : if (ENABLE_DEBUG)
804 : CPLDebug("SNAPSHOT",
805 : "creating snapshot %d : uncompressed_pos=" CPL_FRMT_GUIB
806 : " in=" CPL_FRMT_GUIB
807 : " out=" CPL_FRMT_GUIB
808 : " crc=%X",
809 : (int)((uncompressed_pos - startOff) / snapshot_byte_interval),
810 : uncompressed_pos, in, out, (unsigned int)snapshot->crc);
811 276 : snapshot->uncompressed_pos = uncompressed_pos;
812 276 : inflateCopy(&snapshot->stream, &stream);
813 276 : snapshot->transparent = transparent;
814 276 : snapshot->in = in;
815 276 : snapshot->out = out;
816 :
817 276 : if (out > nLastReadOffset)
818 3 : nLastReadOffset = out;
819 : }
820 :
821 314 : errno = 0;
822 314 : stream.avail_in = (uInt)VSIFReadL(inbuf, 1, Z_BUFSIZE, (VSILFILE*)poBaseHandle);
823 : if (ENABLE_DEBUG)
824 : CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
825 : VSIFTellL((VSILFILE*)poBaseHandle), offsetEndCompressedData);
826 314 : if (VSIFTellL((VSILFILE*)poBaseHandle) > offsetEndCompressedData)
827 : {
828 : if (ENABLE_DEBUG) CPLDebug("GZIP", "avail_in before = %d", stream.avail_in);
829 283 : stream.avail_in = stream.avail_in + (uInt) (offsetEndCompressedData - VSIFTellL((VSILFILE*)poBaseHandle));
830 283 : VSIFSeekL((VSILFILE*)poBaseHandle, offsetEndCompressedData, SEEK_SET);
831 283 : if (ENABLE_DEBUG) CPLDebug("GZIP", "avail_in after = %d", stream.avail_in);
832 : }
833 314 : if (stream.avail_in == 0) {
834 0 : z_eof = 1;
835 0 : if (VSIFTellL((VSILFILE*)poBaseHandle) != offsetEndCompressedData)
836 : {
837 0 : z_err = Z_ERRNO;
838 0 : break;
839 : }
840 : /*if (ferror (file)) {
841 : z_err = Z_ERRNO;
842 : break;
843 : }*/
844 : }
845 314 : stream.next_in = inbuf;
846 : }
847 2012 : in += stream.avail_in;
848 2012 : out += stream.avail_out;
849 2012 : z_err = inflate(& (stream), Z_NO_FLUSH);
850 2012 : in -= stream.avail_in;
851 2012 : out -= stream.avail_out;
852 :
853 2012 : if (z_err == Z_STREAM_END && compressed_size != 2 ) {
854 : /* Check CRC and original size */
855 257 : crc = crc32 (crc, pStart, (uInt) (stream.next_out - pStart));
856 257 : pStart = stream.next_out;
857 257 : if (expected_crc)
858 : {
859 236 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Computed CRC = %X. Expected CRC = %X", (unsigned int)crc, expected_crc);
860 : }
861 257 : if (expected_crc != 0 && expected_crc != crc)
862 : {
863 0 : CPLError(CE_Failure, CPLE_FileIO, "CRC error. Got %X instead of %X", (unsigned int)crc, expected_crc);
864 0 : z_err = Z_DATA_ERROR;
865 : }
866 257 : else if (expected_crc == 0)
867 : {
868 21 : unsigned int read_crc = getLong();
869 21 : if (read_crc != crc)
870 : {
871 0 : CPLError(CE_Failure, CPLE_FileIO, "CRC error. Got %X instead of %X", (unsigned int)crc, read_crc);
872 0 : z_err = Z_DATA_ERROR;
873 : }
874 : else
875 : {
876 21 : (void)getLong();
877 : /* The uncompressed length returned by above getlong() may be
878 : * different from out in case of concatenated .gz files.
879 : * Check for such files:
880 : */
881 21 : check_header();
882 21 : if (z_err == Z_OK) {
883 0 : inflateReset(& (stream));
884 0 : crc = crc32(0L, Z_NULL, 0);
885 : }
886 : }
887 : }
888 : }
889 2012 : if (z_err != Z_OK || z_eof) break;
890 : }
891 2009 : crc = crc32 (crc, pStart, (uInt) (stream.next_out - pStart));
892 :
893 2009 : if (len == stream.avail_out &&
894 : (z_err == Z_DATA_ERROR || z_err == Z_ERRNO))
895 : {
896 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
897 0 : return 0;
898 : }
899 : if (ENABLE_DEBUG)
900 : CPLDebug("GZIP", "Read return %d (z_err=%d, z_eof=%d)",
901 : (int)((len - stream.avail_out) / nSize), z_err, z_eof);
902 2009 : return (int)(len - stream.avail_out) / nSize;
903 : }
904 :
905 : /************************************************************************/
906 : /* getLong() */
907 : /************************************************************************/
908 :
909 42 : uLong VSIGZipHandle::getLong ()
910 : {
911 42 : uLong x = (uLong)get_byte();
912 : int c;
913 :
914 42 : x += ((uLong)get_byte())<<8;
915 42 : x += ((uLong)get_byte())<<16;
916 42 : c = get_byte();
917 42 : if (c == EOF) z_err = Z_DATA_ERROR;
918 42 : x += ((uLong)c)<<24;
919 42 : return x;
920 : }
921 :
922 : /************************************************************************/
923 : /* Write() */
924 : /************************************************************************/
925 :
926 0 : size_t VSIGZipHandle::Write( const void *pBuffer, size_t nSize, size_t nMemb )
927 : {
928 0 : CPLError(CE_Failure, CPLE_NotSupported, "VSIFWriteL is not supported on GZip streams");
929 0 : return 0;
930 : }
931 :
932 : /************************************************************************/
933 : /* Eof() */
934 : /************************************************************************/
935 :
936 :
937 1922 : int VSIGZipHandle::Eof()
938 : {
939 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Eof()");
940 1922 : return z_eof;
941 : }
942 :
943 : /************************************************************************/
944 : /* Flush() */
945 : /************************************************************************/
946 :
947 0 : int VSIGZipHandle::Flush()
948 : {
949 0 : return 0;
950 : }
951 :
952 : /************************************************************************/
953 : /* Close() */
954 : /************************************************************************/
955 :
956 315 : int VSIGZipHandle::Close()
957 : {
958 315 : return 0;
959 : }
960 :
961 :
962 : /************************************************************************/
963 : /* ==================================================================== */
964 : /* VSIGZipWriteHandle */
965 : /* ==================================================================== */
966 : /************************************************************************/
967 :
968 : class VSIGZipWriteHandle : public VSIVirtualHandle
969 : {
970 : VSIVirtualHandle* poBaseHandle;
971 : z_stream sStream;
972 : Byte *pabyInBuf;
973 : Byte *pabyOutBuf;
974 : bool bCompressActive;
975 : vsi_l_offset nCurOffset;
976 : GUInt32 nCRC;
977 : int bRegularZLib;
978 : int bAutoCloseBaseHandle;
979 :
980 : public:
981 :
982 : VSIGZipWriteHandle(VSIVirtualHandle* poBaseHandle, int bRegularZLib, int bAutoCloseBaseHandleIn);
983 :
984 : ~VSIGZipWriteHandle();
985 :
986 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
987 : virtual vsi_l_offset Tell();
988 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
989 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
990 : virtual int Eof();
991 : virtual int Flush();
992 : virtual int Close();
993 : };
994 :
995 : /************************************************************************/
996 : /* VSIGZipWriteHandle() */
997 : /************************************************************************/
998 :
999 301 : VSIGZipWriteHandle::VSIGZipWriteHandle( VSIVirtualHandle *poBaseHandle,
1000 : int bRegularZLibIn,
1001 301 : int bAutoCloseBaseHandleIn )
1002 :
1003 : {
1004 301 : nCurOffset = 0;
1005 :
1006 301 : this->poBaseHandle = poBaseHandle;
1007 301 : bRegularZLib = bRegularZLibIn;
1008 301 : bAutoCloseBaseHandle = bAutoCloseBaseHandleIn;
1009 :
1010 301 : nCRC = crc32(0L, Z_NULL, 0);
1011 301 : sStream.zalloc = (alloc_func)0;
1012 301 : sStream.zfree = (free_func)0;
1013 301 : sStream.opaque = (voidpf)0;
1014 301 : sStream.next_in = Z_NULL;
1015 301 : sStream.next_out = Z_NULL;
1016 301 : sStream.avail_in = sStream.avail_out = 0;
1017 :
1018 301 : pabyInBuf = (Byte *) CPLMalloc( Z_BUFSIZE );
1019 301 : sStream.next_in = pabyInBuf;
1020 :
1021 301 : pabyOutBuf = (Byte *) CPLMalloc( Z_BUFSIZE );
1022 :
1023 301 : if( deflateInit2( &sStream, Z_DEFAULT_COMPRESSION,
1024 : Z_DEFLATED, (bRegularZLib) ? MAX_WBITS : -MAX_WBITS, 8,
1025 : Z_DEFAULT_STRATEGY ) != Z_OK )
1026 0 : bCompressActive = false;
1027 : else
1028 : {
1029 301 : if (!bRegularZLib)
1030 : {
1031 : char header[11];
1032 :
1033 : /* Write a very simple .gz header:
1034 : */
1035 24 : sprintf( header, "%c%c%c%c%c%c%c%c%c%c", gz_magic[0], gz_magic[1],
1036 : Z_DEFLATED, 0 /*flags*/, 0,0,0,0 /*time*/, 0 /*xflags*/,
1037 24 : 0x03 );
1038 24 : poBaseHandle->Write( header, 1, 10 );
1039 : }
1040 :
1041 301 : bCompressActive = true;
1042 : }
1043 301 : }
1044 :
1045 : /************************************************************************/
1046 : /* VSICreateGZipWritable() */
1047 : /************************************************************************/
1048 :
1049 277 : VSIVirtualHandle* VSICreateGZipWritable( VSIVirtualHandle* poBaseHandle,
1050 : int bRegularZLibIn,
1051 : int bAutoCloseBaseHandle )
1052 : {
1053 277 : return new VSIGZipWriteHandle( poBaseHandle, bRegularZLibIn, bAutoCloseBaseHandle );
1054 : }
1055 :
1056 : /************************************************************************/
1057 : /* ~VSIGZipWriteHandle() */
1058 : /************************************************************************/
1059 :
1060 301 : VSIGZipWriteHandle::~VSIGZipWriteHandle()
1061 :
1062 : {
1063 301 : if( bCompressActive )
1064 0 : Close();
1065 :
1066 301 : CPLFree( pabyInBuf );
1067 301 : CPLFree( pabyOutBuf );
1068 301 : }
1069 :
1070 : /************************************************************************/
1071 : /* Close() */
1072 : /************************************************************************/
1073 :
1074 301 : int VSIGZipWriteHandle::Close()
1075 :
1076 : {
1077 301 : if( bCompressActive )
1078 : {
1079 301 : sStream.next_out = pabyOutBuf;
1080 301 : sStream.avail_out = Z_BUFSIZE;
1081 :
1082 301 : deflate( &sStream, Z_FINISH );
1083 :
1084 301 : size_t nOutBytes = Z_BUFSIZE - sStream.avail_out;
1085 :
1086 301 : if( poBaseHandle->Write( pabyOutBuf, 1, nOutBytes ) < nOutBytes )
1087 0 : return EOF;
1088 :
1089 301 : deflateEnd( &sStream );
1090 :
1091 301 : if( !bRegularZLib )
1092 : {
1093 : GUInt32 anTrailer[2];
1094 :
1095 24 : anTrailer[0] = CPL_LSBWORD32( nCRC );
1096 24 : anTrailer[1] = CPL_LSBWORD32( (GUInt32) nCurOffset );
1097 :
1098 24 : poBaseHandle->Write( anTrailer, 1, 8 );
1099 : }
1100 :
1101 301 : if( bAutoCloseBaseHandle )
1102 : {
1103 24 : poBaseHandle->Close();
1104 :
1105 24 : delete poBaseHandle;
1106 : }
1107 :
1108 301 : bCompressActive = false;
1109 : }
1110 :
1111 301 : return 0;
1112 : }
1113 :
1114 : /************************************************************************/
1115 : /* Read() */
1116 : /************************************************************************/
1117 :
1118 0 : size_t VSIGZipWriteHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
1119 :
1120 : {
1121 0 : CPLError(CE_Failure, CPLE_NotSupported, "VSIFReadL is not supported on GZip write streams\n");
1122 0 : return 0;
1123 : }
1124 :
1125 : /************************************************************************/
1126 : /* Write() */
1127 : /************************************************************************/
1128 :
1129 10714 : size_t VSIGZipWriteHandle::Write( const void *pBuffer,
1130 : size_t nSize, size_t nMemb )
1131 :
1132 : {
1133 : int nBytesToWrite, nNextByte;
1134 :
1135 10714 : nBytesToWrite = (int) (nSize * nMemb);
1136 10714 : nNextByte = 0;
1137 :
1138 10714 : nCRC = crc32(nCRC, (const Bytef *)pBuffer, nBytesToWrite);
1139 :
1140 10714 : if( !bCompressActive )
1141 0 : return 0;
1142 :
1143 32142 : while( nNextByte < nBytesToWrite )
1144 : {
1145 10714 : sStream.next_out = pabyOutBuf;
1146 10714 : sStream.avail_out = Z_BUFSIZE;
1147 :
1148 10714 : if( sStream.avail_in > 0 )
1149 0 : memmove( pabyInBuf, sStream.next_in, sStream.avail_in );
1150 :
1151 10714 : int nNewBytesToWrite = MIN((int) (Z_BUFSIZE-sStream.avail_in),
1152 : nBytesToWrite - nNextByte);
1153 : memcpy( pabyInBuf + sStream.avail_in,
1154 : ((Byte *) pBuffer) + nNextByte,
1155 10714 : nNewBytesToWrite );
1156 :
1157 10714 : sStream.next_in = pabyInBuf;
1158 10714 : sStream.avail_in += nNewBytesToWrite;
1159 :
1160 10714 : deflate( &sStream, Z_NO_FLUSH );
1161 :
1162 10714 : size_t nOutBytes = Z_BUFSIZE - sStream.avail_out;
1163 :
1164 10714 : if( nOutBytes > 0 )
1165 : {
1166 294 : if( poBaseHandle->Write( pabyOutBuf, 1, nOutBytes ) < nOutBytes )
1167 0 : return 0;
1168 : }
1169 :
1170 10714 : nNextByte += nNewBytesToWrite;
1171 10714 : nCurOffset += nNewBytesToWrite;
1172 : }
1173 :
1174 10714 : return nMemb;
1175 : }
1176 :
1177 : /************************************************************************/
1178 : /* Flush() */
1179 : /************************************************************************/
1180 :
1181 0 : int VSIGZipWriteHandle::Flush()
1182 :
1183 : {
1184 : // we *could* do something for this but for now we choose not to.
1185 :
1186 0 : return 0;
1187 : }
1188 :
1189 : /************************************************************************/
1190 : /* Eof() */
1191 : /************************************************************************/
1192 :
1193 0 : int VSIGZipWriteHandle::Eof()
1194 :
1195 : {
1196 0 : return 1;
1197 : }
1198 :
1199 : /************************************************************************/
1200 : /* Seek() */
1201 : /************************************************************************/
1202 :
1203 0 : int VSIGZipWriteHandle::Seek( vsi_l_offset nOffset, int nWhence )
1204 :
1205 : {
1206 0 : if( nOffset == 0 && (nWhence == SEEK_END || nWhence == SEEK_CUR) )
1207 0 : return 0;
1208 0 : else if( nWhence == SEEK_SET && nOffset == nCurOffset )
1209 0 : return 0;
1210 : else
1211 : {
1212 : CPLError(CE_Failure, CPLE_NotSupported,
1213 0 : "Seeking on writable compressed data streams not supported." );
1214 :
1215 0 : return -1;
1216 : }
1217 : }
1218 :
1219 : /************************************************************************/
1220 : /* Tell() */
1221 : /************************************************************************/
1222 :
1223 0 : vsi_l_offset VSIGZipWriteHandle::Tell()
1224 :
1225 : {
1226 0 : return nCurOffset;
1227 : }
1228 :
1229 :
1230 : /************************************************************************/
1231 : /* ==================================================================== */
1232 : /* VSIGZipFilesystemHandler */
1233 : /* ==================================================================== */
1234 : /************************************************************************/
1235 :
1236 :
1237 : /************************************************************************/
1238 : /* VSIGZipFilesystemHandler() */
1239 : /************************************************************************/
1240 :
1241 712 : VSIGZipFilesystemHandler::VSIGZipFilesystemHandler()
1242 : {
1243 712 : hMutex = NULL;
1244 :
1245 712 : poHandleLastGZipFile = NULL;
1246 712 : bInSaveInfo = FALSE;
1247 712 : }
1248 :
1249 : /************************************************************************/
1250 : /* ~VSIGZipFilesystemHandler() */
1251 : /************************************************************************/
1252 :
1253 687 : VSIGZipFilesystemHandler::~VSIGZipFilesystemHandler()
1254 : {
1255 687 : if (poHandleLastGZipFile)
1256 1 : delete poHandleLastGZipFile;
1257 :
1258 687 : if( hMutex != NULL )
1259 1 : CPLDestroyMutex( hMutex );
1260 687 : hMutex = NULL;
1261 687 : }
1262 :
1263 : /************************************************************************/
1264 : /* SaveInfo() */
1265 : /************************************************************************/
1266 :
1267 63 : void VSIGZipFilesystemHandler::SaveInfo( VSIGZipHandle* poHandle )
1268 : {
1269 63 : CPLMutexHolder oHolder(&hMutex);
1270 :
1271 63 : if (bInSaveInfo)
1272 : return;
1273 63 : bInSaveInfo = TRUE;
1274 :
1275 63 : CPLAssert(poHandle->GetBaseFileName() != NULL);
1276 :
1277 63 : if (poHandleLastGZipFile &&
1278 : strcmp(poHandleLastGZipFile->GetBaseFileName(), poHandle->GetBaseFileName()) == 0)
1279 : {
1280 52 : if (poHandle->GetLastReadOffset() > poHandleLastGZipFile->GetLastReadOffset())
1281 : {
1282 0 : VSIGZipHandle* poTmp = poHandleLastGZipFile;
1283 0 : poHandleLastGZipFile = NULL;
1284 0 : delete poTmp;
1285 0 : poHandleLastGZipFile = poHandle->Duplicate();
1286 0 : poHandleLastGZipFile->CloseBaseHandle();
1287 : }
1288 : }
1289 : else
1290 : {
1291 11 : VSIGZipHandle* poTmp = poHandleLastGZipFile;
1292 11 : poHandleLastGZipFile = NULL;
1293 11 : delete poTmp;
1294 11 : poHandleLastGZipFile = poHandle->Duplicate();
1295 11 : poHandleLastGZipFile->CloseBaseHandle();
1296 : }
1297 :
1298 63 : bInSaveInfo = FALSE;
1299 : }
1300 :
1301 : /************************************************************************/
1302 : /* Open() */
1303 : /************************************************************************/
1304 :
1305 126 : VSIVirtualHandle* VSIGZipFilesystemHandler::Open( const char *pszFilename,
1306 : const char *pszAccess)
1307 : {
1308 : VSIFilesystemHandler *poFSHandler =
1309 126 : VSIFileManager::GetHandler( pszFilename + strlen("/vsigzip/"));
1310 :
1311 : /* -------------------------------------------------------------------- */
1312 : /* Is this an attempt to write a new file without update (w+) */
1313 : /* access? If so, create a writable handle for the underlying */
1314 : /* filename. */
1315 : /* -------------------------------------------------------------------- */
1316 126 : if (strchr(pszAccess, 'w') != NULL )
1317 : {
1318 26 : if( strchr(pszAccess, '+') != NULL )
1319 : {
1320 : CPLError(CE_Failure, CPLE_AppDefined,
1321 0 : "Write+update (w+) not supported for /vsigzip, only read-only or write-only.");
1322 0 : return NULL;
1323 : }
1324 :
1325 : VSIVirtualHandle* poVirtualHandle =
1326 26 : poFSHandler->Open( pszFilename + strlen("/vsigzip/"), "wb" );
1327 :
1328 26 : if (poVirtualHandle == NULL)
1329 2 : return NULL;
1330 :
1331 : else
1332 24 : return new VSIGZipWriteHandle( poVirtualHandle, strchr(pszAccess, 'z') != NULL, TRUE );
1333 : }
1334 :
1335 : /* -------------------------------------------------------------------- */
1336 : /* Otherwise we are in the read access case. */
1337 : /* -------------------------------------------------------------------- */
1338 :
1339 100 : VSIGZipHandle* poGZIPHandle = OpenGZipReadOnly(pszFilename, pszAccess);
1340 100 : if (poGZIPHandle)
1341 : /* Wrap the VSIGZipHandle inside a buffered reader that will */
1342 : /* improve dramatically performance when doing small backward */
1343 : /* seeks */
1344 52 : return VSICreateBufferedReaderHandle(poGZIPHandle);
1345 : else
1346 48 : return NULL;
1347 : }
1348 :
1349 : /************************************************************************/
1350 : /* OpenGZipReadOnly() */
1351 : /************************************************************************/
1352 :
1353 100 : VSIGZipHandle* VSIGZipFilesystemHandler::OpenGZipReadOnly( const char *pszFilename,
1354 : const char *pszAccess)
1355 : {
1356 : VSIFilesystemHandler *poFSHandler =
1357 100 : VSIFileManager::GetHandler( pszFilename + strlen("/vsigzip/"));
1358 :
1359 100 : CPLMutexHolder oHolder(&hMutex);
1360 :
1361 100 : if (poHandleLastGZipFile != NULL &&
1362 : strcmp(pszFilename + strlen("/vsigzip/"), poHandleLastGZipFile->GetBaseFileName()) == 0 &&
1363 : EQUAL(pszAccess, "rb"))
1364 : {
1365 41 : VSIGZipHandle* poHandle = poHandleLastGZipFile->Duplicate();
1366 41 : if (poHandle)
1367 41 : return poHandle;
1368 : }
1369 :
1370 : unsigned char signature[2];
1371 :
1372 : VSIVirtualHandle* poVirtualHandle =
1373 59 : poFSHandler->Open( pszFilename + strlen("/vsigzip/"), "rb" );
1374 :
1375 59 : if (poVirtualHandle == NULL)
1376 48 : return NULL;
1377 :
1378 11 : if (VSIFReadL(signature, 1, 2, (VSILFILE*)poVirtualHandle) != 2)
1379 0 : return NULL;
1380 :
1381 11 : if (signature[0] != gz_magic[0] || signature[1] != gz_magic[1])
1382 0 : return NULL;
1383 :
1384 11 : if (poHandleLastGZipFile)
1385 10 : delete poHandleLastGZipFile;
1386 11 : poHandleLastGZipFile = NULL;
1387 :
1388 11 : return new VSIGZipHandle(poVirtualHandle, pszFilename + strlen("/vsigzip/"));
1389 : }
1390 :
1391 : /************************************************************************/
1392 : /* Stat() */
1393 : /************************************************************************/
1394 :
1395 44 : int VSIGZipFilesystemHandler::Stat( const char *pszFilename,
1396 : VSIStatBufL *pStatBuf,
1397 : int nFlags )
1398 : {
1399 44 : CPLMutexHolder oHolder(&hMutex);
1400 :
1401 44 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
1402 :
1403 44 : if (poHandleLastGZipFile != NULL &&
1404 : strcmp(pszFilename+strlen("/vsigzip/"), poHandleLastGZipFile->GetBaseFileName()) == 0)
1405 : {
1406 5 : if (poHandleLastGZipFile->GetUncompressedSize() != 0)
1407 : {
1408 0 : pStatBuf->st_mode = S_IFREG;
1409 0 : pStatBuf->st_size = poHandleLastGZipFile->GetUncompressedSize();
1410 0 : return 0;
1411 : }
1412 : }
1413 :
1414 : /* Begin by doing a stat on the real file */
1415 44 : int ret = VSIStatExL(pszFilename+strlen("/vsigzip/"), pStatBuf, nFlags);
1416 :
1417 44 : if (ret == 0 && (nFlags & VSI_STAT_SIZE_FLAG))
1418 : {
1419 0 : CPLString osCacheFilename(pszFilename+strlen("/vsigzip/"));
1420 0 : osCacheFilename += ".properties";
1421 :
1422 : /* Can we save a bit of seeking by using a .properties file ? */
1423 0 : VSILFILE* fpCacheLength = VSIFOpenL(osCacheFilename.c_str(), "rb");
1424 0 : if (fpCacheLength)
1425 : {
1426 : const char* pszLine;
1427 0 : GUIntBig nCompressedSize = 0;
1428 0 : GUIntBig nUncompressedSize = 0;
1429 0 : while ((pszLine = CPLReadLineL(fpCacheLength)) != NULL)
1430 : {
1431 0 : if (EQUALN(pszLine, "compressed_size=", strlen("compressed_size=")))
1432 : {
1433 0 : const char* pszBuffer = pszLine + strlen("compressed_size=");
1434 : nCompressedSize =
1435 0 : CPLScanUIntBig(pszBuffer, strlen(pszBuffer));
1436 : }
1437 0 : else if (EQUALN(pszLine, "uncompressed_size=", strlen("uncompressed_size=")))
1438 : {
1439 0 : const char* pszBuffer = pszLine + strlen("uncompressed_size=");
1440 : nUncompressedSize =
1441 0 : CPLScanUIntBig(pszBuffer, strlen(pszBuffer));
1442 : }
1443 : }
1444 :
1445 0 : VSIFCloseL(fpCacheLength);
1446 :
1447 0 : if (nCompressedSize == (GUIntBig) pStatBuf->st_size)
1448 : {
1449 : /* Patch with the uncompressed size */
1450 0 : pStatBuf->st_size = (long)nUncompressedSize;
1451 :
1452 : VSIGZipHandle* poHandle =
1453 0 : VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename, "rb");
1454 0 : if (poHandle)
1455 : {
1456 0 : poHandle->SetUncompressedSize(nUncompressedSize);
1457 0 : SaveInfo(poHandle);
1458 0 : delete poHandle;
1459 : }
1460 :
1461 0 : return ret;
1462 : }
1463 : }
1464 :
1465 : /* No, then seek at the end of the data (slow) */
1466 : VSIGZipHandle* poHandle =
1467 0 : VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename, "rb");
1468 0 : if (poHandle)
1469 : {
1470 : GUIntBig uncompressed_size;
1471 0 : poHandle->Seek(0, SEEK_END);
1472 0 : uncompressed_size = (GUIntBig) poHandle->Tell();
1473 0 : poHandle->Seek(0, SEEK_SET);
1474 :
1475 : /* Patch with the uncompressed size */
1476 0 : pStatBuf->st_size = (long)uncompressed_size;
1477 :
1478 0 : delete poHandle;
1479 : }
1480 : else
1481 : {
1482 0 : ret = -1;
1483 0 : }
1484 : }
1485 :
1486 44 : return ret;
1487 : }
1488 :
1489 : /************************************************************************/
1490 : /* Unlink() */
1491 : /************************************************************************/
1492 :
1493 0 : int VSIGZipFilesystemHandler::Unlink( const char *pszFilename )
1494 : {
1495 0 : return -1;
1496 : }
1497 :
1498 : /************************************************************************/
1499 : /* Rename() */
1500 : /************************************************************************/
1501 :
1502 0 : int VSIGZipFilesystemHandler::Rename( const char *oldpath, const char *newpath )
1503 : {
1504 0 : return -1;
1505 : }
1506 :
1507 : /************************************************************************/
1508 : /* Mkdir() */
1509 : /************************************************************************/
1510 :
1511 0 : int VSIGZipFilesystemHandler::Mkdir( const char *pszDirname, long nMode )
1512 : {
1513 0 : return -1;
1514 : }
1515 : /************************************************************************/
1516 : /* Rmdir() */
1517 : /************************************************************************/
1518 :
1519 0 : int VSIGZipFilesystemHandler::Rmdir( const char *pszDirname )
1520 : {
1521 0 : return -1;
1522 : }
1523 :
1524 : /************************************************************************/
1525 : /* ReadDir() */
1526 : /************************************************************************/
1527 :
1528 5 : char** VSIGZipFilesystemHandler::ReadDir( const char *pszDirname )
1529 : {
1530 5 : return NULL;
1531 : }
1532 :
1533 : /************************************************************************/
1534 : /* VSIInstallGZipFileHandler() */
1535 : /************************************************************************/
1536 :
1537 :
1538 : /**
1539 : * \brief Install GZip file system handler.
1540 : *
1541 : * A special file handler is installed that allows reading on-the-fly and
1542 : * writing in GZip (.gz) files.
1543 : *
1544 : * All portions of the file system underneath the base
1545 : * path "/vsigzip/" will be handled by this driver.
1546 : *
1547 : * Additional documentation is to be found at http://trac.osgeo.org/gdal/wiki/UserDocs/ReadInZip
1548 : *
1549 : * @since GDAL 1.6.0
1550 : */
1551 :
1552 712 : void VSIInstallGZipFileHandler(void)
1553 : {
1554 712 : VSIFileManager::InstallHandler( "/vsigzip/", new VSIGZipFilesystemHandler );
1555 712 : }
1556 :
1557 :
1558 : /************************************************************************/
1559 : /* ==================================================================== */
1560 : /* VSIZipEntryFileOffset */
1561 : /* ==================================================================== */
1562 : /************************************************************************/
1563 :
1564 : class VSIZipEntryFileOffset : public VSIArchiveEntryFileOffset
1565 2443 : {
1566 : public:
1567 : unz_file_pos file_pos;
1568 :
1569 2443 : VSIZipEntryFileOffset(unz_file_pos file_pos)
1570 2443 : {
1571 2443 : this->file_pos.pos_in_zip_directory = file_pos.pos_in_zip_directory;
1572 2443 : this->file_pos.num_of_file = file_pos.num_of_file;
1573 2443 : }
1574 : };
1575 :
1576 : /************************************************************************/
1577 : /* ==================================================================== */
1578 : /* VSIZipReader */
1579 : /* ==================================================================== */
1580 : /************************************************************************/
1581 :
1582 : class VSIZipReader : public VSIArchiveReader
1583 : {
1584 : private:
1585 : unzFile unzF;
1586 : unz_file_pos file_pos;
1587 : GUIntBig nNextFileSize;
1588 : CPLString osNextFileName;
1589 : GIntBig nModifiedTime;
1590 :
1591 : void SetInfo();
1592 :
1593 : public:
1594 : VSIZipReader(const char* pszZipFileName);
1595 : virtual ~VSIZipReader();
1596 :
1597 351 : int IsValid() { return unzF != NULL; }
1598 :
1599 263 : unzFile GetUnzFileHandle() { return unzF; }
1600 :
1601 : virtual int GotoFirstFile();
1602 : virtual int GotoNextFile();
1603 2443 : virtual VSIArchiveEntryFileOffset* GetFileOffset() { return new VSIZipEntryFileOffset(file_pos); }
1604 2446 : virtual GUIntBig GetFileSize() { return nNextFileSize; }
1605 2461 : virtual CPLString GetFileName() { return osNextFileName; }
1606 3867 : virtual GIntBig GetModifiedTime() { return nModifiedTime; }
1607 : virtual int GotoFileOffset(VSIArchiveEntryFileOffset* pOffset);
1608 : };
1609 :
1610 :
1611 : /************************************************************************/
1612 : /* VSIZipReader() */
1613 : /************************************************************************/
1614 :
1615 351 : VSIZipReader::VSIZipReader(const char* pszZipFileName)
1616 : {
1617 351 : unzF = cpl_unzOpen(pszZipFileName);
1618 351 : nNextFileSize = 0;
1619 351 : nModifiedTime = 0;
1620 351 : }
1621 :
1622 : /************************************************************************/
1623 : /* ~VSIZipReader() */
1624 : /************************************************************************/
1625 :
1626 351 : VSIZipReader::~VSIZipReader()
1627 : {
1628 351 : if (unzF)
1629 345 : cpl_unzClose(unzF);
1630 351 : }
1631 :
1632 : /************************************************************************/
1633 : /* SetInfo() */
1634 : /************************************************************************/
1635 :
1636 3068 : void VSIZipReader::SetInfo()
1637 : {
1638 : char fileName[8193];
1639 : unz_file_info file_info;
1640 3068 : cpl_unzGetCurrentFileInfo (unzF, &file_info, fileName, sizeof(fileName) - 1, NULL, 0, NULL, 0);
1641 3068 : fileName[sizeof(fileName) - 1] = '\0';
1642 3068 : osNextFileName = fileName;
1643 3068 : nNextFileSize = file_info.uncompressed_size;
1644 : struct tm brokendowntime;
1645 3068 : brokendowntime.tm_sec = file_info.tmu_date.tm_sec;
1646 3068 : brokendowntime.tm_min = file_info.tmu_date.tm_min;
1647 3068 : brokendowntime.tm_hour = file_info.tmu_date.tm_hour;
1648 3068 : brokendowntime.tm_mday = file_info.tmu_date.tm_mday;
1649 3068 : brokendowntime.tm_mon = file_info.tmu_date.tm_mon;
1650 3068 : brokendowntime.tm_year = file_info.tmu_date.tm_year - 1900; /* the minizip conventions differs from the Unix one */
1651 3068 : nModifiedTime = CPLYMDHMSToUnixTime(&brokendowntime);
1652 :
1653 3068 : cpl_unzGetFilePos(unzF, &this->file_pos);
1654 3068 : }
1655 :
1656 : /************************************************************************/
1657 : /* GotoNextFile() */
1658 : /************************************************************************/
1659 :
1660 2465 : int VSIZipReader::GotoNextFile()
1661 : {
1662 2465 : if (cpl_unzGoToNextFile(unzF) != UNZ_OK)
1663 63 : return FALSE;
1664 :
1665 2402 : SetInfo();
1666 :
1667 2402 : return TRUE;
1668 : }
1669 :
1670 : /************************************************************************/
1671 : /* GotoFirstFile() */
1672 : /************************************************************************/
1673 :
1674 413 : int VSIZipReader::GotoFirstFile()
1675 : {
1676 413 : if (cpl_unzGoToFirstFile(unzF) != UNZ_OK)
1677 0 : return FALSE;
1678 :
1679 413 : SetInfo();
1680 :
1681 413 : return TRUE;
1682 : }
1683 :
1684 : /************************************************************************/
1685 : /* GotoFileOffset() */
1686 : /************************************************************************/
1687 :
1688 253 : int VSIZipReader::GotoFileOffset(VSIArchiveEntryFileOffset* pOffset)
1689 : {
1690 253 : VSIZipEntryFileOffset* pZipEntryOffset = (VSIZipEntryFileOffset*)pOffset;
1691 253 : cpl_unzGoToFilePos(unzF, &(pZipEntryOffset->file_pos));
1692 :
1693 253 : SetInfo();
1694 :
1695 253 : return TRUE;
1696 : }
1697 :
1698 : /************************************************************************/
1699 : /* ==================================================================== */
1700 : /* VSIZipFilesystemHandler */
1701 : /* ==================================================================== */
1702 : /************************************************************************/
1703 :
1704 : class VSIZipWriteHandle;
1705 :
1706 : class VSIZipFilesystemHandler : public VSIArchiveFilesystemHandler
1707 712 : {
1708 : std::map<CPLString, VSIZipWriteHandle*> oMapZipWriteHandles;
1709 :
1710 : public:
1711 : virtual ~VSIZipFilesystemHandler();
1712 :
1713 16956 : virtual const char* GetPrefix() { return "/vsizip"; }
1714 : virtual std::vector<CPLString> GetExtensions();
1715 : virtual VSIArchiveReader* CreateReader(const char* pszZipFileName);
1716 :
1717 : virtual VSIVirtualHandle *Open( const char *pszFilename,
1718 : const char *pszAccess);
1719 :
1720 : virtual VSIVirtualHandle *OpenForWrite( const char *pszFilename,
1721 : const char *pszAccess );
1722 :
1723 : virtual int Mkdir( const char *pszDirname, long nMode );
1724 : virtual char **ReadDir( const char *pszDirname );
1725 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
1726 :
1727 : void RemoveFromMap(VSIZipWriteHandle* poHandle);
1728 : };
1729 :
1730 : /************************************************************************/
1731 : /* ==================================================================== */
1732 : /* VSIZipWriteHandle */
1733 : /* ==================================================================== */
1734 : /************************************************************************/
1735 :
1736 : class VSIZipWriteHandle : public VSIVirtualHandle
1737 : {
1738 : VSIZipFilesystemHandler *poFS;
1739 : void *hZIP;
1740 : VSIZipWriteHandle *poChildInWriting;
1741 : VSIZipWriteHandle *poParent;
1742 : int bAutoDeleteParent;
1743 :
1744 : public:
1745 :
1746 : VSIZipWriteHandle(VSIZipFilesystemHandler* poFS,
1747 : void *hZIP,
1748 : VSIZipWriteHandle* poParent);
1749 :
1750 : ~VSIZipWriteHandle();
1751 :
1752 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
1753 : virtual vsi_l_offset Tell();
1754 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
1755 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
1756 : virtual int Eof();
1757 : virtual int Flush();
1758 : virtual int Close();
1759 :
1760 : void StartNewFile(VSIZipWriteHandle* poSubFile);
1761 : void StopCurrentFile();
1762 52 : void* GetHandle() { return hZIP; }
1763 53 : VSIZipWriteHandle* GetChildInWriting() { return poChildInWriting; };
1764 13 : void SetAutoDeleteParent() { bAutoDeleteParent = TRUE; }
1765 : };
1766 :
1767 : /************************************************************************/
1768 : /* ~VSIZipFilesystemHandler() */
1769 : /************************************************************************/
1770 :
1771 687 : VSIZipFilesystemHandler::~VSIZipFilesystemHandler()
1772 : {
1773 687 : std::map<CPLString,VSIZipWriteHandle*>::const_iterator iter;
1774 :
1775 687 : for( iter = oMapZipWriteHandles.begin(); iter != oMapZipWriteHandles.end(); ++iter )
1776 : {
1777 : CPLError(CE_Failure, CPLE_AppDefined, "%s has not been closed",
1778 0 : iter->first.c_str());
1779 : }
1780 687 : }
1781 :
1782 : /************************************************************************/
1783 : /* GetExtensions() */
1784 : /************************************************************************/
1785 :
1786 106634 : std::vector<CPLString> VSIZipFilesystemHandler::GetExtensions()
1787 : {
1788 106634 : std::vector<CPLString> oList;
1789 106634 : oList.push_back(".zip");
1790 106634 : oList.push_back(".kmz");
1791 106634 : oList.push_back(".dwf");
1792 106634 : oList.push_back(".ods");
1793 106634 : oList.push_back(".xlsx");
1794 0 : return oList;
1795 : }
1796 :
1797 : /************************************************************************/
1798 : /* CreateReader() */
1799 : /************************************************************************/
1800 :
1801 351 : VSIArchiveReader* VSIZipFilesystemHandler::CreateReader(const char* pszZipFileName)
1802 : {
1803 351 : VSIZipReader* poReader = new VSIZipReader(pszZipFileName);
1804 :
1805 351 : if (!poReader->IsValid())
1806 : {
1807 6 : delete poReader;
1808 6 : return NULL;
1809 : }
1810 :
1811 345 : if (!poReader->GotoFirstFile())
1812 : {
1813 0 : delete poReader;
1814 0 : return NULL;
1815 : }
1816 :
1817 345 : return poReader;
1818 : }
1819 :
1820 : /************************************************************************/
1821 : /* Open() */
1822 : /************************************************************************/
1823 :
1824 375 : VSIVirtualHandle* VSIZipFilesystemHandler::Open( const char *pszFilename,
1825 : const char *pszAccess)
1826 : {
1827 : char* zipFilename;
1828 375 : CPLString osZipInFileName;
1829 :
1830 375 : if (strchr(pszAccess, 'w') != NULL)
1831 : {
1832 57 : return OpenForWrite(pszFilename, pszAccess);
1833 : }
1834 :
1835 318 : if (strchr(pszAccess, '+') != NULL)
1836 : {
1837 : CPLError(CE_Failure, CPLE_AppDefined,
1838 0 : "Random access not supported for /vsizip");
1839 0 : return NULL;
1840 : }
1841 :
1842 318 : zipFilename = SplitFilename(pszFilename, osZipInFileName, TRUE);
1843 318 : if (zipFilename == NULL)
1844 24 : return NULL;
1845 :
1846 : {
1847 294 : CPLMutexHolder oHolder(&hMutex);
1848 294 : if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end() )
1849 : {
1850 : CPLError(CE_Failure, CPLE_AppDefined,
1851 1 : "Cannot read a zip file being written");
1852 1 : CPLFree(zipFilename);
1853 1 : return NULL;
1854 0 : }
1855 : }
1856 :
1857 293 : VSIArchiveReader* poReader = OpenArchiveFile(zipFilename, osZipInFileName);
1858 293 : if (poReader == NULL)
1859 : {
1860 30 : CPLFree(zipFilename);
1861 30 : return NULL;
1862 : }
1863 :
1864 : VSIFilesystemHandler *poFSHandler =
1865 263 : VSIFileManager::GetHandler( zipFilename);
1866 :
1867 : VSIVirtualHandle* poVirtualHandle =
1868 263 : poFSHandler->Open( zipFilename, "rb" );
1869 :
1870 263 : CPLFree(zipFilename);
1871 263 : zipFilename = NULL;
1872 :
1873 263 : if (poVirtualHandle == NULL)
1874 : {
1875 0 : delete poReader;
1876 0 : return NULL;
1877 : }
1878 :
1879 263 : unzFile unzF = ((VSIZipReader*)poReader)->GetUnzFileHandle();
1880 :
1881 263 : cpl_unzOpenCurrentFile(unzF);
1882 :
1883 263 : uLong64 pos = cpl_unzGetCurrentFileZStreamPos(unzF);
1884 :
1885 : unz_file_info file_info;
1886 263 : cpl_unzGetCurrentFileInfo (unzF, &file_info, NULL, 0, NULL, 0, NULL, 0);
1887 :
1888 263 : cpl_unzCloseCurrentFile(unzF);
1889 :
1890 263 : delete poReader;
1891 :
1892 : VSIGZipHandle* poGZIPHandle = new VSIGZipHandle(poVirtualHandle,
1893 : NULL,
1894 : pos,
1895 : file_info.compressed_size,
1896 : file_info.uncompressed_size,
1897 : file_info.crc,
1898 263 : file_info.compression_method == 0);
1899 : /* Wrap the VSIGZipHandle inside a buffered reader that will */
1900 : /* improve dramatically performance when doing small backward */
1901 : /* seeks */
1902 526 : return VSICreateBufferedReaderHandle(poGZIPHandle);
1903 : }
1904 :
1905 : /************************************************************************/
1906 : /* Mkdir() */
1907 : /************************************************************************/
1908 :
1909 1 : int VSIZipFilesystemHandler::Mkdir( const char *pszDirname, long nMode )
1910 : {
1911 1 : CPLString osDirname = pszDirname;
1912 1 : if (osDirname.size() != 0 && osDirname[osDirname.size() - 1] != '/')
1913 1 : osDirname += "/";
1914 1 : VSIVirtualHandle* poZIPHandle = OpenForWrite(osDirname, "wb");
1915 1 : if (poZIPHandle == NULL)
1916 0 : return -1;
1917 1 : delete poZIPHandle;
1918 1 : return 0;
1919 : }
1920 :
1921 : /************************************************************************/
1922 : /* ReadDir() */
1923 : /************************************************************************/
1924 :
1925 1569 : char **VSIZipFilesystemHandler::ReadDir( const char *pszDirname )
1926 : {
1927 1569 : CPLString osInArchiveSubDir;
1928 1569 : char* zipFilename = SplitFilename(pszDirname, osInArchiveSubDir, TRUE);
1929 1569 : if (zipFilename == NULL)
1930 3 : return NULL;
1931 :
1932 : {
1933 1566 : CPLMutexHolder oHolder(&hMutex);
1934 :
1935 1566 : if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end() )
1936 : {
1937 : CPLError(CE_Failure, CPLE_AppDefined,
1938 1 : "Cannot read a zip file being written");
1939 1 : CPLFree(zipFilename);
1940 1 : return NULL;
1941 0 : }
1942 : }
1943 1565 : CPLFree(zipFilename);
1944 :
1945 1565 : return VSIArchiveFilesystemHandler::ReadDir(pszDirname);
1946 : }
1947 :
1948 :
1949 : /************************************************************************/
1950 : /* Stat() */
1951 : /************************************************************************/
1952 :
1953 1068 : int VSIZipFilesystemHandler::Stat( const char *pszFilename,
1954 : VSIStatBufL *pStatBuf, int nFlags )
1955 : {
1956 1068 : CPLString osInArchiveSubDir;
1957 :
1958 1068 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
1959 :
1960 1068 : char* zipFilename = SplitFilename(pszFilename, osInArchiveSubDir, TRUE);
1961 1068 : if (zipFilename == NULL)
1962 7 : return -1;
1963 :
1964 : {
1965 1061 : CPLMutexHolder oHolder(&hMutex);
1966 :
1967 1061 : if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end() )
1968 : {
1969 : CPLError(CE_Failure, CPLE_AppDefined,
1970 0 : "Cannot read a zip file being written");
1971 0 : CPLFree(zipFilename);
1972 0 : return -1;
1973 0 : }
1974 : }
1975 1061 : CPLFree(zipFilename);
1976 :
1977 1061 : return VSIArchiveFilesystemHandler::Stat(pszFilename, pStatBuf, nFlags);
1978 : }
1979 :
1980 : /************************************************************************/
1981 : /* RemoveFromMap() */
1982 : /************************************************************************/
1983 :
1984 18 : void VSIZipFilesystemHandler::RemoveFromMap(VSIZipWriteHandle* poHandle)
1985 : {
1986 18 : CPLMutexHolder oHolder( &hMutex );
1987 18 : std::map<CPLString,VSIZipWriteHandle*>::iterator iter;
1988 :
1989 18 : for( iter = oMapZipWriteHandles.begin();
1990 : iter != oMapZipWriteHandles.end(); ++iter )
1991 : {
1992 18 : if (iter->second == poHandle)
1993 : {
1994 18 : oMapZipWriteHandles.erase(iter);
1995 18 : break;
1996 : }
1997 18 : }
1998 18 : }
1999 :
2000 : /************************************************************************/
2001 : /* OpenForWrite() */
2002 : /************************************************************************/
2003 :
2004 71 : VSIVirtualHandle* VSIZipFilesystemHandler::OpenForWrite( const char *pszFilename,
2005 : const char *pszAccess)
2006 : {
2007 : char* zipFilename;
2008 71 : CPLString osZipInFileName;
2009 :
2010 71 : CPLMutexHolder oHolder( &hMutex );
2011 :
2012 71 : zipFilename = SplitFilename(pszFilename, osZipInFileName, FALSE);
2013 71 : if (zipFilename == NULL)
2014 0 : return NULL;
2015 71 : CPLString osZipFilename = zipFilename;
2016 71 : CPLFree(zipFilename);
2017 71 : zipFilename = NULL;
2018 :
2019 : /* Invalidate cached file list */
2020 71 : std::map<CPLString,VSIArchiveContent*>::iterator iter = oFileList.find(osZipFilename);
2021 71 : if (iter != oFileList.end())
2022 : {
2023 4 : VSIArchiveContent* content = iter->second;
2024 : int i;
2025 53 : for(i=0;i<content->nEntries;i++)
2026 : {
2027 49 : delete content->entries[i].file_pos;
2028 49 : CPLFree(content->entries[i].fileName);
2029 : }
2030 4 : CPLFree(content->entries);
2031 4 : delete content;
2032 :
2033 4 : oFileList.erase(iter);
2034 : }
2035 :
2036 : VSIZipWriteHandle* poZIPHandle;
2037 :
2038 71 : if (oMapZipWriteHandles.find(osZipFilename) != oMapZipWriteHandles.end() )
2039 : {
2040 53 : if (strchr(pszAccess, '+') != NULL)
2041 : {
2042 : CPLError(CE_Failure, CPLE_AppDefined,
2043 0 : "Random access not supported for writable file in /vsizip");
2044 0 : return NULL;
2045 : }
2046 :
2047 53 : poZIPHandle = oMapZipWriteHandles[osZipFilename];
2048 :
2049 53 : if (poZIPHandle->GetChildInWriting() != NULL)
2050 : {
2051 : CPLError(CE_Failure, CPLE_AppDefined,
2052 : "Cannot create %s while another file is being written in the .zip",
2053 1 : osZipInFileName.c_str());
2054 1 : return NULL;
2055 : }
2056 :
2057 52 : poZIPHandle->StopCurrentFile();
2058 :
2059 : /* Re-add path separator when creating directories */
2060 52 : char chLastChar = pszFilename[strlen(pszFilename) - 1];
2061 52 : if (chLastChar == '/' || chLastChar == '\\')
2062 2 : osZipInFileName += chLastChar;
2063 :
2064 52 : if (CPLCreateFileInZip(poZIPHandle->GetHandle(),
2065 : osZipInFileName, NULL) != CE_None)
2066 0 : return NULL;
2067 :
2068 : VSIZipWriteHandle* poChildHandle =
2069 52 : new VSIZipWriteHandle(this, NULL, poZIPHandle);
2070 :
2071 52 : poZIPHandle->StartNewFile(poChildHandle);
2072 :
2073 52 : return poChildHandle;
2074 : }
2075 : else
2076 : {
2077 18 : char** papszOptions = NULL;
2078 18 : if ((strchr(pszAccess, '+') && osZipInFileName.size() == 0) ||
2079 : osZipInFileName.size() != 0)
2080 : {
2081 : VSIStatBufL sBuf;
2082 13 : if (VSIStatExL(osZipFilename, &sBuf, VSI_STAT_EXISTS_FLAG) == 0)
2083 12 : papszOptions = CSLAddNameValue(papszOptions, "APPEND", "TRUE");
2084 : }
2085 :
2086 18 : void* hZIP = CPLCreateZip(osZipFilename, papszOptions);
2087 18 : CSLDestroy(papszOptions);
2088 :
2089 18 : if (hZIP == NULL)
2090 0 : return NULL;
2091 :
2092 : oMapZipWriteHandles[osZipFilename] =
2093 18 : new VSIZipWriteHandle(this, hZIP, NULL);
2094 :
2095 18 : if (osZipInFileName.size() != 0)
2096 : {
2097 : VSIZipWriteHandle* poRes =
2098 13 : (VSIZipWriteHandle*)OpenForWrite(pszFilename, pszAccess);
2099 13 : if (poRes == NULL)
2100 : {
2101 0 : delete oMapZipWriteHandles[osZipFilename];
2102 0 : return NULL;
2103 : }
2104 :
2105 13 : poRes->SetAutoDeleteParent();
2106 :
2107 13 : return poRes;
2108 : }
2109 :
2110 5 : return oMapZipWriteHandles[osZipFilename];
2111 0 : }
2112 : }
2113 :
2114 :
2115 : /************************************************************************/
2116 : /* VSIZipWriteHandle() */
2117 : /************************************************************************/
2118 :
2119 70 : VSIZipWriteHandle::VSIZipWriteHandle(VSIZipFilesystemHandler* poFS,
2120 : void* hZIP,
2121 70 : VSIZipWriteHandle* poParent)
2122 : {
2123 70 : this->poFS = poFS;
2124 70 : this->hZIP = hZIP;
2125 70 : this->poParent = poParent;
2126 70 : poChildInWriting = NULL;
2127 70 : bAutoDeleteParent = FALSE;
2128 70 : }
2129 :
2130 : /************************************************************************/
2131 : /* ~VSIZipWriteHandle() */
2132 : /************************************************************************/
2133 :
2134 70 : VSIZipWriteHandle::~VSIZipWriteHandle()
2135 : {
2136 70 : Close();
2137 70 : }
2138 :
2139 : /************************************************************************/
2140 : /* Seek() */
2141 : /************************************************************************/
2142 :
2143 0 : int VSIZipWriteHandle::Seek( vsi_l_offset nOffset, int nWhence )
2144 : {
2145 : CPLError(CE_Failure, CPLE_NotSupported,
2146 0 : "VSIFSeekL() is not supported on writable Zip files");
2147 0 : return -1;
2148 : }
2149 :
2150 : /************************************************************************/
2151 : /* Tell() */
2152 : /************************************************************************/
2153 :
2154 0 : vsi_l_offset VSIZipWriteHandle::Tell()
2155 : {
2156 : CPLError(CE_Failure, CPLE_NotSupported,
2157 0 : "VSIFTellL() is not supported on writable Zip files");
2158 0 : return 0;
2159 : }
2160 :
2161 : /************************************************************************/
2162 : /* Read() */
2163 : /************************************************************************/
2164 :
2165 0 : size_t VSIZipWriteHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
2166 : {
2167 : CPLError(CE_Failure, CPLE_NotSupported,
2168 0 : "VSIFReadL() is not supported on writable Zip files");
2169 0 : return 0;
2170 : }
2171 :
2172 : /************************************************************************/
2173 : /* Write() */
2174 : /************************************************************************/
2175 :
2176 2046 : size_t VSIZipWriteHandle::Write( const void *pBuffer, size_t nSize, size_t nMemb )
2177 : {
2178 2046 : if (poParent == NULL)
2179 : {
2180 : CPLError(CE_Failure, CPLE_NotSupported,
2181 0 : "VSIFWriteL() is not supported on main Zip file or closed subfiles");
2182 0 : return 0;
2183 : }
2184 :
2185 2046 : if (CPLWriteFileInZip( poParent->hZIP, pBuffer, (int)(nSize * nMemb) ) != CE_None)
2186 0 : return 0;
2187 :
2188 2046 : return nMemb;
2189 : }
2190 :
2191 : /************************************************************************/
2192 : /* Eof() */
2193 : /************************************************************************/
2194 :
2195 0 : int VSIZipWriteHandle::Eof()
2196 : {
2197 : CPLError(CE_Failure, CPLE_NotSupported,
2198 0 : "VSIFEofL() is not supported on writable Zip files");
2199 0 : return FALSE;
2200 : }
2201 :
2202 : /************************************************************************/
2203 : /* Flush() */
2204 : /************************************************************************/
2205 :
2206 0 : int VSIZipWriteHandle::Flush()
2207 : {
2208 : CPLError(CE_Failure, CPLE_NotSupported,
2209 0 : "VSIFFlushL() is not supported on writable Zip files");
2210 0 : return 0;
2211 : }
2212 :
2213 : /************************************************************************/
2214 : /* Close() */
2215 : /************************************************************************/
2216 :
2217 126 : int VSIZipWriteHandle::Close()
2218 : {
2219 126 : if (poParent)
2220 : {
2221 52 : CPLCloseFileInZip(poParent->hZIP);
2222 52 : poParent->poChildInWriting = NULL;
2223 52 : if (bAutoDeleteParent)
2224 13 : delete poParent;
2225 52 : poParent = NULL;
2226 : }
2227 126 : if (poChildInWriting)
2228 : {
2229 0 : poChildInWriting->Close();
2230 0 : poChildInWriting = NULL;
2231 : }
2232 126 : if (hZIP)
2233 : {
2234 18 : CPLCloseZip(hZIP);
2235 18 : hZIP = NULL;
2236 :
2237 18 : poFS->RemoveFromMap(this);
2238 : }
2239 :
2240 126 : return 0;
2241 : }
2242 :
2243 : /************************************************************************/
2244 : /* StopCurrentFile() */
2245 : /************************************************************************/
2246 :
2247 52 : void VSIZipWriteHandle::StopCurrentFile()
2248 : {
2249 52 : if (poChildInWriting)
2250 0 : poChildInWriting->Close();
2251 52 : poChildInWriting = NULL;
2252 52 : }
2253 :
2254 : /************************************************************************/
2255 : /* StartNewFile() */
2256 : /************************************************************************/
2257 :
2258 52 : void VSIZipWriteHandle::StartNewFile(VSIZipWriteHandle* poSubFile)
2259 : {
2260 52 : poChildInWriting = poSubFile;
2261 52 : }
2262 :
2263 : /************************************************************************/
2264 : /* VSIInstallZipFileHandler() */
2265 : /************************************************************************/
2266 :
2267 :
2268 : /**
2269 : * \brief Install ZIP file system handler.
2270 : *
2271 : * A special file handler is installed that allows reading on-the-fly in ZIP
2272 : * (.zip) archives.
2273 : *
2274 : * All portions of the file system underneath the base path "/vsizip/" will be
2275 : * handled by this driver.
2276 : *
2277 : * The syntax to open a file inside a zip file is /vsizip/path/to/the/file.zip/path/inside/the/zip/file
2278 : * were path/to/the/file.zip is relative or absolute and path/inside/the/zip/file
2279 : * is the relative path to the file inside the archive.
2280 : *
2281 : * If the path is absolute, it should begin with a / on a Unix-like OS (or C:\ on Windows),
2282 : * so the line looks like /vsizip//home/gdal/...
2283 : * For example gdalinfo /vsizip/myarchive.zip/subdir1/file1.tif
2284 : *
2285 : * Syntaxic sugar : if the .zip file contains only one file located at its root,
2286 : * just mentionning "/vsizip/path/to/the/file.zip" will work
2287 : *
2288 : * VSIStatL() will return the uncompressed size in st_size member and file
2289 : * nature- file or directory - in st_mode member.
2290 : *
2291 : * Directory listing is available through VSIReadDir().
2292 : *
2293 : * Since GDAL 1.8.0, write capabilities are available. They allow creating
2294 : * a new zip file and adding new files to an already existing (or just created)
2295 : * zip file. Read and write operations cannot be interleaved : the new zip must
2296 : * be closed before being re-opened for read.
2297 : *
2298 : * Additional documentation is to be found at http://trac.osgeo.org/gdal/wiki/UserDocs/ReadInZip
2299 : *
2300 : * @since GDAL 1.6.0
2301 : */
2302 :
2303 712 : void VSIInstallZipFileHandler(void)
2304 : {
2305 712 : VSIFileManager::InstallHandler( "/vsizip/", new VSIZipFilesystemHandler() );
2306 712 : }
2307 :
|
