1 : /******************************************************************************
2 : * $Id: cpl_vsil_gzip.cpp 23995 2012-02-19 22:39:17Z rouault $
3 : *
4 : * Project: CPL - Common Portability Library
5 : * Purpose: Implement VSI large file api for gz/zip files (.gz and .zip).
6 : * Author: Even Rouault, even.rouault at mines-paris.org
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2007, Even Rouault
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : /* gzio.c -- IO on .gz files
31 : Copyright (C) 1995-2005 Jean-loup Gailly.
32 :
33 : This software is provided 'as-is', without any express or implied
34 : warranty. In no event will the authors be held liable for any damages
35 : arising from the use of this software.
36 :
37 : Permission is granted to anyone to use this software for any purpose,
38 : including commercial applications, and to alter it and redistribute it
39 : freely, subject to the following restrictions:
40 :
41 : 1. The origin of this software must not be misrepresented; you must not
42 : claim that you wrote the original software. If you use this software
43 : in a product, an acknowledgment in the product documentation would be
44 : appreciated but is not required.
45 : 2. Altered source versions must be plainly marked as such, and must not be
46 : misrepresented as being the original software.
47 : 3. This notice may not be removed or altered from any source distribution.
48 :
49 : Jean-loup Gailly Mark Adler
50 : jloup@gzip.org madler@alumni.caltech.edu
51 :
52 :
53 : The data format used by the zlib library is described by RFCs (Request for
54 : Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
55 : (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
56 : */
57 :
58 :
59 : /* This file contains a refactoring of gzio.c from zlib project.
60 :
61 : It replaces classical calls operating on FILE* by calls to the VSI large file
62 : API. It also adds the capability to seek at the end of the file, which is not
63 : implemented in original gzSeek. It also implements a concept of in-memory "snapshots",
64 : that are a way of improving efficiency while seeking GZip files. Snapshots are
65 : ceated regularly when decompressing the data a snapshot of the gzip state.
66 : Later we can seek directly in the compressed data to the closest snapshot in order to
67 : reduce the amount of data to uncompress again.
68 :
69 : For .gz files, an effort is done to cache the size of the uncompressed data in
70 : a .gz.properties file, so that we don't need to seek at the end of the file
71 : each time a Stat() is done.
72 :
73 : For .zip and .gz, both reading and writing are supported, but just one mode at a time
74 : (read-only or write-only)
75 : */
76 :
77 :
78 : #include "cpl_vsi_virtual.h"
79 : #include "cpl_string.h"
80 : #include "cpl_multiproc.h"
81 : #include <map>
82 :
83 : #include <zlib.h>
84 : #include "cpl_minizip_unzip.h"
85 : #include "cpl_time.h"
86 :
87 : CPL_CVSID("$Id: cpl_vsil_gzip.cpp 23995 2012-02-19 22:39:17Z rouault $");
88 :
89 : #define Z_BUFSIZE 65536 /* original size is 16384 */
90 : static int const gz_magic[2] = {0x1f, 0x8b}; /* gzip magic header */
91 :
92 : /* gzip flag byte */
93 : #define ASCII_FLAG 0x01 /* bit 0 set: file probably ascii text */
94 : #define HEAD_CRC 0x02 /* bit 1 set: header CRC present */
95 : #define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */
96 : #define ORIG_NAME 0x08 /* bit 3 set: original file name present */
97 : #define COMMENT 0x10 /* bit 4 set: file comment present */
98 : #define RESERVED 0xE0 /* bits 5..7: reserved */
99 :
100 : #define ALLOC(size) malloc(size)
101 : #define TRYFREE(p) {if (p) free(p);}
102 :
103 : #define CPL_VSIL_GZ_RETURN_MINUS_ONE() \
104 : CPLError(CE_Failure, CPLE_AppDefined, "In file %s, at line %d, return -1", __FILE__, __LINE__)
105 :
106 : #define ENABLE_DEBUG 0
107 :
108 : /************************************************************************/
109 : /* ==================================================================== */
110 : /* VSIGZipHandle */
111 : /* ==================================================================== */
112 : /************************************************************************/
113 :
114 : typedef struct
115 : {
116 : vsi_l_offset uncompressed_pos;
117 : z_stream stream;
118 : uLong crc;
119 : int transparent;
120 : vsi_l_offset in;
121 : vsi_l_offset out;
122 : } GZipSnapshot;
123 :
124 : class VSIGZipHandle : public VSIVirtualHandle
125 : {
126 : VSIVirtualHandle* poBaseHandle;
127 : vsi_l_offset offset;
128 : vsi_l_offset compressed_size;
129 : vsi_l_offset uncompressed_size;
130 : vsi_l_offset offsetEndCompressedData;
131 : unsigned int expected_crc;
132 : char *pszBaseFileName; /* optional */
133 :
134 : /* Fields from gz_stream structure */
135 : z_stream stream;
136 : int z_err; /* error code for last stream operation */
137 : int z_eof; /* set if end of input file */
138 : Byte *inbuf; /* input buffer */
139 : Byte *outbuf; /* output buffer */
140 : uLong crc; /* crc32 of uncompressed data */
141 : int transparent; /* 1 if input file is not a .gz file */
142 : vsi_l_offset startOff; /* startOff of compressed data in file (header skipped) */
143 : vsi_l_offset in; /* bytes into deflate or inflate */
144 : vsi_l_offset out; /* bytes out of deflate or inflate */
145 : vsi_l_offset nLastReadOffset;
146 :
147 : GZipSnapshot* snapshots;
148 : vsi_l_offset snapshot_byte_interval; /* number of compressed bytes at which we create a "snapshot" */
149 :
150 : void check_header();
151 : int get_byte();
152 : int gzseek( vsi_l_offset nOffset, int nWhence );
153 : int gzrewind ();
154 : uLong getLong ();
155 :
156 : public:
157 :
158 : VSIGZipHandle(VSIVirtualHandle* poBaseHandle,
159 : const char* pszBaseFileName,
160 : vsi_l_offset offset = 0,
161 : vsi_l_offset compressed_size = 0,
162 : vsi_l_offset uncompressed_size = 0,
163 : unsigned int expected_crc = 0,
164 : int transparent = 0);
165 : ~VSIGZipHandle();
166 :
167 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
168 : virtual vsi_l_offset Tell();
169 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
170 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
171 : virtual int Eof();
172 : virtual int Flush();
173 : virtual int Close();
174 :
175 : VSIGZipHandle* Duplicate();
176 : void CloseBaseHandle();
177 :
178 188 : vsi_l_offset GetLastReadOffset() { return nLastReadOffset; }
179 580 : const char* GetBaseFileName() { return pszBaseFileName; }
180 :
181 0 : void SetUncompressedSize(vsi_l_offset nUncompressedSize) { uncompressed_size = nUncompressedSize; }
182 10 : vsi_l_offset GetUncompressedSize() { return uncompressed_size; }
183 : };
184 :
185 :
186 : class VSIGZipFilesystemHandler : public VSIFilesystemHandler
187 : {
188 : void* hMutex;
189 : VSIGZipHandle* poHandleLastGZipFile;
190 : int bInSaveInfo;
191 :
192 : public:
193 : VSIGZipFilesystemHandler();
194 : ~VSIGZipFilesystemHandler();
195 :
196 : virtual VSIVirtualHandle *Open( const char *pszFilename,
197 : const char *pszAccess);
198 : VSIGZipHandle *OpenGZipReadOnly( const char *pszFilename,
199 : const char *pszAccess);
200 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
201 : virtual int Unlink( const char *pszFilename );
202 : virtual int Rename( const char *oldpath, const char *newpath );
203 : virtual int Mkdir( const char *pszDirname, long nMode );
204 : virtual int Rmdir( const char *pszDirname );
205 : virtual char **ReadDir( const char *pszDirname );
206 :
207 : void SaveInfo( VSIGZipHandle* poHandle );
208 : };
209 :
210 :
211 : /************************************************************************/
212 : /* Duplicate() */
213 : /************************************************************************/
214 :
215 94 : VSIGZipHandle* VSIGZipHandle::Duplicate()
216 : {
217 94 : CPLAssert (offset == 0);
218 94 : CPLAssert (compressed_size != 0);
219 94 : CPLAssert (pszBaseFileName != NULL);
220 :
221 : VSIFilesystemHandler *poFSHandler =
222 94 : VSIFileManager::GetHandler( pszBaseFileName );
223 :
224 : VSIVirtualHandle* poNewBaseHandle =
225 94 : poFSHandler->Open( pszBaseFileName, "rb" );
226 :
227 94 : if (poNewBaseHandle == NULL)
228 0 : return NULL;
229 :
230 : VSIGZipHandle* poHandle = new VSIGZipHandle(poNewBaseHandle,
231 : pszBaseFileName,
232 : 0,
233 : compressed_size,
234 94 : uncompressed_size);
235 :
236 94 : poHandle->nLastReadOffset = nLastReadOffset;
237 :
238 : /* Most important : duplicate the snapshots ! */
239 :
240 : unsigned int i;
241 100 : for(i=0;i<compressed_size / snapshot_byte_interval + 1;i++)
242 : {
243 94 : if (snapshots[i].uncompressed_pos == 0)
244 88 : break;
245 :
246 6 : poHandle->snapshots[i].uncompressed_pos = snapshots[i].uncompressed_pos;
247 6 : inflateCopy( &poHandle->snapshots[i].stream, &snapshots[i].stream);
248 6 : poHandle->snapshots[i].crc = snapshots[i].crc;
249 6 : poHandle->snapshots[i].transparent = snapshots[i].transparent;
250 6 : poHandle->snapshots[i].in = snapshots[i].in;
251 6 : poHandle->snapshots[i].out = snapshots[i].out;
252 : }
253 :
254 94 : return poHandle;
255 : }
256 :
257 : /************************************************************************/
258 : /* CloseBaseHandle() */
259 : /************************************************************************/
260 :
261 22 : void VSIGZipHandle::CloseBaseHandle()
262 : {
263 22 : if (poBaseHandle)
264 22 : VSIFCloseL((VSILFILE*)poBaseHandle);
265 22 : poBaseHandle = NULL;
266 22 : }
267 :
268 : /************************************************************************/
269 : /* VSIGZipHandle() */
270 : /************************************************************************/
271 :
272 514 : VSIGZipHandle::VSIGZipHandle(VSIVirtualHandle* poBaseHandle,
273 : const char* pszBaseFileName,
274 : vsi_l_offset offset,
275 : vsi_l_offset compressed_size,
276 : vsi_l_offset uncompressed_size,
277 : unsigned int expected_crc,
278 514 : int transparent)
279 : {
280 514 : this->poBaseHandle = poBaseHandle;
281 514 : this->expected_crc = expected_crc;
282 514 : this->pszBaseFileName = (pszBaseFileName) ? CPLStrdup(pszBaseFileName) : NULL;
283 514 : this->offset = offset;
284 1006 : if (compressed_size || transparent)
285 : {
286 492 : this->compressed_size = compressed_size;
287 : }
288 : else
289 : {
290 22 : VSIFSeekL((VSILFILE*)poBaseHandle, 0, SEEK_END);
291 22 : this->compressed_size = VSIFTellL((VSILFILE*)poBaseHandle) - offset;
292 22 : compressed_size = this->compressed_size;
293 : }
294 514 : this->uncompressed_size = uncompressed_size;
295 514 : offsetEndCompressedData = offset + compressed_size;
296 :
297 514 : VSIFSeekL((VSILFILE*)poBaseHandle, offset, SEEK_SET);
298 :
299 514 : nLastReadOffset = 0;
300 514 : stream.zalloc = (alloc_func)0;
301 514 : stream.zfree = (free_func)0;
302 514 : stream.opaque = (voidpf)0;
303 514 : stream.next_in = inbuf = Z_NULL;
304 514 : stream.next_out = outbuf = Z_NULL;
305 514 : stream.avail_in = stream.avail_out = 0;
306 514 : z_err = Z_OK;
307 514 : z_eof = 0;
308 514 : in = 0;
309 514 : out = 0;
310 514 : crc = crc32(0L, Z_NULL, 0);
311 514 : this->transparent = transparent;
312 :
313 514 : stream.next_in = inbuf = (Byte*)ALLOC(Z_BUFSIZE);
314 :
315 514 : int err = inflateInit2(&(stream), -MAX_WBITS);
316 : /* windowBits is passed < 0 to tell that there is no zlib header.
317 : * Note that in this case inflate *requires* an extra "dummy" byte
318 : * after the compressed stream in order to complete decompression and
319 : * return Z_STREAM_END. Here the gzip CRC32 ensures that 4 bytes are
320 : * present after the compressed stream.
321 : */
322 514 : if (err != Z_OK || inbuf == Z_NULL) {
323 0 : CPLError(CE_Failure, CPLE_NotSupported, "inflateInit2 init failed");
324 : }
325 514 : stream.avail_out = Z_BUFSIZE;
326 :
327 514 : if (offset == 0) check_header(); /* skip the .gz header */
328 514 : startOff = VSIFTellL((VSILFILE*)poBaseHandle) - stream.avail_in;
329 :
330 514 : if (transparent == 0)
331 : {
332 506 : snapshot_byte_interval = MAX(Z_BUFSIZE, compressed_size / 100);
333 506 : snapshots = (GZipSnapshot*)CPLCalloc(sizeof(GZipSnapshot), (size_t) (compressed_size / snapshot_byte_interval + 1));
334 : }
335 : else
336 : {
337 8 : snapshots = NULL;
338 : }
339 514 : }
340 :
341 : /************************************************************************/
342 : /* ~VSIGZipHandle() */
343 : /************************************************************************/
344 :
345 514 : VSIGZipHandle::~VSIGZipHandle()
346 : {
347 :
348 514 : if (pszBaseFileName)
349 : {
350 : VSIFilesystemHandler *poFSHandler =
351 116 : VSIFileManager::GetHandler( "/vsigzip/" );
352 116 : ((VSIGZipFilesystemHandler*)poFSHandler)->SaveInfo(this);
353 : }
354 :
355 514 : if (stream.state != NULL) {
356 514 : inflateEnd(&(stream));
357 : }
358 :
359 514 : TRYFREE(inbuf);
360 514 : TRYFREE(outbuf);
361 :
362 514 : if (snapshots != NULL)
363 : {
364 : unsigned int i;
365 1064 : for(i=0;i<compressed_size / snapshot_byte_interval + 1;i++)
366 : {
367 558 : if (snapshots[i].uncompressed_pos)
368 : {
369 426 : inflateEnd(&(snapshots[i].stream));
370 : }
371 : }
372 506 : CPLFree(snapshots);
373 : }
374 514 : CPLFree(pszBaseFileName);
375 :
376 514 : if (poBaseHandle)
377 492 : VSIFCloseL((VSILFILE*)poBaseHandle);
378 514 : }
379 :
380 : /************************************************************************/
381 : /* check_header() */
382 : /************************************************************************/
383 :
384 154 : void VSIGZipHandle::check_header()
385 : {
386 : int method; /* method byte */
387 : int flags; /* flags byte */
388 : uInt len;
389 : int c;
390 :
391 : /* Assure two bytes in the buffer so we can peek ahead -- handle case
392 : where first byte of header is at the end of the buffer after the last
393 : gzip segment */
394 154 : len = stream.avail_in;
395 154 : if (len < 2) {
396 154 : if (len) inbuf[0] = stream.next_in[0];
397 154 : errno = 0;
398 154 : len = (uInt)VSIFReadL(inbuf + len, 1, Z_BUFSIZE >> len, (VSILFILE*)poBaseHandle);
399 : if (ENABLE_DEBUG) CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
400 : VSIFTellL((VSILFILE*)poBaseHandle), offsetEndCompressedData);
401 154 : if (VSIFTellL((VSILFILE*)poBaseHandle) > offsetEndCompressedData)
402 : {
403 0 : len = len + (uInt) (offsetEndCompressedData - VSIFTellL((VSILFILE*)poBaseHandle));
404 0 : VSIFSeekL((VSILFILE*)poBaseHandle, offsetEndCompressedData, SEEK_SET);
405 : }
406 154 : if (len == 0 /* && ferror(file)*/)
407 : {
408 38 : if (VSIFTellL((VSILFILE*)poBaseHandle) != offsetEndCompressedData)
409 0 : z_err = Z_ERRNO;
410 : }
411 154 : stream.avail_in += len;
412 154 : stream.next_in = inbuf;
413 154 : if (stream.avail_in < 2) {
414 38 : transparent = stream.avail_in;
415 38 : return;
416 : }
417 : }
418 :
419 : /* Peek ahead to check the gzip magic header */
420 348 : if (stream.next_in[0] != gz_magic[0] ||
421 232 : stream.next_in[1] != gz_magic[1]) {
422 0 : transparent = 1;
423 0 : return;
424 : }
425 116 : stream.avail_in -= 2;
426 116 : stream.next_in += 2;
427 :
428 : /* Check the rest of the gzip header */
429 116 : method = get_byte();
430 116 : flags = get_byte();
431 116 : if (method != Z_DEFLATED || (flags & RESERVED) != 0) {
432 0 : z_err = Z_DATA_ERROR;
433 0 : return;
434 : }
435 :
436 : /* Discard time, xflags and OS code: */
437 116 : for (len = 0; len < 6; len++) (void)get_byte();
438 :
439 116 : if ((flags & EXTRA_FIELD) != 0) { /* skip the extra field */
440 0 : len = (uInt)get_byte();
441 0 : len += ((uInt)get_byte())<<8;
442 : /* len is garbage if EOF but the loop below will quit anyway */
443 0 : while (len-- != 0 && get_byte() != EOF) ;
444 : }
445 116 : if ((flags & ORIG_NAME) != 0) { /* skip the original file name */
446 0 : while ((c = get_byte()) != 0 && c != EOF) ;
447 : }
448 116 : if ((flags & COMMENT) != 0) { /* skip the .gz file comment */
449 0 : while ((c = get_byte()) != 0 && c != EOF) ;
450 : }
451 116 : if ((flags & HEAD_CRC) != 0) { /* skip the header crc */
452 0 : for (len = 0; len < 2; len++) (void)get_byte();
453 : }
454 116 : z_err = z_eof ? Z_DATA_ERROR : Z_OK;
455 : }
456 :
457 : /************************************************************************/
458 : /* get_byte() */
459 : /************************************************************************/
460 :
461 1232 : int VSIGZipHandle::get_byte()
462 : {
463 1232 : if (z_eof) return EOF;
464 1232 : if (stream.avail_in == 0) {
465 0 : errno = 0;
466 0 : stream.avail_in = (uInt)VSIFReadL(inbuf, 1, Z_BUFSIZE, (VSILFILE*)poBaseHandle);
467 : if (ENABLE_DEBUG) CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
468 : VSIFTellL((VSILFILE*)poBaseHandle), offsetEndCompressedData);
469 0 : if (VSIFTellL((VSILFILE*)poBaseHandle) > offsetEndCompressedData)
470 : {
471 0 : stream.avail_in = stream.avail_in + (uInt) (offsetEndCompressedData - VSIFTellL((VSILFILE*)poBaseHandle));
472 0 : VSIFSeekL((VSILFILE*)poBaseHandle, offsetEndCompressedData, SEEK_SET);
473 : }
474 0 : if (stream.avail_in == 0) {
475 0 : z_eof = 1;
476 0 : if (VSIFTellL((VSILFILE*)poBaseHandle) != offsetEndCompressedData)
477 0 : z_err = Z_ERRNO;
478 : /*if (ferror(file)) z_err = Z_ERRNO;*/
479 0 : return EOF;
480 : }
481 0 : stream.next_in = inbuf;
482 : }
483 1232 : stream.avail_in--;
484 1232 : return *(stream.next_in)++;
485 : }
486 :
487 : /************************************************************************/
488 : /* gzrewind() */
489 : /************************************************************************/
490 :
491 180 : int VSIGZipHandle::gzrewind ()
492 : {
493 180 : z_err = Z_OK;
494 180 : z_eof = 0;
495 180 : stream.avail_in = 0;
496 180 : stream.next_in = inbuf;
497 180 : crc = crc32(0L, Z_NULL, 0);
498 180 : if (!transparent) (void)inflateReset(&stream);
499 180 : in = 0;
500 180 : out = 0;
501 180 : return VSIFSeekL((VSILFILE*)poBaseHandle, startOff, SEEK_SET);
502 : }
503 :
504 : /************************************************************************/
505 : /* Seek() */
506 : /************************************************************************/
507 :
508 984 : int VSIGZipHandle::Seek( vsi_l_offset nOffset, int nWhence )
509 : {
510 : /* The semantics of gzseek is different from ::Seek */
511 : /* It returns the current offset, where as ::Seek shoud return 0 */
512 : /* if successfull */
513 984 : int ret = gzseek(nOffset, nWhence);
514 984 : return (ret >= 0) ? 0 : ret;
515 : }
516 :
517 : /************************************************************************/
518 : /* gzseek() */
519 : /************************************************************************/
520 :
521 984 : int VSIGZipHandle::gzseek( vsi_l_offset offset, int whence )
522 : {
523 984 : vsi_l_offset original_offset = offset;
524 984 : int original_nWhence = whence;
525 :
526 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Seek(" CPL_FRMT_GUIB ",%d)", offset, whence);
527 :
528 984 : if (transparent)
529 : {
530 6 : stream.avail_in = 0;
531 6 : stream.next_in = inbuf;
532 6 : if (whence == SEEK_CUR)
533 : {
534 0 : if (out + offset > compressed_size)
535 : {
536 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
537 0 : return -1L;
538 : }
539 :
540 0 : offset = startOff + out + offset;
541 : }
542 6 : else if (whence == SEEK_SET)
543 : {
544 4 : if (offset > compressed_size)
545 : {
546 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
547 0 : return -1L;
548 : }
549 :
550 4 : offset = startOff + offset;
551 : }
552 2 : else if (whence == SEEK_END)
553 : {
554 : /* Commented test : because vsi_l_offset is unsigned (for the moment) */
555 : /* so no way to seek backward. See #1590 */
556 2 : if (offset > 0 /*|| -offset > compressed_size*/)
557 : {
558 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
559 0 : return -1L;
560 : }
561 :
562 2 : offset = startOff + compressed_size - offset;
563 : }
564 : else
565 : {
566 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
567 0 : return -1L;
568 : }
569 6 : if (VSIFSeekL((VSILFILE*)poBaseHandle, offset, SEEK_SET) < 0)
570 : {
571 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
572 0 : return -1L;
573 : }
574 :
575 6 : in = out = offset - startOff;
576 : if (ENABLE_DEBUG) CPLDebug("GZIP", "return " CPL_FRMT_GUIB, in);
577 6 : return (int) in;
578 : }
579 :
580 : /* whence == SEEK_END is unsuppored in original gzseek. */
581 978 : if (whence == SEEK_END)
582 : {
583 : /* If we known the uncompressed size, we can fake a jump to */
584 : /* the end of the stream */
585 108 : if (offset == 0 && uncompressed_size != 0)
586 : {
587 96 : out = uncompressed_size;
588 96 : return 1;
589 : }
590 :
591 : /* We don't know the uncompressed size. This is unfortunate. Let's do the slow version... */
592 : static int firstWarning = 1;
593 12 : if (compressed_size > 10 * 1024 * 1024 && firstWarning)
594 : {
595 : CPLError(CE_Warning, CPLE_AppDefined,
596 0 : "VSIFSeekL(xxx, SEEK_END) may be really slow on GZip streams.");
597 0 : firstWarning = 0;
598 : }
599 :
600 12 : whence = SEEK_CUR;
601 12 : offset = 1024 * 1024 * 1024;
602 12 : offset *= 1024 * 1024;
603 : }
604 :
605 882 : if (/*whence == SEEK_END ||*/
606 : z_err == Z_ERRNO || z_err == Z_DATA_ERROR) {
607 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
608 0 : return -1L;
609 : }
610 :
611 : /* Rest of function is for reading only */
612 :
613 : /* compute absolute position */
614 882 : if (whence == SEEK_CUR) {
615 12 : offset += out;
616 : }
617 :
618 : /* For a negative seek, rewind and use positive seek */
619 882 : if (offset >= out) {
620 702 : offset -= out;
621 180 : } else if (gzrewind() < 0) {
622 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
623 0 : return -1L;
624 : }
625 :
626 : unsigned int i;
627 890 : for(i=0;i<compressed_size / snapshot_byte_interval + 1;i++)
628 : {
629 890 : if (snapshots[i].uncompressed_pos == 0)
630 738 : break;
631 198 : if (snapshots[i].out <= out + offset &&
632 46 : (i == compressed_size / snapshot_byte_interval || snapshots[i+1].out == 0 || snapshots[i+1].out > out+offset))
633 : {
634 144 : if (out >= snapshots[i].out)
635 144 : break;
636 :
637 : if (ENABLE_DEBUG)
638 : CPLDebug("SNAPSHOT", "using snapshot %d : uncompressed_pos(snapshot)=" CPL_FRMT_GUIB
639 : " in(snapshot)=" CPL_FRMT_GUIB
640 : " out(snapshot)=" CPL_FRMT_GUIB
641 : " out=" CPL_FRMT_GUIB
642 : " offset=" CPL_FRMT_GUIB,
643 : i, snapshots[i].uncompressed_pos, snapshots[i].in, snapshots[i].out, out, offset);
644 0 : offset = out + offset - snapshots[i].out;
645 0 : VSIFSeekL((VSILFILE*)poBaseHandle, snapshots[i].uncompressed_pos, SEEK_SET);
646 0 : inflateEnd(&stream);
647 0 : inflateCopy(&stream, &snapshots[i].stream);
648 0 : crc = snapshots[i].crc;
649 0 : transparent = snapshots[i].transparent;
650 0 : in = snapshots[i].in;
651 0 : out = snapshots[i].out;
652 0 : break;
653 : }
654 : }
655 :
656 : /* offset is now the number of bytes to skip. */
657 :
658 882 : if (offset != 0 && outbuf == Z_NULL) {
659 82 : outbuf = (Byte*)ALLOC(Z_BUFSIZE);
660 82 : if (outbuf == Z_NULL) {
661 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
662 0 : return -1L;
663 : }
664 : }
665 :
666 882 : if (original_nWhence == SEEK_END && z_err == Z_STREAM_END)
667 : {
668 : if (ENABLE_DEBUG) CPLDebug("GZIP", "gzseek return " CPL_FRMT_GUIB, out);
669 2 : return (int) out;
670 : }
671 :
672 2164 : while (offset > 0) {
673 416 : int size = Z_BUFSIZE;
674 416 : if (offset < Z_BUFSIZE) size = (int)offset;
675 :
676 416 : int read_size = Read(outbuf, 1, (uInt)size);
677 416 : if (read_size == 0) {
678 : //CPL_VSIL_GZ_RETURN_MINUS_ONE();
679 2 : return -1L;
680 : }
681 414 : if (original_nWhence == SEEK_END)
682 : {
683 10 : if (size != read_size)
684 : {
685 10 : z_err = Z_STREAM_END;
686 10 : break;
687 : }
688 : }
689 404 : offset -= read_size;
690 : }
691 : if (ENABLE_DEBUG) CPLDebug("GZIP", "gzseek return " CPL_FRMT_GUIB, out);
692 :
693 878 : if (original_offset == 0 && original_nWhence == SEEK_END)
694 : {
695 10 : uncompressed_size = out;
696 :
697 10 : if (pszBaseFileName)
698 : {
699 10 : CPLString osCacheFilename (pszBaseFileName);
700 10 : osCacheFilename += ".properties";
701 :
702 : /* Write a .properties file to avoid seeking next time */
703 10 : VSILFILE* fpCacheLength = VSIFOpenL(osCacheFilename.c_str(), "wb");
704 10 : if (fpCacheLength)
705 : {
706 : char* pszFirstNonSpace;
707 : char szBuffer[32];
708 10 : szBuffer[31] = 0;
709 :
710 10 : CPLPrintUIntBig(szBuffer, compressed_size, 31);
711 10 : pszFirstNonSpace = szBuffer;
712 10 : while (*pszFirstNonSpace == ' ') pszFirstNonSpace ++;
713 10 : VSIFPrintfL(fpCacheLength, "compressed_size=%s\n", pszFirstNonSpace);
714 :
715 10 : CPLPrintUIntBig(szBuffer, uncompressed_size, 31);
716 10 : pszFirstNonSpace = szBuffer;
717 10 : while (*pszFirstNonSpace == ' ') pszFirstNonSpace ++;
718 10 : VSIFPrintfL(fpCacheLength, "uncompressed_size=%s\n", pszFirstNonSpace);
719 :
720 10 : VSIFCloseL(fpCacheLength);
721 10 : }
722 : }
723 : }
724 :
725 878 : return (int) out;
726 : }
727 :
728 : /************************************************************************/
729 : /* Tell() */
730 : /************************************************************************/
731 :
732 110 : vsi_l_offset VSIGZipHandle::Tell()
733 : {
734 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Tell() = " CPL_FRMT_GUIB, out);
735 110 : return out;
736 : }
737 :
738 : /************************************************************************/
739 : /* Read() */
740 : /************************************************************************/
741 :
742 3902 : size_t VSIGZipHandle::Read( void *buf, size_t nSize, size_t nMemb )
743 : {
744 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Read(%p, %d, %d)", buf, (int)nSize, (int)nMemb);
745 :
746 3902 : unsigned len = nSize * nMemb;
747 :
748 3902 : Bytef *pStart = (Bytef*)buf; /* startOffing point for crc computation */
749 : Byte *next_out; /* == stream.next_out but not forced far (for MSDOS) */
750 :
751 3902 : if (z_err == Z_DATA_ERROR || z_err == Z_ERRNO)
752 : {
753 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
754 0 : return 0;
755 : }
756 3902 : if (z_err == Z_STREAM_END)
757 : {
758 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Read: Eof");
759 14 : return 0; /* EOF */
760 : }
761 :
762 3888 : next_out = (Byte*)buf;
763 3888 : stream.next_out = (Bytef*)buf;
764 3888 : stream.avail_out = len;
765 :
766 11282 : while (stream.avail_out != 0) {
767 :
768 3892 : if (transparent) {
769 : /* Copy first the lookahead bytes: */
770 4 : uInt nRead = 0;
771 4 : uInt n = stream.avail_in;
772 4 : if (n > stream.avail_out) n = stream.avail_out;
773 4 : if (n > 0) {
774 0 : memcpy (stream.next_out, stream.next_in, n);
775 0 : next_out += n;
776 0 : stream.next_out = next_out;
777 0 : stream.next_in += n;
778 0 : stream.avail_out -= n;
779 0 : stream.avail_in -= n;
780 0 : nRead += n;
781 : }
782 4 : if (stream.avail_out > 0) {
783 4 : uInt nToRead = (uInt) MIN(compressed_size - (in + nRead), stream.avail_out);
784 : uInt nReadFromFile =
785 4 : (uInt)VSIFReadL(next_out, 1, nToRead, (VSILFILE*)poBaseHandle);
786 4 : stream.avail_out -= nReadFromFile;
787 4 : nRead += nReadFromFile;
788 : }
789 4 : in += nRead;
790 4 : out += nRead;
791 4 : if (nRead < len) z_eof = 1;
792 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Read return %d", (int)(nRead / nSize));
793 4 : return (int)nRead / nSize;
794 : }
795 3888 : if (stream.avail_in == 0 && !z_eof)
796 : {
797 498 : vsi_l_offset uncompressed_pos = VSIFTellL((VSILFILE*)poBaseHandle);
798 498 : GZipSnapshot* snapshot = &snapshots[(uncompressed_pos - startOff) / snapshot_byte_interval];
799 498 : if (snapshot->uncompressed_pos == 0)
800 : {
801 420 : snapshot->crc = crc32 (crc, pStart, (uInt) (stream.next_out - pStart));
802 : if (ENABLE_DEBUG)
803 : CPLDebug("SNAPSHOT",
804 : "creating snapshot %d : uncompressed_pos=" CPL_FRMT_GUIB
805 : " in=" CPL_FRMT_GUIB
806 : " out=" CPL_FRMT_GUIB
807 : " crc=%X",
808 : (int)((uncompressed_pos - startOff) / snapshot_byte_interval),
809 : uncompressed_pos, in, out, (unsigned int)snapshot->crc);
810 420 : snapshot->uncompressed_pos = uncompressed_pos;
811 420 : inflateCopy(&snapshot->stream, &stream);
812 420 : snapshot->transparent = transparent;
813 420 : snapshot->in = in;
814 420 : snapshot->out = out;
815 :
816 420 : if (out > nLastReadOffset)
817 2 : nLastReadOffset = out;
818 : }
819 :
820 498 : errno = 0;
821 498 : stream.avail_in = (uInt)VSIFReadL(inbuf, 1, Z_BUFSIZE, (VSILFILE*)poBaseHandle);
822 : if (ENABLE_DEBUG)
823 : CPLDebug("GZIP", CPL_FRMT_GUIB " " CPL_FRMT_GUIB,
824 : VSIFTellL((VSILFILE*)poBaseHandle), offsetEndCompressedData);
825 498 : if (VSIFTellL((VSILFILE*)poBaseHandle) > offsetEndCompressedData)
826 : {
827 : if (ENABLE_DEBUG) CPLDebug("GZIP", "avail_in before = %d", stream.avail_in);
828 438 : stream.avail_in = stream.avail_in + (uInt) (offsetEndCompressedData - VSIFTellL((VSILFILE*)poBaseHandle));
829 438 : VSIFSeekL((VSILFILE*)poBaseHandle, offsetEndCompressedData, SEEK_SET);
830 438 : if (ENABLE_DEBUG) CPLDebug("GZIP", "avail_in after = %d", stream.avail_in);
831 : }
832 498 : if (stream.avail_in == 0) {
833 0 : z_eof = 1;
834 0 : if (VSIFTellL((VSILFILE*)poBaseHandle) != offsetEndCompressedData)
835 : {
836 0 : z_err = Z_ERRNO;
837 0 : break;
838 : }
839 : /*if (ferror (file)) {
840 : z_err = Z_ERRNO;
841 : break;
842 : }*/
843 : }
844 498 : stream.next_in = inbuf;
845 : }
846 3888 : in += stream.avail_in;
847 3888 : out += stream.avail_out;
848 3888 : z_err = inflate(& (stream), Z_NO_FLUSH);
849 3888 : in -= stream.avail_in;
850 3888 : out -= stream.avail_out;
851 :
852 3888 : if (z_err == Z_STREAM_END && compressed_size != 2 ) {
853 : /* Check CRC and original size */
854 382 : crc = crc32 (crc, pStart, (uInt) (stream.next_out - pStart));
855 382 : pStart = stream.next_out;
856 382 : if (expected_crc)
857 : {
858 344 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Computed CRC = %X. Expected CRC = %X", (unsigned int)crc, expected_crc);
859 : }
860 382 : if (expected_crc != 0 && expected_crc != crc)
861 : {
862 0 : CPLError(CE_Failure, CPLE_FileIO, "CRC error. Got %X instead of %X", (unsigned int)crc, expected_crc);
863 0 : z_err = Z_DATA_ERROR;
864 : }
865 382 : else if (expected_crc == 0)
866 : {
867 38 : unsigned int read_crc = getLong();
868 38 : if (read_crc != crc)
869 : {
870 0 : CPLError(CE_Failure, CPLE_FileIO, "CRC error. Got %X instead of %X", (unsigned int)crc, read_crc);
871 0 : z_err = Z_DATA_ERROR;
872 : }
873 : else
874 : {
875 38 : (void)getLong();
876 : /* The uncompressed length returned by above getlong() may be
877 : * different from out in case of concatenated .gz files.
878 : * Check for such files:
879 : */
880 38 : check_header();
881 38 : if (z_err == Z_OK) {
882 0 : inflateReset(& (stream));
883 0 : crc = crc32(0L, Z_NULL, 0);
884 : }
885 : }
886 : }
887 : }
888 3888 : if (z_err != Z_OK || z_eof) break;
889 : }
890 3884 : crc = crc32 (crc, pStart, (uInt) (stream.next_out - pStart));
891 :
892 3884 : if (len == stream.avail_out &&
893 : (z_err == Z_DATA_ERROR || z_err == Z_ERRNO))
894 : {
895 0 : CPL_VSIL_GZ_RETURN_MINUS_ONE();
896 0 : return 0;
897 : }
898 : if (ENABLE_DEBUG)
899 : CPLDebug("GZIP", "Read return %d (z_err=%d, z_eof=%d)",
900 : (int)((len - stream.avail_out) / nSize), z_err, z_eof);
901 3884 : return (int)(len - stream.avail_out) / nSize;
902 : }
903 :
904 : /************************************************************************/
905 : /* getLong() */
906 : /************************************************************************/
907 :
908 76 : uLong VSIGZipHandle::getLong ()
909 : {
910 76 : uLong x = (uLong)get_byte();
911 : int c;
912 :
913 76 : x += ((uLong)get_byte())<<8;
914 76 : x += ((uLong)get_byte())<<16;
915 76 : c = get_byte();
916 76 : if (c == EOF) z_err = Z_DATA_ERROR;
917 76 : x += ((uLong)c)<<24;
918 76 : return x;
919 : }
920 :
921 : /************************************************************************/
922 : /* Write() */
923 : /************************************************************************/
924 :
925 0 : size_t VSIGZipHandle::Write( const void *pBuffer, size_t nSize, size_t nMemb )
926 : {
927 0 : CPLError(CE_Failure, CPLE_NotSupported, "VSIFWriteL is not supported on GZip streams");
928 0 : return 0;
929 : }
930 :
931 : /************************************************************************/
932 : /* Eof() */
933 : /************************************************************************/
934 :
935 :
936 3486 : int VSIGZipHandle::Eof()
937 : {
938 : if (ENABLE_DEBUG) CPLDebug("GZIP", "Eof()");
939 3486 : if (z_eof) return 1;
940 3482 : return z_err == Z_STREAM_END;
941 : }
942 :
943 : /************************************************************************/
944 : /* Flush() */
945 : /************************************************************************/
946 :
947 0 : int VSIGZipHandle::Flush()
948 : {
949 0 : return 0;
950 : }
951 :
952 : /************************************************************************/
953 : /* Close() */
954 : /************************************************************************/
955 :
956 492 : int VSIGZipHandle::Close()
957 : {
958 492 : return 0;
959 : }
960 :
961 :
962 : /************************************************************************/
963 : /* ==================================================================== */
964 : /* VSIGZipWriteHandle */
965 : /* ==================================================================== */
966 : /************************************************************************/
967 :
968 : class VSIGZipWriteHandle : public VSIVirtualHandle
969 : {
970 : VSIVirtualHandle* poBaseHandle;
971 : z_stream sStream;
972 : Byte *pabyInBuf;
973 : Byte *pabyOutBuf;
974 : bool bCompressActive;
975 : vsi_l_offset nCurOffset;
976 : GUInt32 nCRC;
977 : int bRegularZLib;
978 : int bAutoCloseBaseHandle;
979 :
980 : public:
981 :
982 : VSIGZipWriteHandle(VSIVirtualHandle* poBaseHandle, int bRegularZLib, int bAutoCloseBaseHandleIn);
983 :
984 : ~VSIGZipWriteHandle();
985 :
986 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
987 : virtual vsi_l_offset Tell();
988 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
989 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
990 : virtual int Eof();
991 : virtual int Flush();
992 : virtual int Close();
993 : };
994 :
995 : /************************************************************************/
996 : /* VSIGZipWriteHandle() */
997 : /************************************************************************/
998 :
999 264 : VSIGZipWriteHandle::VSIGZipWriteHandle( VSIVirtualHandle *poBaseHandle,
1000 : int bRegularZLibIn,
1001 264 : int bAutoCloseBaseHandleIn )
1002 :
1003 : {
1004 264 : nCurOffset = 0;
1005 :
1006 264 : this->poBaseHandle = poBaseHandle;
1007 264 : bRegularZLib = bRegularZLibIn;
1008 264 : bAutoCloseBaseHandle = bAutoCloseBaseHandleIn;
1009 :
1010 264 : nCRC = crc32(0L, Z_NULL, 0);
1011 264 : sStream.zalloc = (alloc_func)0;
1012 264 : sStream.zfree = (free_func)0;
1013 264 : sStream.opaque = (voidpf)0;
1014 264 : sStream.next_in = Z_NULL;
1015 264 : sStream.next_out = Z_NULL;
1016 264 : sStream.avail_in = sStream.avail_out = 0;
1017 :
1018 264 : pabyInBuf = (Byte *) CPLMalloc( Z_BUFSIZE );
1019 264 : sStream.next_in = pabyInBuf;
1020 :
1021 264 : pabyOutBuf = (Byte *) CPLMalloc( Z_BUFSIZE );
1022 :
1023 264 : if( deflateInit2( &sStream, Z_DEFAULT_COMPRESSION,
1024 : Z_DEFLATED, (bRegularZLib) ? MAX_WBITS : -MAX_WBITS, 8,
1025 : Z_DEFAULT_STRATEGY ) != Z_OK )
1026 0 : bCompressActive = false;
1027 : else
1028 : {
1029 264 : if (!bRegularZLib)
1030 : {
1031 : char header[11];
1032 :
1033 : /* Write a very simple .gz header:
1034 : */
1035 48 : sprintf( header, "%c%c%c%c%c%c%c%c%c%c", gz_magic[0], gz_magic[1],
1036 : Z_DEFLATED, 0 /*flags*/, 0,0,0,0 /*time*/, 0 /*xflags*/,
1037 48 : 0x03 );
1038 48 : poBaseHandle->Write( header, 1, 10 );
1039 : }
1040 :
1041 264 : bCompressActive = true;
1042 : }
1043 264 : }
1044 :
1045 : /************************************************************************/
1046 : /* VSICreateGZipWritable() */
1047 : /************************************************************************/
1048 :
1049 216 : VSIVirtualHandle* VSICreateGZipWritable( VSIVirtualHandle* poBaseHandle,
1050 : int bRegularZLibIn,
1051 : int bAutoCloseBaseHandle )
1052 : {
1053 216 : return new VSIGZipWriteHandle( poBaseHandle, bRegularZLibIn, bAutoCloseBaseHandle );
1054 : }
1055 :
1056 : /************************************************************************/
1057 : /* ~VSIGZipWriteHandle() */
1058 : /************************************************************************/
1059 :
1060 264 : VSIGZipWriteHandle::~VSIGZipWriteHandle()
1061 :
1062 : {
1063 264 : if( bCompressActive )
1064 0 : Close();
1065 :
1066 264 : CPLFree( pabyInBuf );
1067 264 : CPLFree( pabyOutBuf );
1068 264 : }
1069 :
1070 : /************************************************************************/
1071 : /* Close() */
1072 : /************************************************************************/
1073 :
1074 264 : int VSIGZipWriteHandle::Close()
1075 :
1076 : {
1077 264 : if( bCompressActive )
1078 : {
1079 264 : sStream.next_out = pabyOutBuf;
1080 264 : sStream.avail_out = Z_BUFSIZE;
1081 :
1082 264 : deflate( &sStream, Z_FINISH );
1083 :
1084 264 : size_t nOutBytes = Z_BUFSIZE - sStream.avail_out;
1085 :
1086 264 : if( poBaseHandle->Write( pabyOutBuf, 1, nOutBytes ) < nOutBytes )
1087 0 : return EOF;
1088 :
1089 264 : deflateEnd( &sStream );
1090 :
1091 264 : if( !bRegularZLib )
1092 : {
1093 : GUInt32 anTrailer[2];
1094 :
1095 48 : anTrailer[0] = CPL_LSBWORD32( nCRC );
1096 48 : anTrailer[1] = CPL_LSBWORD32( (GUInt32) nCurOffset );
1097 :
1098 48 : poBaseHandle->Write( anTrailer, 1, 8 );
1099 : }
1100 :
1101 264 : if( bAutoCloseBaseHandle )
1102 : {
1103 48 : poBaseHandle->Close();
1104 :
1105 48 : delete poBaseHandle;
1106 : }
1107 :
1108 264 : bCompressActive = false;
1109 : }
1110 :
1111 264 : return 0;
1112 : }
1113 :
1114 : /************************************************************************/
1115 : /* Read() */
1116 : /************************************************************************/
1117 :
1118 0 : size_t VSIGZipWriteHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
1119 :
1120 : {
1121 0 : CPLError(CE_Failure, CPLE_NotSupported, "VSIFReadL is not supported on GZip write streams\n");
1122 0 : return 0;
1123 : }
1124 :
1125 : /************************************************************************/
1126 : /* Write() */
1127 : /************************************************************************/
1128 :
1129 15842 : size_t VSIGZipWriteHandle::Write( const void *pBuffer,
1130 : size_t nSize, size_t nMemb )
1131 :
1132 : {
1133 : int nBytesToWrite, nNextByte;
1134 :
1135 15842 : nBytesToWrite = (int) (nSize * nMemb);
1136 15842 : nNextByte = 0;
1137 :
1138 15842 : nCRC = crc32(nCRC, (const Bytef *)pBuffer, nBytesToWrite);
1139 :
1140 15842 : if( !bCompressActive )
1141 0 : return 0;
1142 :
1143 47526 : while( nNextByte < nBytesToWrite )
1144 : {
1145 15842 : sStream.next_out = pabyOutBuf;
1146 15842 : sStream.avail_out = Z_BUFSIZE;
1147 :
1148 15842 : if( sStream.avail_in > 0 )
1149 0 : memmove( pabyInBuf, sStream.next_in, sStream.avail_in );
1150 :
1151 15842 : int nNewBytesToWrite = MIN((int) (Z_BUFSIZE-sStream.avail_in),
1152 : nBytesToWrite - nNextByte);
1153 : memcpy( pabyInBuf + sStream.avail_in,
1154 : ((Byte *) pBuffer) + nNextByte,
1155 15842 : nNewBytesToWrite );
1156 :
1157 15842 : sStream.next_in = pabyInBuf;
1158 15842 : sStream.avail_in += nNewBytesToWrite;
1159 :
1160 15842 : deflate( &sStream, Z_NO_FLUSH );
1161 :
1162 15842 : size_t nOutBytes = Z_BUFSIZE - sStream.avail_out;
1163 :
1164 15842 : if( nOutBytes > 0 )
1165 : {
1166 250 : if( poBaseHandle->Write( pabyOutBuf, 1, nOutBytes ) < nOutBytes )
1167 0 : return 0;
1168 : }
1169 :
1170 15842 : nNextByte += nNewBytesToWrite;
1171 15842 : nCurOffset += nNewBytesToWrite;
1172 : }
1173 :
1174 15842 : return nMemb;
1175 : }
1176 :
1177 : /************************************************************************/
1178 : /* Flush() */
1179 : /************************************************************************/
1180 :
1181 0 : int VSIGZipWriteHandle::Flush()
1182 :
1183 : {
1184 : // we *could* do something for this but for now we choose not to.
1185 :
1186 0 : return 0;
1187 : }
1188 :
1189 : /************************************************************************/
1190 : /* Eof() */
1191 : /************************************************************************/
1192 :
1193 0 : int VSIGZipWriteHandle::Eof()
1194 :
1195 : {
1196 0 : return 1;
1197 : }
1198 :
1199 : /************************************************************************/
1200 : /* Seek() */
1201 : /************************************************************************/
1202 :
1203 0 : int VSIGZipWriteHandle::Seek( vsi_l_offset nOffset, int nWhence )
1204 :
1205 : {
1206 0 : if( nOffset == 0 && (nWhence == SEEK_END || nWhence == SEEK_CUR) )
1207 0 : return 0;
1208 0 : else if( nWhence == SEEK_SET && nOffset == nCurOffset )
1209 0 : return 0;
1210 : else
1211 : {
1212 : CPLError(CE_Failure, CPLE_NotSupported,
1213 0 : "Seeking on writable compressed data streams not supported." );
1214 :
1215 0 : return -1;
1216 : }
1217 : }
1218 :
1219 : /************************************************************************/
1220 : /* Tell() */
1221 : /************************************************************************/
1222 :
1223 0 : vsi_l_offset VSIGZipWriteHandle::Tell()
1224 :
1225 : {
1226 0 : return nCurOffset;
1227 : }
1228 :
1229 :
1230 : /************************************************************************/
1231 : /* ==================================================================== */
1232 : /* VSIGZipFilesystemHandler */
1233 : /* ==================================================================== */
1234 : /************************************************************************/
1235 :
1236 :
1237 : /************************************************************************/
1238 : /* VSIGZipFilesystemHandler() */
1239 : /************************************************************************/
1240 :
1241 1341 : VSIGZipFilesystemHandler::VSIGZipFilesystemHandler()
1242 : {
1243 1341 : hMutex = NULL;
1244 :
1245 1341 : poHandleLastGZipFile = NULL;
1246 1341 : bInSaveInfo = FALSE;
1247 1341 : }
1248 :
1249 : /************************************************************************/
1250 : /* ~VSIGZipFilesystemHandler() */
1251 : /************************************************************************/
1252 :
1253 1297 : VSIGZipFilesystemHandler::~VSIGZipFilesystemHandler()
1254 : {
1255 1297 : if (poHandleLastGZipFile)
1256 2 : delete poHandleLastGZipFile;
1257 :
1258 1297 : if( hMutex != NULL )
1259 2 : CPLDestroyMutex( hMutex );
1260 1297 : hMutex = NULL;
1261 1297 : }
1262 :
1263 : /************************************************************************/
1264 : /* SaveInfo() */
1265 : /************************************************************************/
1266 :
1267 116 : void VSIGZipFilesystemHandler::SaveInfo( VSIGZipHandle* poHandle )
1268 : {
1269 116 : CPLMutexHolder oHolder(&hMutex);
1270 :
1271 116 : if (bInSaveInfo)
1272 : return;
1273 116 : bInSaveInfo = TRUE;
1274 :
1275 116 : CPLAssert(poHandle->GetBaseFileName() != NULL);
1276 :
1277 116 : if (poHandleLastGZipFile &&
1278 : strcmp(poHandleLastGZipFile->GetBaseFileName(), poHandle->GetBaseFileName()) == 0)
1279 : {
1280 94 : if (poHandle->GetLastReadOffset() > poHandleLastGZipFile->GetLastReadOffset())
1281 : {
1282 0 : VSIGZipHandle* poTmp = poHandleLastGZipFile;
1283 0 : poHandleLastGZipFile = NULL;
1284 0 : delete poTmp;
1285 0 : poHandleLastGZipFile = poHandle->Duplicate();
1286 0 : poHandleLastGZipFile->CloseBaseHandle();
1287 : }
1288 : }
1289 : else
1290 : {
1291 22 : VSIGZipHandle* poTmp = poHandleLastGZipFile;
1292 22 : poHandleLastGZipFile = NULL;
1293 22 : delete poTmp;
1294 22 : poHandleLastGZipFile = poHandle->Duplicate();
1295 22 : poHandleLastGZipFile->CloseBaseHandle();
1296 : }
1297 :
1298 116 : bInSaveInfo = FALSE;
1299 : }
1300 :
1301 : /************************************************************************/
1302 : /* Open() */
1303 : /************************************************************************/
1304 :
1305 242 : VSIVirtualHandle* VSIGZipFilesystemHandler::Open( const char *pszFilename,
1306 : const char *pszAccess)
1307 : {
1308 : VSIFilesystemHandler *poFSHandler =
1309 242 : VSIFileManager::GetHandler( pszFilename + strlen("/vsigzip/"));
1310 :
1311 : /* -------------------------------------------------------------------- */
1312 : /* Is this an attempt to write a new file without update (w+) */
1313 : /* access? If so, create a writable handle for the underlying */
1314 : /* filename. */
1315 : /* -------------------------------------------------------------------- */
1316 242 : if (strchr(pszAccess, 'w') != NULL )
1317 : {
1318 52 : if( strchr(pszAccess, '+') != NULL )
1319 : {
1320 : CPLError(CE_Failure, CPLE_AppDefined,
1321 0 : "Write+update (w+) not supported for /vsigzip, only read-only or write-only.");
1322 0 : return NULL;
1323 : }
1324 :
1325 : VSIVirtualHandle* poVirtualHandle =
1326 52 : poFSHandler->Open( pszFilename + strlen("/vsigzip/"), "wb" );
1327 :
1328 52 : if (poVirtualHandle == NULL)
1329 4 : return NULL;
1330 :
1331 : else
1332 48 : return new VSIGZipWriteHandle( poVirtualHandle, strchr(pszAccess, 'z') != NULL, TRUE );
1333 : }
1334 :
1335 : /* -------------------------------------------------------------------- */
1336 : /* Otherwise we are in the read access case. */
1337 : /* -------------------------------------------------------------------- */
1338 :
1339 190 : VSIGZipHandle* poGZIPHandle = OpenGZipReadOnly(pszFilename, pszAccess);
1340 190 : if (poGZIPHandle)
1341 : /* Wrap the VSIGZipHandle inside a buffered reader that will */
1342 : /* improve dramatically performance when doing small backward */
1343 : /* seeks */
1344 94 : return VSICreateBufferedReaderHandle(poGZIPHandle);
1345 : else
1346 96 : return NULL;
1347 : }
1348 :
1349 : /************************************************************************/
1350 : /* OpenGZipReadOnly() */
1351 : /************************************************************************/
1352 :
1353 190 : VSIGZipHandle* VSIGZipFilesystemHandler::OpenGZipReadOnly( const char *pszFilename,
1354 : const char *pszAccess)
1355 : {
1356 : VSIFilesystemHandler *poFSHandler =
1357 190 : VSIFileManager::GetHandler( pszFilename + strlen("/vsigzip/"));
1358 :
1359 190 : CPLMutexHolder oHolder(&hMutex);
1360 :
1361 190 : if (poHandleLastGZipFile != NULL &&
1362 : strcmp(pszFilename + strlen("/vsigzip/"), poHandleLastGZipFile->GetBaseFileName()) == 0 &&
1363 : EQUAL(pszAccess, "rb"))
1364 : {
1365 72 : VSIGZipHandle* poHandle = poHandleLastGZipFile->Duplicate();
1366 72 : if (poHandle)
1367 72 : return poHandle;
1368 : }
1369 :
1370 : unsigned char signature[2];
1371 :
1372 : VSIVirtualHandle* poVirtualHandle =
1373 118 : poFSHandler->Open( pszFilename + strlen("/vsigzip/"), "rb" );
1374 :
1375 118 : if (poVirtualHandle == NULL)
1376 96 : return NULL;
1377 :
1378 22 : if (VSIFReadL(signature, 1, 2, (VSILFILE*)poVirtualHandle) != 2)
1379 0 : return NULL;
1380 :
1381 22 : if (signature[0] != gz_magic[0] || signature[1] != gz_magic[1])
1382 0 : return NULL;
1383 :
1384 22 : if (poHandleLastGZipFile)
1385 20 : delete poHandleLastGZipFile;
1386 22 : poHandleLastGZipFile = NULL;
1387 :
1388 22 : return new VSIGZipHandle(poVirtualHandle, pszFilename + strlen("/vsigzip/"));
1389 : }
1390 :
1391 : /************************************************************************/
1392 : /* Stat() */
1393 : /************************************************************************/
1394 :
1395 88 : int VSIGZipFilesystemHandler::Stat( const char *pszFilename,
1396 : VSIStatBufL *pStatBuf,
1397 : int nFlags )
1398 : {
1399 88 : CPLMutexHolder oHolder(&hMutex);
1400 :
1401 88 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
1402 :
1403 88 : if (poHandleLastGZipFile != NULL &&
1404 : strcmp(pszFilename+strlen("/vsigzip/"), poHandleLastGZipFile->GetBaseFileName()) == 0)
1405 : {
1406 10 : if (poHandleLastGZipFile->GetUncompressedSize() != 0)
1407 : {
1408 0 : pStatBuf->st_mode = S_IFREG;
1409 0 : pStatBuf->st_size = poHandleLastGZipFile->GetUncompressedSize();
1410 0 : return 0;
1411 : }
1412 : }
1413 :
1414 : /* Begin by doing a stat on the real file */
1415 88 : int ret = VSIStatExL(pszFilename+strlen("/vsigzip/"), pStatBuf, nFlags);
1416 :
1417 88 : if (ret == 0 && (nFlags & VSI_STAT_SIZE_FLAG))
1418 : {
1419 0 : CPLString osCacheFilename(pszFilename+strlen("/vsigzip/"));
1420 0 : osCacheFilename += ".properties";
1421 :
1422 : /* Can we save a bit of seeking by using a .properties file ? */
1423 0 : VSILFILE* fpCacheLength = VSIFOpenL(osCacheFilename.c_str(), "rb");
1424 0 : if (fpCacheLength)
1425 : {
1426 : const char* pszLine;
1427 0 : GUIntBig nCompressedSize = 0;
1428 0 : GUIntBig nUncompressedSize = 0;
1429 0 : while ((pszLine = CPLReadLineL(fpCacheLength)) != NULL)
1430 : {
1431 0 : if (EQUALN(pszLine, "compressed_size=", strlen("compressed_size=")))
1432 : {
1433 0 : const char* pszBuffer = pszLine + strlen("compressed_size=");
1434 : nCompressedSize =
1435 0 : CPLScanUIntBig(pszBuffer, strlen(pszBuffer));
1436 : }
1437 0 : else if (EQUALN(pszLine, "uncompressed_size=", strlen("uncompressed_size=")))
1438 : {
1439 0 : const char* pszBuffer = pszLine + strlen("uncompressed_size=");
1440 : nUncompressedSize =
1441 0 : CPLScanUIntBig(pszBuffer, strlen(pszBuffer));
1442 : }
1443 : }
1444 :
1445 0 : VSIFCloseL(fpCacheLength);
1446 :
1447 0 : if (nCompressedSize == (GUIntBig) pStatBuf->st_size)
1448 : {
1449 : /* Patch with the uncompressed size */
1450 0 : pStatBuf->st_size = (long)nUncompressedSize;
1451 :
1452 : VSIGZipHandle* poHandle =
1453 0 : VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename, "rb");
1454 0 : if (poHandle)
1455 : {
1456 0 : poHandle->SetUncompressedSize(nUncompressedSize);
1457 0 : SaveInfo(poHandle);
1458 0 : delete poHandle;
1459 : }
1460 :
1461 0 : return ret;
1462 : }
1463 : }
1464 :
1465 : /* No, then seek at the end of the data (slow) */
1466 : VSIGZipHandle* poHandle =
1467 0 : VSIGZipFilesystemHandler::OpenGZipReadOnly(pszFilename, "rb");
1468 0 : if (poHandle)
1469 : {
1470 : GUIntBig uncompressed_size;
1471 0 : poHandle->Seek(0, SEEK_END);
1472 0 : uncompressed_size = (GUIntBig) poHandle->Tell();
1473 0 : poHandle->Seek(0, SEEK_SET);
1474 :
1475 : /* Patch with the uncompressed size */
1476 0 : pStatBuf->st_size = (long)uncompressed_size;
1477 :
1478 0 : delete poHandle;
1479 : }
1480 : else
1481 : {
1482 0 : ret = -1;
1483 0 : }
1484 : }
1485 :
1486 88 : return ret;
1487 : }
1488 :
1489 : /************************************************************************/
1490 : /* Unlink() */
1491 : /************************************************************************/
1492 :
1493 0 : int VSIGZipFilesystemHandler::Unlink( const char *pszFilename )
1494 : {
1495 0 : return -1;
1496 : }
1497 :
1498 : /************************************************************************/
1499 : /* Rename() */
1500 : /************************************************************************/
1501 :
1502 0 : int VSIGZipFilesystemHandler::Rename( const char *oldpath, const char *newpath )
1503 : {
1504 0 : return -1;
1505 : }
1506 :
1507 : /************************************************************************/
1508 : /* Mkdir() */
1509 : /************************************************************************/
1510 :
1511 0 : int VSIGZipFilesystemHandler::Mkdir( const char *pszDirname, long nMode )
1512 : {
1513 0 : return -1;
1514 : }
1515 : /************************************************************************/
1516 : /* Rmdir() */
1517 : /************************************************************************/
1518 :
1519 0 : int VSIGZipFilesystemHandler::Rmdir( const char *pszDirname )
1520 : {
1521 0 : return -1;
1522 : }
1523 :
1524 : /************************************************************************/
1525 : /* ReadDir() */
1526 : /************************************************************************/
1527 :
1528 10 : char** VSIGZipFilesystemHandler::ReadDir( const char *pszDirname )
1529 : {
1530 10 : return NULL;
1531 : }
1532 :
1533 : /************************************************************************/
1534 : /* VSIInstallGZipFileHandler() */
1535 : /************************************************************************/
1536 :
1537 :
1538 : /**
1539 : * \brief Install GZip file system handler.
1540 : *
1541 : * A special file handler is installed that allows reading on-the-fly and
1542 : * writing in GZip (.gz) files.
1543 : *
1544 : * All portions of the file system underneath the base
1545 : * path "/vsigzip/" will be handled by this driver.
1546 : *
1547 : * Additional documentation is to be found at http://trac.osgeo.org/gdal/wiki/UserDocs/ReadInZip
1548 : *
1549 : * @since GDAL 1.6.0
1550 : */
1551 :
1552 1341 : void VSIInstallGZipFileHandler(void)
1553 : {
1554 1341 : VSIFileManager::InstallHandler( "/vsigzip/", new VSIGZipFilesystemHandler );
1555 1341 : }
1556 :
1557 :
1558 : /************************************************************************/
1559 : /* ==================================================================== */
1560 : /* VSIZipEntryFileOffset */
1561 : /* ==================================================================== */
1562 : /************************************************************************/
1563 :
1564 : class VSIZipEntryFileOffset : public VSIArchiveEntryFileOffset
1565 4758 : {
1566 : public:
1567 : unz_file_pos file_pos;
1568 :
1569 4758 : VSIZipEntryFileOffset(unz_file_pos file_pos)
1570 4758 : {
1571 4758 : this->file_pos.pos_in_zip_directory = file_pos.pos_in_zip_directory;
1572 4758 : this->file_pos.num_of_file = file_pos.num_of_file;
1573 4758 : }
1574 : };
1575 :
1576 : /************************************************************************/
1577 : /* ==================================================================== */
1578 : /* VSIZipReader */
1579 : /* ==================================================================== */
1580 : /************************************************************************/
1581 :
1582 : class VSIZipReader : public VSIArchiveReader
1583 : {
1584 : private:
1585 : unzFile unzF;
1586 : unz_file_pos file_pos;
1587 : GUIntBig nNextFileSize;
1588 : CPLString osNextFileName;
1589 : GIntBig nModifiedTime;
1590 :
1591 : void SetInfo();
1592 :
1593 : public:
1594 : VSIZipReader(const char* pszZipFileName);
1595 : virtual ~VSIZipReader();
1596 :
1597 526 : int IsValid() { return unzF != NULL; }
1598 :
1599 398 : unzFile GetUnzFileHandle() { return unzF; }
1600 :
1601 : virtual int GotoFirstFile();
1602 : virtual int GotoNextFile();
1603 4758 : virtual VSIArchiveEntryFileOffset* GetFileOffset() { return new VSIZipEntryFileOffset(file_pos); }
1604 4764 : virtual GUIntBig GetFileSize() { return nNextFileSize; }
1605 4788 : virtual CPLString GetFileName() { return osNextFileName; }
1606 5544 : virtual GIntBig GetModifiedTime() { return nModifiedTime; }
1607 : virtual int GotoFileOffset(VSIArchiveEntryFileOffset* pOffset);
1608 : };
1609 :
1610 :
1611 : /************************************************************************/
1612 : /* VSIZipReader() */
1613 : /************************************************************************/
1614 :
1615 526 : VSIZipReader::VSIZipReader(const char* pszZipFileName)
1616 : {
1617 526 : unzF = cpl_unzOpen(pszZipFileName);
1618 526 : nNextFileSize = 0;
1619 526 : nModifiedTime = 0;
1620 526 : }
1621 :
1622 : /************************************************************************/
1623 : /* ~VSIZipReader() */
1624 : /************************************************************************/
1625 :
1626 526 : VSIZipReader::~VSIZipReader()
1627 : {
1628 526 : if (unzF)
1629 514 : cpl_unzClose(unzF);
1630 526 : }
1631 :
1632 : /************************************************************************/
1633 : /* SetInfo() */
1634 : /************************************************************************/
1635 :
1636 5698 : void VSIZipReader::SetInfo()
1637 : {
1638 : char fileName[512];
1639 : unz_file_info file_info;
1640 5698 : cpl_unzGetCurrentFileInfo (unzF, &file_info, fileName, 512, NULL, 0, NULL, 0);
1641 5698 : osNextFileName = fileName;
1642 5698 : nNextFileSize = file_info.uncompressed_size;
1643 : struct tm brokendowntime;
1644 5698 : brokendowntime.tm_sec = file_info.tmu_date.tm_sec;
1645 5698 : brokendowntime.tm_min = file_info.tmu_date.tm_min;
1646 5698 : brokendowntime.tm_hour = file_info.tmu_date.tm_hour;
1647 5698 : brokendowntime.tm_mday = file_info.tmu_date.tm_mday;
1648 5698 : brokendowntime.tm_mon = file_info.tmu_date.tm_mon;
1649 5698 : brokendowntime.tm_year = file_info.tmu_date.tm_year - 1900; /* the minizip conventions differs from the Unix one */
1650 5698 : nModifiedTime = CPLYMDHMSToUnixTime(&brokendowntime);
1651 :
1652 5698 : cpl_unzGetFilePos(unzF, &this->file_pos);
1653 5698 : }
1654 :
1655 : /************************************************************************/
1656 : /* GotoNextFile() */
1657 : /************************************************************************/
1658 :
1659 4794 : int VSIZipReader::GotoNextFile()
1660 : {
1661 4794 : if (cpl_unzGoToNextFile(unzF) != UNZ_OK)
1662 88 : return FALSE;
1663 :
1664 4706 : SetInfo();
1665 :
1666 4706 : return TRUE;
1667 : }
1668 :
1669 : /************************************************************************/
1670 : /* GotoFirstFile() */
1671 : /************************************************************************/
1672 :
1673 610 : int VSIZipReader::GotoFirstFile()
1674 : {
1675 610 : if (cpl_unzGoToFirstFile(unzF) != UNZ_OK)
1676 0 : return FALSE;
1677 :
1678 610 : SetInfo();
1679 :
1680 610 : return TRUE;
1681 : }
1682 :
1683 : /************************************************************************/
1684 : /* GotoFileOffset() */
1685 : /************************************************************************/
1686 :
1687 382 : int VSIZipReader::GotoFileOffset(VSIArchiveEntryFileOffset* pOffset)
1688 : {
1689 382 : VSIZipEntryFileOffset* pZipEntryOffset = (VSIZipEntryFileOffset*)pOffset;
1690 382 : cpl_unzGoToFilePos(unzF, &(pZipEntryOffset->file_pos));
1691 :
1692 382 : SetInfo();
1693 :
1694 382 : return TRUE;
1695 : }
1696 :
1697 : /************************************************************************/
1698 : /* ==================================================================== */
1699 : /* VSIZipFilesystemHandler */
1700 : /* ==================================================================== */
1701 : /************************************************************************/
1702 :
1703 : class VSIZipWriteHandle;
1704 :
1705 : class VSIZipFilesystemHandler : public VSIArchiveFilesystemHandler
1706 1341 : {
1707 : std::map<CPLString, VSIZipWriteHandle*> oMapZipWriteHandles;
1708 :
1709 : public:
1710 : virtual ~VSIZipFilesystemHandler();
1711 :
1712 9120 : virtual const char* GetPrefix() { return "/vsizip"; }
1713 : virtual std::vector<CPLString> GetExtensions();
1714 : virtual VSIArchiveReader* CreateReader(const char* pszZipFileName);
1715 :
1716 : virtual VSIVirtualHandle *Open( const char *pszFilename,
1717 : const char *pszAccess);
1718 :
1719 : virtual VSIVirtualHandle *OpenForWrite( const char *pszFilename,
1720 : const char *pszAccess );
1721 :
1722 : virtual int Mkdir( const char *pszDirname, long nMode );
1723 : virtual char **ReadDir( const char *pszDirname );
1724 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
1725 :
1726 : void RemoveFromMap(VSIZipWriteHandle* poHandle);
1727 : };
1728 :
1729 : /************************************************************************/
1730 : /* ==================================================================== */
1731 : /* VSIZipWriteHandle */
1732 : /* ==================================================================== */
1733 : /************************************************************************/
1734 :
1735 : class VSIZipWriteHandle : public VSIVirtualHandle
1736 : {
1737 : VSIZipFilesystemHandler *poFS;
1738 : void *hZIP;
1739 : VSIZipWriteHandle *poChildInWriting;
1740 : VSIZipWriteHandle *poParent;
1741 : int bAutoDeleteParent;
1742 :
1743 : public:
1744 :
1745 : VSIZipWriteHandle(VSIZipFilesystemHandler* poFS,
1746 : void *hZIP,
1747 : VSIZipWriteHandle* poParent);
1748 :
1749 : ~VSIZipWriteHandle();
1750 :
1751 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
1752 : virtual vsi_l_offset Tell();
1753 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
1754 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
1755 : virtual int Eof();
1756 : virtual int Flush();
1757 : virtual int Close();
1758 :
1759 : void StartNewFile(VSIZipWriteHandle* poSubFile);
1760 : void StopCurrentFile();
1761 102 : void* GetHandle() { return hZIP; }
1762 104 : VSIZipWriteHandle* GetChildInWriting() { return poChildInWriting; };
1763 26 : void SetAutoDeleteParent() { bAutoDeleteParent = TRUE; }
1764 : };
1765 :
1766 : /************************************************************************/
1767 : /* ~VSIZipFilesystemHandler() */
1768 : /************************************************************************/
1769 :
1770 1297 : VSIZipFilesystemHandler::~VSIZipFilesystemHandler()
1771 : {
1772 1297 : std::map<CPLString,VSIZipWriteHandle*>::const_iterator iter;
1773 :
1774 1297 : for( iter = oMapZipWriteHandles.begin(); iter != oMapZipWriteHandles.end(); ++iter )
1775 : {
1776 : CPLError(CE_Failure, CPLE_AppDefined, "%s has not been closed",
1777 0 : iter->first.c_str());
1778 : }
1779 1297 : }
1780 :
1781 : /************************************************************************/
1782 : /* GetExtensions() */
1783 : /************************************************************************/
1784 :
1785 45198 : std::vector<CPLString> VSIZipFilesystemHandler::GetExtensions()
1786 : {
1787 45198 : std::vector<CPLString> oList;
1788 45198 : oList.push_back(".zip");
1789 45198 : oList.push_back(".kmz");
1790 45198 : oList.push_back(".dwf");
1791 45198 : oList.push_back(".ods");
1792 45198 : oList.push_back(".xlsx");
1793 0 : return oList;
1794 : }
1795 :
1796 : /************************************************************************/
1797 : /* CreateReader() */
1798 : /************************************************************************/
1799 :
1800 526 : VSIArchiveReader* VSIZipFilesystemHandler::CreateReader(const char* pszZipFileName)
1801 : {
1802 526 : VSIZipReader* poReader = new VSIZipReader(pszZipFileName);
1803 :
1804 526 : if (!poReader->IsValid())
1805 : {
1806 12 : delete poReader;
1807 12 : return NULL;
1808 : }
1809 :
1810 514 : if (!poReader->GotoFirstFile())
1811 : {
1812 0 : delete poReader;
1813 0 : return NULL;
1814 : }
1815 :
1816 514 : return poReader;
1817 : }
1818 :
1819 : /************************************************************************/
1820 : /* Open() */
1821 : /************************************************************************/
1822 :
1823 606 : VSIVirtualHandle* VSIZipFilesystemHandler::Open( const char *pszFilename,
1824 : const char *pszAccess)
1825 : {
1826 : char* zipFilename;
1827 606 : CPLString osZipInFileName;
1828 :
1829 606 : if (strchr(pszAccess, 'w') != NULL)
1830 : {
1831 110 : return OpenForWrite(pszFilename, pszAccess);
1832 : }
1833 :
1834 496 : if (strchr(pszAccess, '+') != NULL)
1835 : {
1836 : CPLError(CE_Failure, CPLE_AppDefined,
1837 0 : "Random access not supported for /vsizip");
1838 0 : return NULL;
1839 : }
1840 :
1841 496 : zipFilename = SplitFilename(pszFilename, osZipInFileName, TRUE);
1842 496 : if (zipFilename == NULL)
1843 48 : return NULL;
1844 :
1845 : {
1846 448 : CPLMutexHolder oHolder(&hMutex);
1847 448 : if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end() )
1848 : {
1849 : CPLError(CE_Failure, CPLE_AppDefined,
1850 2 : "Cannot read a zip file being written");
1851 2 : CPLFree(zipFilename);
1852 2 : return NULL;
1853 0 : }
1854 : }
1855 :
1856 446 : VSIArchiveReader* poReader = OpenArchiveFile(zipFilename, osZipInFileName);
1857 446 : if (poReader == NULL)
1858 : {
1859 48 : CPLFree(zipFilename);
1860 48 : return NULL;
1861 : }
1862 :
1863 : VSIFilesystemHandler *poFSHandler =
1864 398 : VSIFileManager::GetHandler( zipFilename);
1865 :
1866 : VSIVirtualHandle* poVirtualHandle =
1867 398 : poFSHandler->Open( zipFilename, "rb" );
1868 :
1869 398 : CPLFree(zipFilename);
1870 398 : zipFilename = NULL;
1871 :
1872 398 : if (poVirtualHandle == NULL)
1873 : {
1874 0 : delete poReader;
1875 0 : return NULL;
1876 : }
1877 :
1878 398 : unzFile unzF = ((VSIZipReader*)poReader)->GetUnzFileHandle();
1879 :
1880 398 : cpl_unzOpenCurrentFile(unzF);
1881 :
1882 398 : uLong64 pos = cpl_unzGetCurrentFileZStreamPos(unzF);
1883 :
1884 : unz_file_info file_info;
1885 398 : cpl_unzGetCurrentFileInfo (unzF, &file_info, NULL, 0, NULL, 0, NULL, 0);
1886 :
1887 398 : cpl_unzCloseCurrentFile(unzF);
1888 :
1889 398 : delete poReader;
1890 :
1891 : VSIGZipHandle* poGZIPHandle = new VSIGZipHandle(poVirtualHandle,
1892 : NULL,
1893 : pos,
1894 : file_info.compressed_size,
1895 : file_info.uncompressed_size,
1896 : file_info.crc,
1897 398 : file_info.compression_method == 0);
1898 : /* Wrap the VSIGZipHandle inside a buffered reader that will */
1899 : /* improve dramatically performance when doing small backward */
1900 : /* seeks */
1901 796 : return VSICreateBufferedReaderHandle(poGZIPHandle);
1902 : }
1903 :
1904 : /************************************************************************/
1905 : /* Mkdir() */
1906 : /************************************************************************/
1907 :
1908 2 : int VSIZipFilesystemHandler::Mkdir( const char *pszDirname, long nMode )
1909 : {
1910 2 : CPLString osDirname = pszDirname;
1911 2 : if (osDirname.size() != 0 && osDirname[osDirname.size() - 1] != '/')
1912 2 : osDirname += "/";
1913 2 : VSIVirtualHandle* poZIPHandle = OpenForWrite(osDirname, "wb");
1914 2 : if (poZIPHandle == NULL)
1915 0 : return -1;
1916 2 : delete poZIPHandle;
1917 2 : return 0;
1918 : }
1919 :
1920 : /************************************************************************/
1921 : /* ReadDir() */
1922 : /************************************************************************/
1923 :
1924 1098 : char **VSIZipFilesystemHandler::ReadDir( const char *pszDirname )
1925 : {
1926 1098 : CPLString osInArchiveSubDir;
1927 1098 : char* zipFilename = SplitFilename(pszDirname, osInArchiveSubDir, TRUE);
1928 1098 : if (zipFilename == NULL)
1929 6 : return NULL;
1930 :
1931 : {
1932 1092 : CPLMutexHolder oHolder(&hMutex);
1933 :
1934 1092 : if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end() )
1935 : {
1936 : CPLError(CE_Failure, CPLE_AppDefined,
1937 2 : "Cannot read a zip file being written");
1938 2 : CPLFree(zipFilename);
1939 2 : return NULL;
1940 0 : }
1941 : }
1942 1090 : CPLFree(zipFilename);
1943 :
1944 1090 : return VSIArchiveFilesystemHandler::ReadDir(pszDirname);
1945 : }
1946 :
1947 :
1948 : /************************************************************************/
1949 : /* Stat() */
1950 : /************************************************************************/
1951 :
1952 116 : int VSIZipFilesystemHandler::Stat( const char *pszFilename,
1953 : VSIStatBufL *pStatBuf, int nFlags )
1954 : {
1955 116 : CPLString osInArchiveSubDir;
1956 :
1957 116 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
1958 :
1959 116 : char* zipFilename = SplitFilename(pszFilename, osInArchiveSubDir, TRUE);
1960 116 : if (zipFilename == NULL)
1961 14 : return -1;
1962 :
1963 : {
1964 102 : CPLMutexHolder oHolder(&hMutex);
1965 :
1966 102 : if (oMapZipWriteHandles.find(zipFilename) != oMapZipWriteHandles.end() )
1967 : {
1968 : CPLError(CE_Failure, CPLE_AppDefined,
1969 0 : "Cannot read a zip file being written");
1970 0 : CPLFree(zipFilename);
1971 0 : return -1;
1972 0 : }
1973 : }
1974 102 : CPLFree(zipFilename);
1975 :
1976 102 : return VSIArchiveFilesystemHandler::Stat(pszFilename, pStatBuf, nFlags);
1977 : }
1978 :
1979 : /************************************************************************/
1980 : /* RemoveFromMap() */
1981 : /************************************************************************/
1982 :
1983 34 : void VSIZipFilesystemHandler::RemoveFromMap(VSIZipWriteHandle* poHandle)
1984 : {
1985 34 : CPLMutexHolder oHolder( &hMutex );
1986 34 : std::map<CPLString,VSIZipWriteHandle*>::iterator iter;
1987 :
1988 34 : for( iter = oMapZipWriteHandles.begin();
1989 : iter != oMapZipWriteHandles.end(); ++iter )
1990 : {
1991 34 : if (iter->second == poHandle)
1992 : {
1993 34 : oMapZipWriteHandles.erase(iter);
1994 34 : break;
1995 : }
1996 34 : }
1997 34 : }
1998 :
1999 : /************************************************************************/
2000 : /* OpenForWrite() */
2001 : /************************************************************************/
2002 :
2003 138 : VSIVirtualHandle* VSIZipFilesystemHandler::OpenForWrite( const char *pszFilename,
2004 : const char *pszAccess)
2005 : {
2006 : char* zipFilename;
2007 138 : CPLString osZipInFileName;
2008 :
2009 138 : CPLMutexHolder oHolder( &hMutex );
2010 :
2011 138 : zipFilename = SplitFilename(pszFilename, osZipInFileName, FALSE);
2012 138 : if (zipFilename == NULL)
2013 0 : return NULL;
2014 138 : CPLString osZipFilename = zipFilename;
2015 138 : CPLFree(zipFilename);
2016 138 : zipFilename = NULL;
2017 :
2018 : /* Invalidate cached file list */
2019 138 : std::map<CPLString,VSIArchiveContent*>::iterator iter = oFileList.find(osZipFilename);
2020 138 : if (iter != oFileList.end())
2021 : {
2022 8 : VSIArchiveContent* content = iter->second;
2023 : int i;
2024 106 : for(i=0;i<content->nEntries;i++)
2025 : {
2026 98 : delete content->entries[i].file_pos;
2027 98 : CPLFree(content->entries[i].fileName);
2028 : }
2029 8 : CPLFree(content->entries);
2030 8 : delete content;
2031 :
2032 8 : oFileList.erase(iter);
2033 : }
2034 :
2035 : VSIZipWriteHandle* poZIPHandle;
2036 :
2037 138 : if (oMapZipWriteHandles.find(osZipFilename) != oMapZipWriteHandles.end() )
2038 : {
2039 104 : if (strchr(pszAccess, '+') != NULL)
2040 : {
2041 : CPLError(CE_Failure, CPLE_AppDefined,
2042 0 : "Random access not supported for writable file in /vsizip");
2043 0 : return NULL;
2044 : }
2045 :
2046 104 : poZIPHandle = oMapZipWriteHandles[osZipFilename];
2047 :
2048 104 : if (poZIPHandle->GetChildInWriting() != NULL)
2049 : {
2050 : CPLError(CE_Failure, CPLE_AppDefined,
2051 : "Cannot create %s while another file is being written in the .zip",
2052 2 : osZipInFileName.c_str());
2053 2 : return NULL;
2054 : }
2055 :
2056 102 : poZIPHandle->StopCurrentFile();
2057 :
2058 : /* Re-add path separator when creating directories */
2059 102 : char chLastChar = pszFilename[strlen(pszFilename) - 1];
2060 102 : if (chLastChar == '/' || chLastChar == '\\')
2061 4 : osZipInFileName += chLastChar;
2062 :
2063 102 : if (CPLCreateFileInZip(poZIPHandle->GetHandle(),
2064 : osZipInFileName, NULL) != CE_None)
2065 0 : return NULL;
2066 :
2067 : VSIZipWriteHandle* poChildHandle =
2068 102 : new VSIZipWriteHandle(this, NULL, poZIPHandle);
2069 :
2070 102 : poZIPHandle->StartNewFile(poChildHandle);
2071 :
2072 102 : return poChildHandle;
2073 : }
2074 : else
2075 : {
2076 34 : char** papszOptions = NULL;
2077 34 : if ((strchr(pszAccess, '+') && osZipInFileName.size() == 0) ||
2078 : osZipInFileName.size() != 0)
2079 : {
2080 : VSIStatBufL sBuf;
2081 26 : if (VSIStatExL(osZipFilename, &sBuf, VSI_STAT_EXISTS_FLAG) == 0)
2082 24 : papszOptions = CSLAddNameValue(papszOptions, "APPEND", "TRUE");
2083 : }
2084 :
2085 34 : void* hZIP = CPLCreateZip(osZipFilename, papszOptions);
2086 34 : CSLDestroy(papszOptions);
2087 :
2088 34 : if (hZIP == NULL)
2089 0 : return NULL;
2090 :
2091 : oMapZipWriteHandles[osZipFilename] =
2092 34 : new VSIZipWriteHandle(this, hZIP, NULL);
2093 :
2094 34 : if (osZipInFileName.size() != 0)
2095 : {
2096 : VSIZipWriteHandle* poRes =
2097 26 : (VSIZipWriteHandle*)OpenForWrite(pszFilename, pszAccess);
2098 26 : if (poRes == NULL)
2099 : {
2100 0 : delete oMapZipWriteHandles[osZipFilename];
2101 0 : return NULL;
2102 : }
2103 :
2104 26 : poRes->SetAutoDeleteParent();
2105 :
2106 26 : return poRes;
2107 : }
2108 :
2109 8 : return oMapZipWriteHandles[osZipFilename];
2110 0 : }
2111 : }
2112 :
2113 :
2114 : /************************************************************************/
2115 : /* VSIZipWriteHandle() */
2116 : /************************************************************************/
2117 :
2118 136 : VSIZipWriteHandle::VSIZipWriteHandle(VSIZipFilesystemHandler* poFS,
2119 : void* hZIP,
2120 136 : VSIZipWriteHandle* poParent)
2121 : {
2122 136 : this->poFS = poFS;
2123 136 : this->hZIP = hZIP;
2124 136 : this->poParent = poParent;
2125 136 : poChildInWriting = NULL;
2126 136 : bAutoDeleteParent = FALSE;
2127 136 : }
2128 :
2129 : /************************************************************************/
2130 : /* ~VSIZipWriteHandle() */
2131 : /************************************************************************/
2132 :
2133 136 : VSIZipWriteHandle::~VSIZipWriteHandle()
2134 : {
2135 136 : Close();
2136 136 : }
2137 :
2138 : /************************************************************************/
2139 : /* Seek() */
2140 : /************************************************************************/
2141 :
2142 0 : int VSIZipWriteHandle::Seek( vsi_l_offset nOffset, int nWhence )
2143 : {
2144 : CPLError(CE_Failure, CPLE_NotSupported,
2145 0 : "VSIFSeekL() is not supported on writable Zip files");
2146 0 : return -1;
2147 : }
2148 :
2149 : /************************************************************************/
2150 : /* Tell() */
2151 : /************************************************************************/
2152 :
2153 0 : vsi_l_offset VSIZipWriteHandle::Tell()
2154 : {
2155 : CPLError(CE_Failure, CPLE_NotSupported,
2156 0 : "VSIFTellL() is not supported on writable Zip files");
2157 0 : return 0;
2158 : }
2159 :
2160 : /************************************************************************/
2161 : /* Read() */
2162 : /************************************************************************/
2163 :
2164 0 : size_t VSIZipWriteHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
2165 : {
2166 : CPLError(CE_Failure, CPLE_NotSupported,
2167 0 : "VSIFReadL() is not supported on writable Zip files");
2168 0 : return 0;
2169 : }
2170 :
2171 : /************************************************************************/
2172 : /* Write() */
2173 : /************************************************************************/
2174 :
2175 4092 : size_t VSIZipWriteHandle::Write( const void *pBuffer, size_t nSize, size_t nMemb )
2176 : {
2177 4092 : if (poParent == NULL)
2178 : {
2179 : CPLError(CE_Failure, CPLE_NotSupported,
2180 0 : "VSIFWriteL() is not supported on main Zip file or closed subfiles");
2181 0 : return 0;
2182 : }
2183 :
2184 4092 : if (CPLWriteFileInZip( poParent->hZIP, pBuffer, (int)(nSize * nMemb) ) != CE_None)
2185 0 : return 0;
2186 :
2187 4092 : return nMemb;
2188 : }
2189 :
2190 : /************************************************************************/
2191 : /* Eof() */
2192 : /************************************************************************/
2193 :
2194 0 : int VSIZipWriteHandle::Eof()
2195 : {
2196 : CPLError(CE_Failure, CPLE_NotSupported,
2197 0 : "VSIFEofL() is not supported on writable Zip files");
2198 0 : return FALSE;
2199 : }
2200 :
2201 : /************************************************************************/
2202 : /* Flush() */
2203 : /************************************************************************/
2204 :
2205 0 : int VSIZipWriteHandle::Flush()
2206 : {
2207 : CPLError(CE_Failure, CPLE_NotSupported,
2208 0 : "VSIFFlushL() is not supported on writable Zip files");
2209 0 : return 0;
2210 : }
2211 :
2212 : /************************************************************************/
2213 : /* Close() */
2214 : /************************************************************************/
2215 :
2216 244 : int VSIZipWriteHandle::Close()
2217 : {
2218 244 : if (poParent)
2219 : {
2220 102 : CPLCloseFileInZip(poParent->hZIP);
2221 102 : poParent->poChildInWriting = NULL;
2222 102 : if (bAutoDeleteParent)
2223 26 : delete poParent;
2224 102 : poParent = NULL;
2225 : }
2226 244 : if (poChildInWriting)
2227 : {
2228 0 : poChildInWriting->Close();
2229 0 : poChildInWriting = NULL;
2230 : }
2231 244 : if (hZIP)
2232 : {
2233 34 : CPLCloseZip(hZIP);
2234 34 : hZIP = NULL;
2235 :
2236 34 : poFS->RemoveFromMap(this);
2237 : }
2238 :
2239 244 : return 0;
2240 : }
2241 :
2242 : /************************************************************************/
2243 : /* StopCurrentFile() */
2244 : /************************************************************************/
2245 :
2246 102 : void VSIZipWriteHandle::StopCurrentFile()
2247 : {
2248 102 : if (poChildInWriting)
2249 0 : poChildInWriting->Close();
2250 102 : poChildInWriting = NULL;
2251 102 : }
2252 :
2253 : /************************************************************************/
2254 : /* StartNewFile() */
2255 : /************************************************************************/
2256 :
2257 102 : void VSIZipWriteHandle::StartNewFile(VSIZipWriteHandle* poSubFile)
2258 : {
2259 102 : poChildInWriting = poSubFile;
2260 102 : }
2261 :
2262 : /************************************************************************/
2263 : /* VSIInstallZipFileHandler() */
2264 : /************************************************************************/
2265 :
2266 :
2267 : /**
2268 : * \brief Install ZIP file system handler.
2269 : *
2270 : * A special file handler is installed that allows reading on-the-fly in ZIP
2271 : * (.zip) archives.
2272 : *
2273 : * All portions of the file system underneath the base path "/vsizip/" will be
2274 : * handled by this driver.
2275 : *
2276 : * The syntax to open a file inside a zip file is /vsizip/path/to/the/file.zip/path/inside/the/zip/file
2277 : * were path/to/the/file.zip is relative or absolute and path/inside/the/zip/file
2278 : * is the relative path to the file inside the archive.
2279 : *
2280 : * If the path is absolute, it should begin with a / on a Unix-like OS (or C:\ on Windows),
2281 : * so the line looks like /vsizip//home/gdal/...
2282 : * For example gdalinfo /vsizip/myarchive.zip/subdir1/file1.tif
2283 : *
2284 : * Syntaxic sugar : if the .zip file contains only one file located at its root,
2285 : * just mentionning "/vsizip/path/to/the/file.zip" will work
2286 : *
2287 : * VSIStatL() will return the uncompressed size in st_size member and file
2288 : * nature- file or directory - in st_mode member.
2289 : *
2290 : * Directory listing is available through VSIReadDir().
2291 : *
2292 : * Since GDAL 1.8.0, write capabilities are available. They allow creating
2293 : * a new zip file and adding new files to an already existing (or just created)
2294 : * zip file. Read and write operations cannot be interleaved : the new zip must
2295 : * be closed before being re-opened for read.
2296 : *
2297 : * Additional documentation is to be found at http://trac.osgeo.org/gdal/wiki/UserDocs/ReadInZip
2298 : *
2299 : * @since GDAL 1.6.0
2300 : */
2301 :
2302 1341 : void VSIInstallZipFileHandler(void)
2303 : {
2304 1341 : VSIFileManager::InstallHandler( "/vsizip/", new VSIZipFilesystemHandler() );
2305 1341 : }
2306 :
|
