1 : /******************************************************************************
2 : * $Id: cpl_vsil_sparsefile.cpp 21167 2010-11-24 15:19:51Z warmerdam $
3 : *
4 : * Project: VSI Virtual File System
5 : * Purpose: Implementation of sparse file virtual io driver.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2010, Frank Warmerdam <warmerdam@pobox.com>
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_string.h"
32 : #include "cpl_multiproc.h"
33 : #include "cpl_minixml.h"
34 : #include <map>
35 :
36 : #if defined(WIN32CE)
37 : # include <wce_errno.h>
38 : #endif
39 :
40 : CPL_CVSID("$Id: cpl_vsil_sparsefile.cpp 21167 2010-11-24 15:19:51Z warmerdam $");
41 :
42 54 : class SFRegion {
43 : public:
44 12 : SFRegion() : fp(NULL), nDstOffset(0), nSrcOffset(0), nLength(0),
45 12 : byValue(0), bTriedOpen(FALSE) {}
46 :
47 : CPLString osFilename;
48 : VSILFILE *fp;
49 : GUIntBig nDstOffset;
50 : GUIntBig nSrcOffset;
51 : GUIntBig nLength;
52 : GByte byValue;
53 : int bTriedOpen;
54 : };
55 :
56 : /************************************************************************/
57 : /* ==================================================================== */
58 : /* VSISparseFileHandle */
59 : /* ==================================================================== */
60 : /************************************************************************/
61 :
62 : class VSISparseFileHandle : public VSIVirtualHandle
63 3 : {
64 : public:
65 3 : VSISparseFileHandle() : nCurOffset(0) {}
66 :
67 : GUIntBig nOverallLength;
68 : GUIntBig nCurOffset;
69 :
70 : std::vector<SFRegion> aoRegions;
71 :
72 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
73 : virtual vsi_l_offset Tell();
74 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
75 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
76 : virtual int Eof();
77 : virtual int Close();
78 : };
79 :
80 : /************************************************************************/
81 : /* ==================================================================== */
82 : /* VSISparseFileFilesystemHandler */
83 : /* ==================================================================== */
84 : /************************************************************************/
85 :
86 : class VSISparseFileFilesystemHandler : public VSIFilesystemHandler
87 : {
88 : public:
89 : VSISparseFileFilesystemHandler();
90 : virtual ~VSISparseFileFilesystemHandler();
91 :
92 : int DecomposePath( const char *pszPath,
93 : CPLString &osFilename,
94 : vsi_l_offset &nSparseFileOffset,
95 : vsi_l_offset &nSparseFileSize );
96 :
97 : virtual VSIVirtualHandle *Open( const char *pszFilename,
98 : const char *pszAccess);
99 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
100 : virtual int Unlink( const char *pszFilename );
101 : virtual int Mkdir( const char *pszDirname, long nMode );
102 : virtual int Rmdir( const char *pszDirname );
103 : virtual char **ReadDir( const char *pszDirname );
104 : };
105 :
106 : /************************************************************************/
107 : /* ==================================================================== */
108 : /* VSISparseFileHandle */
109 : /* ==================================================================== */
110 : /************************************************************************/
111 :
112 : /************************************************************************/
113 : /* Close() */
114 : /************************************************************************/
115 :
116 2 : int VSISparseFileHandle::Close()
117 :
118 : {
119 : unsigned int i;
120 :
121 10 : for( i = 0; i < aoRegions.size(); i++ )
122 : {
123 8 : if( aoRegions[i].fp != NULL )
124 4 : VSIFCloseL( aoRegions[i].fp );
125 : }
126 :
127 2 : return 0;
128 : }
129 :
130 : /************************************************************************/
131 : /* Seek() */
132 : /************************************************************************/
133 :
134 35 : int VSISparseFileHandle::Seek( vsi_l_offset nOffset, int nWhence )
135 :
136 : {
137 35 : if( nWhence == SEEK_SET )
138 33 : nCurOffset = nOffset;
139 2 : else if( nWhence == SEEK_CUR )
140 : {
141 0 : nCurOffset += nOffset;
142 : }
143 2 : else if( nWhence == SEEK_END )
144 : {
145 2 : nCurOffset = nOverallLength + nOffset;
146 : }
147 : else
148 : {
149 0 : errno = EINVAL;
150 0 : return -1;
151 : }
152 :
153 35 : return 0;
154 : }
155 :
156 : /************************************************************************/
157 : /* Tell() */
158 : /************************************************************************/
159 :
160 2 : vsi_l_offset VSISparseFileHandle::Tell()
161 :
162 : {
163 2 : return nCurOffset;
164 : }
165 :
166 : /************************************************************************/
167 : /* Read() */
168 : /************************************************************************/
169 :
170 4979 : size_t VSISparseFileHandle::Read( void * pBuffer, size_t nSize, size_t nCount )
171 :
172 : {
173 : /* -------------------------------------------------------------------- */
174 : /* Find what region we are in, searching linearly from the */
175 : /* start. */
176 : /* -------------------------------------------------------------------- */
177 : unsigned int iRegion;
178 :
179 14818 : for( iRegion = 0; iRegion < aoRegions.size(); iRegion++ )
180 : {
181 14817 : if( nCurOffset >= aoRegions[iRegion].nDstOffset
182 : && nCurOffset < aoRegions[iRegion].nDstOffset + aoRegions[iRegion].nLength )
183 4978 : break;
184 : }
185 :
186 : /* -------------------------------------------------------------------- */
187 : /* Default to zeroing the buffer if no corresponding region was */
188 : /* found. */
189 : /* -------------------------------------------------------------------- */
190 4979 : if( iRegion == aoRegions.size() )
191 : {
192 1 : memset( pBuffer, 0, nSize * nCount );
193 1 : nCurOffset += nSize * nSize;
194 1 : return nCount;
195 : }
196 :
197 : /* -------------------------------------------------------------------- */
198 : /* If this request crosses region boundaries, split it into two */
199 : /* requests. */
200 : /* -------------------------------------------------------------------- */
201 4978 : size_t nReturnCount = nCount;
202 4978 : GUIntBig nBytesRequested = nSize * nCount;
203 : GUIntBig nBytesAvailable =
204 4978 : aoRegions[iRegion].nDstOffset + aoRegions[iRegion].nLength;
205 :
206 4978 : if( nCurOffset + nBytesRequested > nBytesAvailable )
207 : {
208 : size_t nExtraBytes =
209 1 : (size_t) (nCurOffset + nBytesRequested - nBytesAvailable);
210 : // Recurse to get the rest of the request.
211 :
212 1 : GUIntBig nCurOffsetSave = nCurOffset;
213 1 : nCurOffset += nBytesRequested - nExtraBytes;
214 : size_t nBytesRead =
215 : this->Read( ((char *) pBuffer) + nBytesRequested - nExtraBytes,
216 1 : 1, nExtraBytes );
217 1 : nCurOffset = nCurOffsetSave;
218 :
219 1 : if( nBytesRead < nExtraBytes )
220 0 : nReturnCount -= (nExtraBytes-nBytesRead) / nSize;
221 :
222 1 : nBytesRequested -= nExtraBytes;
223 : }
224 :
225 : /* -------------------------------------------------------------------- */
226 : /* Handle a constant region. */
227 : /* -------------------------------------------------------------------- */
228 4978 : if( aoRegions[iRegion].osFilename.size() == 0 )
229 : {
230 : memset( pBuffer, aoRegions[iRegion].byValue,
231 1 : (size_t) nBytesRequested );
232 : }
233 :
234 : /* -------------------------------------------------------------------- */
235 : /* Otherwise handle as a file. */
236 : /* -------------------------------------------------------------------- */
237 : else
238 : {
239 4977 : if( aoRegions[iRegion].fp == NULL )
240 : {
241 4 : if( !aoRegions[iRegion].bTriedOpen )
242 : {
243 : aoRegions[iRegion].fp =
244 4 : VSIFOpenL( aoRegions[iRegion].osFilename, "r" );
245 4 : if( aoRegions[iRegion].fp == NULL )
246 : {
247 : CPLDebug( "/vsisparse/", "Failed to open '%s'.",
248 0 : aoRegions[iRegion].osFilename.c_str() );
249 : }
250 4 : aoRegions[iRegion].bTriedOpen = TRUE;
251 : }
252 4 : if( aoRegions[iRegion].fp == NULL )
253 : {
254 0 : return 0;
255 : }
256 : }
257 :
258 4977 : if( VSIFSeekL( aoRegions[iRegion].fp,
259 : nCurOffset
260 : - aoRegions[iRegion].nDstOffset
261 : + aoRegions[iRegion].nSrcOffset,
262 : SEEK_SET ) != 0 )
263 0 : return 0;
264 :
265 : size_t nBytesRead = VSIFReadL( pBuffer, 1, (size_t) nBytesRequested,
266 4977 : aoRegions[iRegion].fp );
267 :
268 4977 : if( nBytesAvailable < nBytesRequested )
269 0 : nReturnCount = nBytesRead / nSize;
270 : }
271 :
272 4978 : nCurOffset += nReturnCount * nSize;
273 :
274 4978 : return nReturnCount;
275 : }
276 :
277 : /************************************************************************/
278 : /* Write() */
279 : /************************************************************************/
280 :
281 0 : size_t VSISparseFileHandle::Write( const void * pBuffer, size_t nSize, size_t nCount )
282 :
283 : {
284 0 : errno = EBADF;
285 0 : return 0;
286 : }
287 :
288 : /************************************************************************/
289 : /* Eof() */
290 : /************************************************************************/
291 :
292 0 : int VSISparseFileHandle::Eof()
293 :
294 : {
295 0 : return nCurOffset >= nOverallLength;
296 : }
297 :
298 : /************************************************************************/
299 : /* ==================================================================== */
300 : /* VSISparseFileFilesystemHandler */
301 : /* ==================================================================== */
302 : /************************************************************************/
303 :
304 : /************************************************************************/
305 : /* VSISparseFileFilesystemHandler() */
306 : /************************************************************************/
307 :
308 712 : VSISparseFileFilesystemHandler::VSISparseFileFilesystemHandler()
309 :
310 : {
311 712 : }
312 :
313 : /************************************************************************/
314 : /* ~VSISparseFileFilesystemHandler() */
315 : /************************************************************************/
316 :
317 687 : VSISparseFileFilesystemHandler::~VSISparseFileFilesystemHandler()
318 :
319 : {
320 687 : }
321 :
322 : /************************************************************************/
323 : /* Open() */
324 : /************************************************************************/
325 :
326 : VSIVirtualHandle *
327 8 : VSISparseFileFilesystemHandler::Open( const char *pszFilename,
328 : const char *pszAccess )
329 :
330 : {
331 8 : CPLAssert( EQUALN(pszFilename,"/vsisparse/", 11) );
332 :
333 8 : if( !EQUAL(pszAccess,"r") && !EQUAL(pszAccess,"rb") )
334 : {
335 0 : errno = EACCES;
336 0 : return NULL;
337 : }
338 :
339 8 : CPLString osSparseFilePath = pszFilename + 11;
340 :
341 : /* -------------------------------------------------------------------- */
342 : /* Does this file even exist? */
343 : /* -------------------------------------------------------------------- */
344 8 : VSILFILE *fp = VSIFOpenL( osSparseFilePath, "r" );
345 8 : if( fp == NULL )
346 5 : return NULL;
347 3 : VSIFCloseL( fp );
348 :
349 : /* -------------------------------------------------------------------- */
350 : /* Read the XML file. */
351 : /* -------------------------------------------------------------------- */
352 3 : CPLXMLNode *psXMLRoot = CPLParseXMLFile( osSparseFilePath );
353 :
354 3 : if( psXMLRoot == NULL )
355 0 : return NULL;
356 :
357 : /* -------------------------------------------------------------------- */
358 : /* Setup the file handle on this file. */
359 : /* -------------------------------------------------------------------- */
360 3 : VSISparseFileHandle *poHandle = new VSISparseFileHandle;
361 :
362 : /* -------------------------------------------------------------------- */
363 : /* Translate the desired fields out of the XML tree. */
364 : /* -------------------------------------------------------------------- */
365 : CPLXMLNode *psRegion;
366 :
367 15 : for( psRegion = psXMLRoot->psChild;
368 : psRegion != NULL;
369 : psRegion = psRegion->psNext )
370 : {
371 15 : if( psRegion->eType != CXT_Element )
372 0 : continue;
373 :
374 15 : if( !EQUAL(psRegion->pszValue,"SubfileRegion")
375 : && !EQUAL(psRegion->pszValue,"ConstantRegion") )
376 3 : continue;
377 :
378 12 : SFRegion oRegion;
379 :
380 12 : oRegion.osFilename = CPLGetXMLValue( psRegion, "Filename", "" );
381 12 : if( atoi(CPLGetXMLValue( psRegion, "Filename.relative", "0" )) != 0 )
382 : {
383 9 : CPLString osSFPath = CPLGetPath(osSparseFilePath);
384 : oRegion.osFilename = CPLFormFilename( osSFPath,
385 9 : oRegion.osFilename, NULL );
386 : }
387 :
388 : oRegion.nDstOffset =
389 : CPLScanUIntBig( CPLGetXMLValue(psRegion,"DestinationOffset","0" ),
390 12 : 32 );
391 :
392 : oRegion.nSrcOffset =
393 12 : CPLScanUIntBig( CPLGetXMLValue(psRegion,"SourceOffset","0" ), 32);
394 :
395 : oRegion.nLength =
396 12 : CPLScanUIntBig( CPLGetXMLValue(psRegion,"RegionLength","0" ), 32);
397 :
398 12 : oRegion.byValue = (GByte) atoi(CPLGetXMLValue(psRegion,"Value","0" ));
399 :
400 12 : poHandle->aoRegions.push_back( oRegion );
401 : }
402 :
403 : /* -------------------------------------------------------------------- */
404 : /* Get sparse file length, use maximum bound of regions if not */
405 : /* explicit in file. */
406 : /* -------------------------------------------------------------------- */
407 : poHandle->nOverallLength =
408 3 : CPLScanUIntBig( CPLGetXMLValue(psXMLRoot,"Length","0" ), 32);
409 3 : if( poHandle->nOverallLength == 0 )
410 : {
411 : unsigned int i;
412 :
413 0 : for( i = 0; i < poHandle->aoRegions.size(); i++ )
414 : {
415 : poHandle->nOverallLength = MAX(poHandle->nOverallLength,
416 : poHandle->aoRegions[i].nDstOffset
417 0 : + poHandle->aoRegions[i].nLength);
418 : }
419 : }
420 :
421 3 : CPLDestroyXMLNode( psXMLRoot );
422 :
423 3 : return poHandle;
424 : }
425 :
426 : /************************************************************************/
427 : /* Stat() */
428 : /************************************************************************/
429 :
430 2 : int VSISparseFileFilesystemHandler::Stat( const char * pszFilename,
431 : VSIStatBufL * psStatBuf,
432 : int nFlags )
433 :
434 : {
435 2 : VSIVirtualHandle *poFile = Open( pszFilename, "r" );
436 :
437 2 : memset( psStatBuf, 0, sizeof(VSIStatBufL) );
438 :
439 2 : if( poFile == NULL )
440 1 : return -1;
441 :
442 1 : poFile->Seek( 0, SEEK_END );
443 1 : size_t nLength = (size_t) poFile->Tell();
444 1 : delete poFile;
445 :
446 : int nResult = VSIStatExL( pszFilename + strlen("/vsisparse/"),
447 1 : psStatBuf, nFlags );
448 :
449 1 : psStatBuf->st_size = nLength;
450 :
451 1 : return nResult;
452 : }
453 :
454 : /************************************************************************/
455 : /* Unlink() */
456 : /************************************************************************/
457 :
458 0 : int VSISparseFileFilesystemHandler::Unlink( const char * pszFilename )
459 :
460 : {
461 0 : errno = EACCES;
462 0 : return -1;
463 : }
464 :
465 : /************************************************************************/
466 : /* Mkdir() */
467 : /************************************************************************/
468 :
469 0 : int VSISparseFileFilesystemHandler::Mkdir( const char * pszPathname,
470 : long nMode )
471 :
472 : {
473 0 : errno = EACCES;
474 0 : return -1;
475 : }
476 :
477 : /************************************************************************/
478 : /* Rmdir() */
479 : /************************************************************************/
480 :
481 0 : int VSISparseFileFilesystemHandler::Rmdir( const char * pszPathname )
482 :
483 : {
484 0 : errno = EACCES;
485 0 : return -1;
486 : }
487 :
488 : /************************************************************************/
489 : /* ReadDir() */
490 : /************************************************************************/
491 :
492 1 : char **VSISparseFileFilesystemHandler::ReadDir( const char *pszPath )
493 :
494 : {
495 1 : errno = EACCES;
496 1 : return NULL;
497 : }
498 :
499 : /************************************************************************/
500 : /* VSIInstallSparseFileFilesystemHandler() */
501 : /************************************************************************/
502 :
503 : /**
504 : * Install /vsisparse/ virtual file handler.
505 : *
506 : * The sparse virtual file handler allows a virtual file to be composed
507 : * from chunks of data in other files, potentially with large spaces in
508 : * the virtual file set to a constant value. This can make it possible to
509 : * test some sorts of operations on what seems to be a large file with
510 : * image data set to a constant value. It is also helpful when wanting to
511 : * add test files to the test suite that are too large, but for which most
512 : * of the data can be ignored. It could, in theory, also be used to
513 : * treat several files on different file systems as one large virtual file.
514 : *
515 : * The file referenced by /vsisparse/ should be an XML control file
516 : * formatted something like:
517 : *
518 : *
519 : \verbatim
520 : <VSISparseFile>
521 : <Length>87629264</Length>
522 : <SubfileRegion> Stuff at start of file.
523 : <Filename relative="1">251_head.dat</Filename>
524 : <DestinationOffset>0</DestinationOffset>
525 : <SourceOffset>0</SourceOffset>
526 : <RegionLength>2768</RegionLength>
527 : </SubfileRegion>
528 :
529 : <SubfileRegion> RasterDMS node.
530 : <Filename relative="1">251_rasterdms.dat</Filename>
531 : <DestinationOffset>87313104</DestinationOffset>
532 : <SourceOffset>0</SourceOffset>
533 : <RegionLength>160</RegionLength>
534 : </SubfileRegion>
535 :
536 : <SubfileRegion> Stuff at end of file.
537 : <Filename relative="1">251_tail.dat</Filename>
538 : <DestinationOffset>87611924</DestinationOffset>
539 : <SourceOffset>0</SourceOffset>
540 : <RegionLength>17340</RegionLength>
541 : </SubfileRegion>
542 :
543 : <ConstantRegion> Default for the rest of the file.
544 : <DestinationOffset>0</DestinationOffset>
545 : <RegionLength>87629264</RegionLength>
546 : <Value>0</Value>
547 : </ConstantRegion>
548 : </VSISparseFile>
549 : \endverbatim
550 : *
551 : * Hopefully the values and semantics are fairly obvious.
552 : *
553 : * This driver is installed by default.
554 : */
555 :
556 712 : void VSIInstallSparseFileHandler()
557 : {
558 : VSIFileManager::InstallHandler( "/vsisparse/",
559 712 : new VSISparseFileFilesystemHandler );
560 712 : }
561 :
|
