1 : /******************************************************************************
2 : * $Id: cpl_vsil_sparsefile.cpp 20025 2010-07-11 17:13:02Z rouault $
3 : *
4 : * Project: VSI Virtual File System
5 : * Purpose: Implementation of sparse file virtual io driver.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2010, Frank Warmerdam <warmerdam@pobox.com>
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_string.h"
32 : #include "cpl_multiproc.h"
33 : #include "cpl_minixml.h"
34 : #include <map>
35 :
36 : #if defined(WIN32CE)
37 : # include <wce_errno.h>
38 : #endif
39 :
40 : CPL_CVSID("$Id: cpl_vsil_sparsefile.cpp 20025 2010-07-11 17:13:02Z rouault $");
41 :
42 54 : class SFRegion {
43 : public:
44 12 : SFRegion() : fp(NULL), nDstOffset(0), nSrcOffset(0), nLength(0),
45 12 : byValue(0), bTriedOpen(FALSE) {}
46 :
47 : CPLString osFilename;
48 : FILE *fp;
49 : GUIntBig nDstOffset;
50 : GUIntBig nSrcOffset;
51 : GUIntBig nLength;
52 : GByte byValue;
53 : int bTriedOpen;
54 : };
55 :
56 : /************************************************************************/
57 : /* ==================================================================== */
58 : /* VSISparseFileHandle */
59 : /* ==================================================================== */
60 : /************************************************************************/
61 :
62 : class VSISparseFileHandle : public VSIVirtualHandle
63 3 : {
64 : public:
65 3 : VSISparseFileHandle() : nCurOffset(0) {}
66 :
67 : GUIntBig nOverallLength;
68 : GUIntBig nCurOffset;
69 :
70 : std::vector<SFRegion> aoRegions;
71 :
72 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
73 : virtual vsi_l_offset Tell();
74 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
75 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
76 : virtual int Eof();
77 : virtual int Close();
78 : };
79 :
80 : /************************************************************************/
81 : /* ==================================================================== */
82 : /* VSISparseFileFilesystemHandler */
83 : /* ==================================================================== */
84 : /************************************************************************/
85 :
86 : class VSISparseFileFilesystemHandler : public VSIFilesystemHandler
87 : {
88 : public:
89 : VSISparseFileFilesystemHandler();
90 : virtual ~VSISparseFileFilesystemHandler();
91 :
92 : int DecomposePath( const char *pszPath,
93 : CPLString &osFilename,
94 : vsi_l_offset &nSparseFileOffset,
95 : vsi_l_offset &nSparseFileSize );
96 :
97 : virtual VSIVirtualHandle *Open( const char *pszFilename,
98 : const char *pszAccess);
99 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf );
100 : virtual int Unlink( const char *pszFilename );
101 : virtual int Mkdir( const char *pszDirname, long nMode );
102 : virtual int Rmdir( const char *pszDirname );
103 : virtual char **ReadDir( const char *pszDirname );
104 : };
105 :
106 : /************************************************************************/
107 : /* ==================================================================== */
108 : /* VSISparseFileHandle */
109 : /* ==================================================================== */
110 : /************************************************************************/
111 :
112 : /************************************************************************/
113 : /* Close() */
114 : /************************************************************************/
115 :
116 2 : int VSISparseFileHandle::Close()
117 :
118 : {
119 : unsigned int i;
120 :
121 10 : for( i = 0; i < aoRegions.size(); i++ )
122 : {
123 8 : if( aoRegions[i].fp != NULL )
124 4 : VSIFCloseL( aoRegions[i].fp );
125 : }
126 :
127 2 : return 0;
128 : }
129 :
130 : /************************************************************************/
131 : /* Seek() */
132 : /************************************************************************/
133 :
134 35 : int VSISparseFileHandle::Seek( vsi_l_offset nOffset, int nWhence )
135 :
136 : {
137 35 : if( nWhence == SEEK_SET )
138 33 : nCurOffset = nOffset;
139 2 : else if( nWhence == SEEK_CUR )
140 : {
141 0 : nCurOffset += nOffset;
142 : }
143 2 : else if( nWhence == SEEK_END )
144 : {
145 2 : nCurOffset = nOverallLength + nOffset;
146 : }
147 : else
148 : {
149 0 : errno = EINVAL;
150 0 : return -1;
151 : }
152 :
153 35 : return 0;
154 : }
155 :
156 : /************************************************************************/
157 : /* Tell() */
158 : /************************************************************************/
159 :
160 2 : vsi_l_offset VSISparseFileHandle::Tell()
161 :
162 : {
163 2 : return nCurOffset;
164 : }
165 :
166 : /************************************************************************/
167 : /* Read() */
168 : /************************************************************************/
169 :
170 4979 : size_t VSISparseFileHandle::Read( void * pBuffer, size_t nSize, size_t nCount )
171 :
172 : {
173 : /* -------------------------------------------------------------------- */
174 : /* Find what region we are in, searching linearly from the */
175 : /* start. */
176 : /* -------------------------------------------------------------------- */
177 : unsigned int iRegion;
178 :
179 14818 : for( iRegion = 0; iRegion < aoRegions.size(); iRegion++ )
180 : {
181 14817 : if( nCurOffset >= aoRegions[iRegion].nDstOffset
182 : && nCurOffset < aoRegions[iRegion].nDstOffset + aoRegions[iRegion].nLength )
183 4978 : break;
184 : }
185 :
186 : /* -------------------------------------------------------------------- */
187 : /* Default to zeroing the buffer if no corresponding region was */
188 : /* found. */
189 : /* -------------------------------------------------------------------- */
190 4979 : if( iRegion == aoRegions.size() )
191 : {
192 1 : memset( pBuffer, 0, nSize * nCount );
193 1 : nCurOffset += nSize * nSize;
194 1 : return nCount;
195 : }
196 :
197 : /* -------------------------------------------------------------------- */
198 : /* If this request crosses region boundaries, split it into two */
199 : /* requests. */
200 : /* -------------------------------------------------------------------- */
201 4978 : size_t nReturnCount = nCount;
202 4978 : GUIntBig nBytesRequested = nSize * nCount;
203 : GUIntBig nBytesAvailable =
204 4978 : aoRegions[iRegion].nDstOffset + aoRegions[iRegion].nLength;
205 :
206 4978 : if( nCurOffset + nBytesRequested > nBytesAvailable )
207 : {
208 1 : size_t nExtraBytes = nCurOffset + nBytesRequested - nBytesAvailable;
209 : // Recurse to get the rest of the request.
210 :
211 1 : GUIntBig nCurOffsetSave = nCurOffset;
212 1 : nCurOffset += nBytesRequested - nExtraBytes;
213 : size_t nBytesRead =
214 : this->Read( ((char *) pBuffer) + nBytesRequested - nExtraBytes,
215 1 : 1, nExtraBytes );
216 1 : nCurOffset = nCurOffsetSave;
217 :
218 1 : if( nBytesRead < nExtraBytes )
219 0 : nReturnCount -= (nExtraBytes-nBytesRead) / nSize;
220 :
221 1 : nBytesRequested -= nExtraBytes;
222 : }
223 :
224 : /* -------------------------------------------------------------------- */
225 : /* Handle a constant region. */
226 : /* -------------------------------------------------------------------- */
227 4978 : if( aoRegions[iRegion].osFilename.size() == 0 )
228 : {
229 1 : memset( pBuffer, aoRegions[iRegion].byValue, nBytesRequested );
230 : }
231 :
232 : /* -------------------------------------------------------------------- */
233 : /* Otherwise handle as a file. */
234 : /* -------------------------------------------------------------------- */
235 : else
236 : {
237 4977 : if( aoRegions[iRegion].fp == NULL )
238 : {
239 4 : if( !aoRegions[iRegion].bTriedOpen )
240 : {
241 : aoRegions[iRegion].fp =
242 4 : VSIFOpenL( aoRegions[iRegion].osFilename, "r" );
243 4 : if( aoRegions[iRegion].fp == NULL )
244 : {
245 : CPLDebug( "/vsisparse/", "Failed to open '%s'.",
246 0 : aoRegions[iRegion].osFilename.c_str() );
247 : }
248 4 : aoRegions[iRegion].bTriedOpen = TRUE;
249 : }
250 4 : if( aoRegions[iRegion].fp == NULL )
251 : {
252 0 : return 0;
253 : }
254 : }
255 :
256 4977 : if( VSIFSeekL( aoRegions[iRegion].fp,
257 : nCurOffset
258 : - aoRegions[iRegion].nDstOffset
259 : + aoRegions[iRegion].nSrcOffset,
260 : SEEK_SET ) != 0 )
261 0 : return 0;
262 :
263 : size_t nBytesRead = VSIFReadL( pBuffer, 1, nBytesRequested,
264 4977 : aoRegions[iRegion].fp );
265 :
266 4977 : if( nBytesAvailable < nBytesRequested )
267 0 : nReturnCount = nBytesRead / nSize;
268 : }
269 :
270 4978 : nCurOffset += nReturnCount * nSize;
271 :
272 4978 : return nReturnCount;
273 : }
274 :
275 : /************************************************************************/
276 : /* Write() */
277 : /************************************************************************/
278 :
279 0 : size_t VSISparseFileHandle::Write( const void * pBuffer, size_t nSize, size_t nCount )
280 :
281 : {
282 0 : errno = EBADF;
283 0 : return 0;
284 : }
285 :
286 : /************************************************************************/
287 : /* Eof() */
288 : /************************************************************************/
289 :
290 0 : int VSISparseFileHandle::Eof()
291 :
292 : {
293 0 : return nCurOffset >= nOverallLength;
294 : }
295 :
296 : /************************************************************************/
297 : /* ==================================================================== */
298 : /* VSISparseFileFilesystemHandler */
299 : /* ==================================================================== */
300 : /************************************************************************/
301 :
302 : /************************************************************************/
303 : /* VSISparseFileFilesystemHandler() */
304 : /************************************************************************/
305 :
306 447 : VSISparseFileFilesystemHandler::VSISparseFileFilesystemHandler()
307 :
308 : {
309 447 : }
310 :
311 : /************************************************************************/
312 : /* ~VSISparseFileFilesystemHandler() */
313 : /************************************************************************/
314 :
315 432 : VSISparseFileFilesystemHandler::~VSISparseFileFilesystemHandler()
316 :
317 : {
318 432 : }
319 :
320 : /************************************************************************/
321 : /* Open() */
322 : /************************************************************************/
323 :
324 : VSIVirtualHandle *
325 : VSISparseFileFilesystemHandler::Open( const char *pszFilename,
326 8 : const char *pszAccess )
327 :
328 : {
329 8 : CPLAssert( EQUALN(pszFilename,"/vsisparse/", 11) );
330 :
331 8 : if( !EQUAL(pszAccess,"r") && !EQUAL(pszAccess,"rb") )
332 : {
333 0 : errno = EACCES;
334 0 : return NULL;
335 : }
336 :
337 8 : CPLString osSparseFilePath = pszFilename + 11;
338 :
339 : /* -------------------------------------------------------------------- */
340 : /* Does this file even exist? */
341 : /* -------------------------------------------------------------------- */
342 8 : FILE *fp = VSIFOpenL( osSparseFilePath, "r" );
343 8 : if( fp == NULL )
344 5 : return NULL;
345 3 : VSIFCloseL( fp );
346 :
347 : /* -------------------------------------------------------------------- */
348 : /* Read the XML file. */
349 : /* -------------------------------------------------------------------- */
350 3 : CPLXMLNode *psXMLRoot = CPLParseXMLFile( osSparseFilePath );
351 :
352 3 : if( psXMLRoot == NULL )
353 0 : return NULL;
354 :
355 : /* -------------------------------------------------------------------- */
356 : /* Setup the file handle on this file. */
357 : /* -------------------------------------------------------------------- */
358 3 : VSISparseFileHandle *poHandle = new VSISparseFileHandle;
359 :
360 : /* -------------------------------------------------------------------- */
361 : /* Translate the desired fields out of the XML tree. */
362 : /* -------------------------------------------------------------------- */
363 : CPLXMLNode *psRegion;
364 :
365 30 : for( psRegion = psXMLRoot->psChild;
366 : psRegion != NULL;
367 : psRegion = psRegion->psNext )
368 : {
369 15 : if( psRegion->eType != CXT_Element )
370 0 : continue;
371 :
372 15 : if( !EQUAL(psRegion->pszValue,"SubfileRegion")
373 : && !EQUAL(psRegion->pszValue,"ConstantRegion") )
374 3 : continue;
375 :
376 12 : SFRegion oRegion;
377 :
378 12 : oRegion.osFilename = CPLGetXMLValue( psRegion, "Filename", "" );
379 12 : if( atoi(CPLGetXMLValue( psRegion, "Filename.relative", "0" )) != 0 )
380 : {
381 9 : CPLString osSFPath = CPLGetPath(osSparseFilePath);
382 : oRegion.osFilename = CPLFormFilename( osSFPath,
383 9 : oRegion.osFilename, NULL );
384 : }
385 :
386 : oRegion.nDstOffset =
387 : CPLScanUIntBig( CPLGetXMLValue(psRegion,"DestinationOffset","0" ),
388 12 : 32 );
389 :
390 : oRegion.nSrcOffset =
391 12 : CPLScanUIntBig( CPLGetXMLValue(psRegion,"SourceOffset","0" ), 32);
392 :
393 : oRegion.nLength =
394 12 : CPLScanUIntBig( CPLGetXMLValue(psRegion,"RegionLength","0" ), 32);
395 :
396 12 : oRegion.byValue = atoi(CPLGetXMLValue(psRegion,"Value","0" ));
397 :
398 12 : poHandle->aoRegions.push_back( oRegion );
399 : }
400 :
401 : /* -------------------------------------------------------------------- */
402 : /* Get sparse file length, use maximum bound of regions if not */
403 : /* explicit in file. */
404 : /* -------------------------------------------------------------------- */
405 : poHandle->nOverallLength =
406 3 : CPLScanUIntBig( CPLGetXMLValue(psXMLRoot,"Length","0" ), 32);
407 3 : if( poHandle->nOverallLength == 0 )
408 : {
409 : unsigned int i;
410 :
411 0 : for( i = 0; i < poHandle->aoRegions.size(); i++ )
412 : {
413 0 : poHandle->nOverallLength = MAX(poHandle->nOverallLength,
414 : poHandle->aoRegions[i].nDstOffset
415 : + poHandle->aoRegions[i].nLength);
416 : }
417 : }
418 :
419 3 : CPLDestroyXMLNode( psXMLRoot );
420 :
421 3 : return poHandle;
422 : }
423 :
424 : /************************************************************************/
425 : /* Stat() */
426 : /************************************************************************/
427 :
428 : int VSISparseFileFilesystemHandler::Stat( const char * pszFilename,
429 2 : VSIStatBufL * psStatBuf )
430 :
431 : {
432 2 : VSIVirtualHandle *poFile = Open( pszFilename, "r" );
433 :
434 2 : memset( psStatBuf, 0, sizeof(VSIStatBufL) );
435 :
436 2 : if( poFile == NULL )
437 1 : return -1;
438 :
439 1 : poFile->Seek( 0, SEEK_END );
440 1 : size_t nLength = poFile->Tell();
441 1 : delete poFile;
442 :
443 : int nResult = VSIStatL( pszFilename + strlen("/vsisparse/"),
444 1 : psStatBuf );
445 :
446 1 : psStatBuf->st_size = nLength;
447 :
448 1 : return nResult;
449 : }
450 :
451 : /************************************************************************/
452 : /* Unlink() */
453 : /************************************************************************/
454 :
455 0 : int VSISparseFileFilesystemHandler::Unlink( const char * pszFilename )
456 :
457 : {
458 0 : errno = EACCES;
459 0 : return -1;
460 : }
461 :
462 : /************************************************************************/
463 : /* Mkdir() */
464 : /************************************************************************/
465 :
466 : int VSISparseFileFilesystemHandler::Mkdir( const char * pszPathname,
467 0 : long nMode )
468 :
469 : {
470 0 : errno = EACCES;
471 0 : return -1;
472 : }
473 :
474 : /************************************************************************/
475 : /* Rmdir() */
476 : /************************************************************************/
477 :
478 0 : int VSISparseFileFilesystemHandler::Rmdir( const char * pszPathname )
479 :
480 : {
481 0 : errno = EACCES;
482 0 : return -1;
483 : }
484 :
485 : /************************************************************************/
486 : /* ReadDir() */
487 : /************************************************************************/
488 :
489 1 : char **VSISparseFileFilesystemHandler::ReadDir( const char *pszPath )
490 :
491 : {
492 1 : errno = EACCES;
493 1 : return NULL;
494 : }
495 :
496 : /************************************************************************/
497 : /* VSIInstallSparseFileFilesystemHandler() */
498 : /************************************************************************/
499 :
500 : /**
501 : * Install /vsisparse/ virtual file handler.
502 : *
503 : * The sparse virtual file handler allows a virtual file to be composed
504 : * from chunks of data in other files, potentially with large spaces in
505 : * the virtual file set to a constant value. This can make it possible to
506 : * test some sorts of operations on what seems to be a large file with
507 : * image data set to a constant value. It is also helpful when wanting to
508 : * add test files to the test suite that are too large, but for which most
509 : * of the data can be ignored. It could, in theory, also be used to
510 : * treat several files on different file systems as one large virtual file.
511 : *
512 : * The file referenced by /vsisparse/ should be an XML control file
513 : * formatted something like:
514 : *
515 : *
516 : \verbatim
517 : <VSISparseFile>
518 : <Length>87629264</Length>
519 : <SubfileRegion> Stuff at start of file.
520 : <Filename relative="1">251_head.dat</Filename>
521 : <DestinationOffset>0</DestinationOffset>
522 : <SourceOffset>0</SourceOffset>
523 : <RegionLength>2768</RegionLength>
524 : </SubfileRegion>
525 :
526 : <SubfileRegion> RasterDMS node.
527 : <Filename relative="1">251_rasterdms.dat</Filename>
528 : <DestinationOffset>87313104</DestinationOffset>
529 : <SourceOffset>0</SourceOffset>
530 : <RegionLength>160</RegionLength>
531 : </SubfileRegion>
532 :
533 : <SubfileRegion> Stuff at end of file.
534 : <Filename relative="1">251_tail.dat</Filename>
535 : <DestinationOffset>87611924</DestinationOffset>
536 : <SourceOffset>0</SourceOffset>
537 : <RegionLength>17340</RegionLength>
538 : </SubfileRegion>
539 :
540 : <ConstantRegion> Default for the rest of the file.
541 : <DestinationOffset>0</DestinationOffset>
542 : <RegionLength>87629264</RegionLength>
543 : <Value>0</Value>
544 : </ConstantRegion>
545 : </VSISparseFile>
546 : \endverbatim
547 : *
548 : * Hopefully the values and semantics are fairly obvious.
549 : *
550 : * This driver is installed by default.
551 : */
552 :
553 447 : void VSIInstallSparseFileHandler()
554 : {
555 : VSIFileManager::InstallHandler( "/vsisparse/",
556 447 : new VSISparseFileFilesystemHandler );
557 447 : }
558 :
|
