1 : /******************************************************************************
2 : * $Id: cpl_vsil_sparsefile.cpp 21167 2010-11-24 15:19:51Z warmerdam $
3 : *
4 : * Project: VSI Virtual File System
5 : * Purpose: Implementation of sparse file virtual io driver.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2010, Frank Warmerdam <warmerdam@pobox.com>
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_string.h"
32 : #include "cpl_multiproc.h"
33 : #include "cpl_minixml.h"
34 : #include <map>
35 :
36 : #if defined(WIN32CE)
37 : # include <wce_errno.h>
38 : #endif
39 :
40 : CPL_CVSID("$Id: cpl_vsil_sparsefile.cpp 21167 2010-11-24 15:19:51Z warmerdam $");
41 :
42 108 : class SFRegion {
43 : public:
44 24 : SFRegion() : fp(NULL), nDstOffset(0), nSrcOffset(0), nLength(0),
45 24 : byValue(0), bTriedOpen(FALSE) {}
46 :
47 : CPLString osFilename;
48 : VSILFILE *fp;
49 : GUIntBig nDstOffset;
50 : GUIntBig nSrcOffset;
51 : GUIntBig nLength;
52 : GByte byValue;
53 : int bTriedOpen;
54 : };
55 :
56 : /************************************************************************/
57 : /* ==================================================================== */
58 : /* VSISparseFileHandle */
59 : /* ==================================================================== */
60 : /************************************************************************/
61 :
62 : class VSISparseFileHandle : public VSIVirtualHandle
63 6 : {
64 : public:
65 6 : VSISparseFileHandle() : nCurOffset(0) {}
66 :
67 : GUIntBig nOverallLength;
68 : GUIntBig nCurOffset;
69 :
70 : std::vector<SFRegion> aoRegions;
71 :
72 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
73 : virtual vsi_l_offset Tell();
74 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
75 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
76 : virtual int Eof();
77 : virtual int Close();
78 : };
79 :
80 : /************************************************************************/
81 : /* ==================================================================== */
82 : /* VSISparseFileFilesystemHandler */
83 : /* ==================================================================== */
84 : /************************************************************************/
85 :
86 : class VSISparseFileFilesystemHandler : public VSIFilesystemHandler
87 : {
88 : public:
89 : VSISparseFileFilesystemHandler();
90 : virtual ~VSISparseFileFilesystemHandler();
91 :
92 : int DecomposePath( const char *pszPath,
93 : CPLString &osFilename,
94 : vsi_l_offset &nSparseFileOffset,
95 : vsi_l_offset &nSparseFileSize );
96 :
97 : virtual VSIVirtualHandle *Open( const char *pszFilename,
98 : const char *pszAccess);
99 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
100 : virtual int Unlink( const char *pszFilename );
101 : virtual int Mkdir( const char *pszDirname, long nMode );
102 : virtual int Rmdir( const char *pszDirname );
103 : virtual char **ReadDir( const char *pszDirname );
104 : };
105 :
106 : /************************************************************************/
107 : /* ==================================================================== */
108 : /* VSISparseFileHandle */
109 : /* ==================================================================== */
110 : /************************************************************************/
111 :
112 : /************************************************************************/
113 : /* Close() */
114 : /************************************************************************/
115 :
116 4 : int VSISparseFileHandle::Close()
117 :
118 : {
119 : unsigned int i;
120 :
121 20 : for( i = 0; i < aoRegions.size(); i++ )
122 : {
123 16 : if( aoRegions[i].fp != NULL )
124 8 : VSIFCloseL( aoRegions[i].fp );
125 : }
126 :
127 4 : return 0;
128 : }
129 :
130 : /************************************************************************/
131 : /* Seek() */
132 : /************************************************************************/
133 :
134 70 : int VSISparseFileHandle::Seek( vsi_l_offset nOffset, int nWhence )
135 :
136 : {
137 70 : if( nWhence == SEEK_SET )
138 66 : nCurOffset = nOffset;
139 4 : else if( nWhence == SEEK_CUR )
140 : {
141 0 : nCurOffset += nOffset;
142 : }
143 4 : else if( nWhence == SEEK_END )
144 : {
145 4 : nCurOffset = nOverallLength + nOffset;
146 : }
147 : else
148 : {
149 0 : errno = EINVAL;
150 0 : return -1;
151 : }
152 :
153 70 : return 0;
154 : }
155 :
156 : /************************************************************************/
157 : /* Tell() */
158 : /************************************************************************/
159 :
160 4 : vsi_l_offset VSISparseFileHandle::Tell()
161 :
162 : {
163 4 : return nCurOffset;
164 : }
165 :
166 : /************************************************************************/
167 : /* Read() */
168 : /************************************************************************/
169 :
170 9958 : size_t VSISparseFileHandle::Read( void * pBuffer, size_t nSize, size_t nCount )
171 :
172 : {
173 : /* -------------------------------------------------------------------- */
174 : /* Find what region we are in, searching linearly from the */
175 : /* start. */
176 : /* -------------------------------------------------------------------- */
177 : unsigned int iRegion;
178 :
179 29636 : for( iRegion = 0; iRegion < aoRegions.size(); iRegion++ )
180 : {
181 29634 : if( nCurOffset >= aoRegions[iRegion].nDstOffset
182 : && nCurOffset < aoRegions[iRegion].nDstOffset + aoRegions[iRegion].nLength )
183 9956 : break;
184 : }
185 :
186 : /* -------------------------------------------------------------------- */
187 : /* Default to zeroing the buffer if no corresponding region was */
188 : /* found. */
189 : /* -------------------------------------------------------------------- */
190 9958 : if( iRegion == aoRegions.size() )
191 : {
192 2 : memset( pBuffer, 0, nSize * nCount );
193 2 : nCurOffset += nSize * nSize;
194 2 : return nCount;
195 : }
196 :
197 : /* -------------------------------------------------------------------- */
198 : /* If this request crosses region boundaries, split it into two */
199 : /* requests. */
200 : /* -------------------------------------------------------------------- */
201 9956 : size_t nReturnCount = nCount;
202 9956 : GUIntBig nBytesRequested = nSize * nCount;
203 : GUIntBig nBytesAvailable =
204 9956 : aoRegions[iRegion].nDstOffset + aoRegions[iRegion].nLength;
205 :
206 9956 : if( nCurOffset + nBytesRequested > nBytesAvailable )
207 : {
208 : size_t nExtraBytes =
209 2 : (size_t) (nCurOffset + nBytesRequested - nBytesAvailable);
210 : // Recurse to get the rest of the request.
211 :
212 2 : GUIntBig nCurOffsetSave = nCurOffset;
213 2 : nCurOffset += nBytesRequested - nExtraBytes;
214 : size_t nBytesRead =
215 : this->Read( ((char *) pBuffer) + nBytesRequested - nExtraBytes,
216 2 : 1, nExtraBytes );
217 2 : nCurOffset = nCurOffsetSave;
218 :
219 2 : if( nBytesRead < nExtraBytes )
220 0 : nReturnCount -= (nExtraBytes-nBytesRead) / nSize;
221 :
222 2 : nBytesRequested -= nExtraBytes;
223 : }
224 :
225 : /* -------------------------------------------------------------------- */
226 : /* Handle a constant region. */
227 : /* -------------------------------------------------------------------- */
228 9956 : if( aoRegions[iRegion].osFilename.size() == 0 )
229 : {
230 : memset( pBuffer, aoRegions[iRegion].byValue,
231 2 : (size_t) nBytesRequested );
232 : }
233 :
234 : /* -------------------------------------------------------------------- */
235 : /* Otherwise handle as a file. */
236 : /* -------------------------------------------------------------------- */
237 : else
238 : {
239 9954 : if( aoRegions[iRegion].fp == NULL )
240 : {
241 8 : if( !aoRegions[iRegion].bTriedOpen )
242 : {
243 : aoRegions[iRegion].fp =
244 8 : VSIFOpenL( aoRegions[iRegion].osFilename, "r" );
245 8 : if( aoRegions[iRegion].fp == NULL )
246 : {
247 : CPLDebug( "/vsisparse/", "Failed to open '%s'.",
248 0 : aoRegions[iRegion].osFilename.c_str() );
249 : }
250 8 : aoRegions[iRegion].bTriedOpen = TRUE;
251 : }
252 8 : if( aoRegions[iRegion].fp == NULL )
253 : {
254 0 : return 0;
255 : }
256 : }
257 :
258 9954 : if( VSIFSeekL( aoRegions[iRegion].fp,
259 : nCurOffset
260 : - aoRegions[iRegion].nDstOffset
261 : + aoRegions[iRegion].nSrcOffset,
262 : SEEK_SET ) != 0 )
263 0 : return 0;
264 :
265 : size_t nBytesRead = VSIFReadL( pBuffer, 1, (size_t) nBytesRequested,
266 9954 : aoRegions[iRegion].fp );
267 :
268 9954 : if( nBytesAvailable < nBytesRequested )
269 0 : nReturnCount = nBytesRead / nSize;
270 : }
271 :
272 9956 : nCurOffset += nReturnCount * nSize;
273 :
274 9956 : return nReturnCount;
275 : }
276 :
277 : /************************************************************************/
278 : /* Write() */
279 : /************************************************************************/
280 :
281 0 : size_t VSISparseFileHandle::Write( const void * pBuffer, size_t nSize, size_t nCount )
282 :
283 : {
284 0 : errno = EBADF;
285 0 : return 0;
286 : }
287 :
288 : /************************************************************************/
289 : /* Eof() */
290 : /************************************************************************/
291 :
292 0 : int VSISparseFileHandle::Eof()
293 :
294 : {
295 0 : return nCurOffset >= nOverallLength;
296 : }
297 :
298 : /************************************************************************/
299 : /* ==================================================================== */
300 : /* VSISparseFileFilesystemHandler */
301 : /* ==================================================================== */
302 : /************************************************************************/
303 :
304 : /************************************************************************/
305 : /* VSISparseFileFilesystemHandler() */
306 : /************************************************************************/
307 :
308 1341 : VSISparseFileFilesystemHandler::VSISparseFileFilesystemHandler()
309 :
310 : {
311 1341 : }
312 :
313 : /************************************************************************/
314 : /* ~VSISparseFileFilesystemHandler() */
315 : /************************************************************************/
316 :
317 1297 : VSISparseFileFilesystemHandler::~VSISparseFileFilesystemHandler()
318 :
319 : {
320 1297 : }
321 :
322 : /************************************************************************/
323 : /* Open() */
324 : /************************************************************************/
325 :
326 : VSIVirtualHandle *
327 16 : VSISparseFileFilesystemHandler::Open( const char *pszFilename,
328 : const char *pszAccess )
329 :
330 : {
331 16 : CPLAssert( EQUALN(pszFilename,"/vsisparse/", 11) );
332 :
333 16 : if( !EQUAL(pszAccess,"r") && !EQUAL(pszAccess,"rb") )
334 : {
335 0 : errno = EACCES;
336 0 : return NULL;
337 : }
338 :
339 16 : CPLString osSparseFilePath = pszFilename + 11;
340 :
341 : /* -------------------------------------------------------------------- */
342 : /* Does this file even exist? */
343 : /* -------------------------------------------------------------------- */
344 16 : VSILFILE *fp = VSIFOpenL( osSparseFilePath, "r" );
345 16 : if( fp == NULL )
346 10 : return NULL;
347 6 : VSIFCloseL( fp );
348 :
349 : /* -------------------------------------------------------------------- */
350 : /* Read the XML file. */
351 : /* -------------------------------------------------------------------- */
352 6 : CPLXMLNode *psXMLRoot = CPLParseXMLFile( osSparseFilePath );
353 :
354 6 : if( psXMLRoot == NULL )
355 0 : return NULL;
356 :
357 : /* -------------------------------------------------------------------- */
358 : /* Setup the file handle on this file. */
359 : /* -------------------------------------------------------------------- */
360 6 : VSISparseFileHandle *poHandle = new VSISparseFileHandle;
361 :
362 : /* -------------------------------------------------------------------- */
363 : /* Translate the desired fields out of the XML tree. */
364 : /* -------------------------------------------------------------------- */
365 : CPLXMLNode *psRegion;
366 :
367 30 : for( psRegion = psXMLRoot->psChild;
368 : psRegion != NULL;
369 : psRegion = psRegion->psNext )
370 : {
371 30 : if( psRegion->eType != CXT_Element )
372 0 : continue;
373 :
374 30 : if( !EQUAL(psRegion->pszValue,"SubfileRegion")
375 : && !EQUAL(psRegion->pszValue,"ConstantRegion") )
376 6 : continue;
377 :
378 24 : SFRegion oRegion;
379 :
380 24 : oRegion.osFilename = CPLGetXMLValue( psRegion, "Filename", "" );
381 24 : if( atoi(CPLGetXMLValue( psRegion, "Filename.relative", "0" )) != 0 )
382 : {
383 18 : CPLString osSFPath = CPLGetPath(osSparseFilePath);
384 : oRegion.osFilename = CPLFormFilename( osSFPath,
385 18 : oRegion.osFilename, NULL );
386 : }
387 :
388 : oRegion.nDstOffset =
389 : CPLScanUIntBig( CPLGetXMLValue(psRegion,"DestinationOffset","0" ),
390 24 : 32 );
391 :
392 : oRegion.nSrcOffset =
393 24 : CPLScanUIntBig( CPLGetXMLValue(psRegion,"SourceOffset","0" ), 32);
394 :
395 : oRegion.nLength =
396 24 : CPLScanUIntBig( CPLGetXMLValue(psRegion,"RegionLength","0" ), 32);
397 :
398 24 : oRegion.byValue = (GByte) atoi(CPLGetXMLValue(psRegion,"Value","0" ));
399 :
400 24 : poHandle->aoRegions.push_back( oRegion );
401 : }
402 :
403 : /* -------------------------------------------------------------------- */
404 : /* Get sparse file length, use maximum bound of regions if not */
405 : /* explicit in file. */
406 : /* -------------------------------------------------------------------- */
407 : poHandle->nOverallLength =
408 6 : CPLScanUIntBig( CPLGetXMLValue(psXMLRoot,"Length","0" ), 32);
409 6 : if( poHandle->nOverallLength == 0 )
410 : {
411 : unsigned int i;
412 :
413 0 : for( i = 0; i < poHandle->aoRegions.size(); i++ )
414 : {
415 : poHandle->nOverallLength = MAX(poHandle->nOverallLength,
416 : poHandle->aoRegions[i].nDstOffset
417 0 : + poHandle->aoRegions[i].nLength);
418 : }
419 : }
420 :
421 6 : CPLDestroyXMLNode( psXMLRoot );
422 :
423 6 : return poHandle;
424 : }
425 :
426 : /************************************************************************/
427 : /* Stat() */
428 : /************************************************************************/
429 :
430 4 : int VSISparseFileFilesystemHandler::Stat( const char * pszFilename,
431 : VSIStatBufL * psStatBuf,
432 : int nFlags )
433 :
434 : {
435 4 : VSIVirtualHandle *poFile = Open( pszFilename, "r" );
436 :
437 4 : memset( psStatBuf, 0, sizeof(VSIStatBufL) );
438 :
439 4 : if( poFile == NULL )
440 2 : return -1;
441 :
442 2 : poFile->Seek( 0, SEEK_END );
443 2 : size_t nLength = (size_t) poFile->Tell();
444 2 : delete poFile;
445 :
446 : int nResult = VSIStatExL( pszFilename + strlen("/vsisparse/"),
447 2 : psStatBuf, nFlags );
448 :
449 2 : psStatBuf->st_size = nLength;
450 :
451 2 : return nResult;
452 : }
453 :
454 : /************************************************************************/
455 : /* Unlink() */
456 : /************************************************************************/
457 :
458 0 : int VSISparseFileFilesystemHandler::Unlink( const char * pszFilename )
459 :
460 : {
461 0 : errno = EACCES;
462 0 : return -1;
463 : }
464 :
465 : /************************************************************************/
466 : /* Mkdir() */
467 : /************************************************************************/
468 :
469 0 : int VSISparseFileFilesystemHandler::Mkdir( const char * pszPathname,
470 : long nMode )
471 :
472 : {
473 0 : errno = EACCES;
474 0 : return -1;
475 : }
476 :
477 : /************************************************************************/
478 : /* Rmdir() */
479 : /************************************************************************/
480 :
481 0 : int VSISparseFileFilesystemHandler::Rmdir( const char * pszPathname )
482 :
483 : {
484 0 : errno = EACCES;
485 0 : return -1;
486 : }
487 :
488 : /************************************************************************/
489 : /* ReadDir() */
490 : /************************************************************************/
491 :
492 2 : char **VSISparseFileFilesystemHandler::ReadDir( const char *pszPath )
493 :
494 : {
495 2 : errno = EACCES;
496 2 : return NULL;
497 : }
498 :
499 : /************************************************************************/
500 : /* VSIInstallSparseFileFilesystemHandler() */
501 : /************************************************************************/
502 :
503 : /**
504 : * Install /vsisparse/ virtual file handler.
505 : *
506 : * The sparse virtual file handler allows a virtual file to be composed
507 : * from chunks of data in other files, potentially with large spaces in
508 : * the virtual file set to a constant value. This can make it possible to
509 : * test some sorts of operations on what seems to be a large file with
510 : * image data set to a constant value. It is also helpful when wanting to
511 : * add test files to the test suite that are too large, but for which most
512 : * of the data can be ignored. It could, in theory, also be used to
513 : * treat several files on different file systems as one large virtual file.
514 : *
515 : * The file referenced by /vsisparse/ should be an XML control file
516 : * formatted something like:
517 : *
518 : *
519 : \verbatim
520 : <VSISparseFile>
521 : <Length>87629264</Length>
522 : <SubfileRegion> Stuff at start of file.
523 : <Filename relative="1">251_head.dat</Filename>
524 : <DestinationOffset>0</DestinationOffset>
525 : <SourceOffset>0</SourceOffset>
526 : <RegionLength>2768</RegionLength>
527 : </SubfileRegion>
528 :
529 : <SubfileRegion> RasterDMS node.
530 : <Filename relative="1">251_rasterdms.dat</Filename>
531 : <DestinationOffset>87313104</DestinationOffset>
532 : <SourceOffset>0</SourceOffset>
533 : <RegionLength>160</RegionLength>
534 : </SubfileRegion>
535 :
536 : <SubfileRegion> Stuff at end of file.
537 : <Filename relative="1">251_tail.dat</Filename>
538 : <DestinationOffset>87611924</DestinationOffset>
539 : <SourceOffset>0</SourceOffset>
540 : <RegionLength>17340</RegionLength>
541 : </SubfileRegion>
542 :
543 : <ConstantRegion> Default for the rest of the file.
544 : <DestinationOffset>0</DestinationOffset>
545 : <RegionLength>87629264</RegionLength>
546 : <Value>0</Value>
547 : </ConstantRegion>
548 : </VSISparseFile>
549 : \endverbatim
550 : *
551 : * Hopefully the values and semantics are fairly obvious.
552 : *
553 : * This driver is installed by default.
554 : */
555 :
556 1341 : void VSIInstallSparseFileHandler()
557 : {
558 : VSIFileManager::InstallHandler( "/vsisparse/",
559 1341 : new VSISparseFileFilesystemHandler );
560 1341 : }
561 :
|
