1 : /******************************************************************************
2 : * $Id: gdalrasterblock.cpp 22669 2011-07-07 22:37:44Z rouault $
3 : *
4 : * Project: GDAL Core
5 : * Purpose: Implementation of GDALRasterBlock class and related global
6 : * raster block cache management.
7 : * Author: Frank Warmerdam, warmerdam@pobox.com
8 : *
9 : **********************************************************************
10 : * Copyright (c) 1998, Frank Warmerdam <warmerdam@pobox.com>
11 : *
12 : * Permission is hereby granted, free of charge, to any person obtaining a
13 : * copy of this software and associated documentation files (the "Software"),
14 : * to deal in the Software without restriction, including without limitation
15 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 : * and/or sell copies of the Software, and to permit persons to whom the
17 : * Software is furnished to do so, subject to the following conditions:
18 : *
19 : * The above copyright notice and this permission notice shall be included
20 : * in all copies or substantial portions of the Software.
21 : *
22 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 : * DEALINGS IN THE SOFTWARE.
29 : ****************************************************************************/
30 :
31 : #include "gdal_priv.h"
32 : #include "cpl_multiproc.h"
33 :
34 : CPL_CVSID("$Id: gdalrasterblock.cpp 22669 2011-07-07 22:37:44Z rouault $");
35 :
36 : static int bCacheMaxInitialized = FALSE;
37 : static GIntBig nCacheMax = 40 * 1024*1024;
38 : static volatile GIntBig nCacheUsed = 0;
39 :
40 : static volatile GDALRasterBlock *poOldest = NULL; /* tail */
41 : static volatile GDALRasterBlock *poNewest = NULL; /* head */
42 :
43 : static void *hRBMutex = NULL;
44 :
45 :
46 : /************************************************************************/
47 : /* GDALSetCacheMax() */
48 : /************************************************************************/
49 :
50 : /**
51 : * \brief Set maximum cache memory.
52 : *
53 : * This function sets the maximum amount of memory that GDAL is permitted
54 : * to use for GDALRasterBlock caching. The unit of the value is bytes.
55 : *
56 : * The maximum value is 2GB, due to the use of a signed 32 bit integer.
57 : * Use GDALSetCacheMax64() to be able to set a higher value.
58 : *
59 : * @param nNewSizeInBytes the maximum number of bytes for caching.
60 : */
61 :
62 16 : void CPL_STDCALL GDALSetCacheMax( int nNewSizeInBytes )
63 :
64 : {
65 16 : GDALSetCacheMax64(nNewSizeInBytes);
66 16 : }
67 :
68 :
69 : /************************************************************************/
70 : /* GDALSetCacheMax64() */
71 : /************************************************************************/
72 :
73 : /**
74 : * \brief Set maximum cache memory.
75 : *
76 : * This function sets the maximum amount of memory that GDAL is permitted
77 : * to use for GDALRasterBlock caching. The unit of the value is bytes.
78 : *
79 : * Note: On 32 bit platforms, the maximum amount of memory that can be addressed
80 : * by a process might be 2 GB or 3 GB, depending on the operating system
81 : * capabilities. This function will not make any attempt to check the
82 : * consistency of the passed value with the effective capabilities of the OS.
83 : *
84 : * @param nNewSizeInBytes the maximum number of bytes for caching.
85 : *
86 : * @since GDAL 1.8.0
87 : */
88 :
89 40 : void CPL_STDCALL GDALSetCacheMax64( GIntBig nNewSizeInBytes )
90 :
91 : {
92 40 : nCacheMax = nNewSizeInBytes;
93 :
94 : /* -------------------------------------------------------------------- */
95 : /* Flush blocks till we are under the new limit or till we */
96 : /* can't seem to flush anymore. */
97 : /* -------------------------------------------------------------------- */
98 82 : while( nCacheUsed > nCacheMax )
99 : {
100 2 : GIntBig nOldCacheUsed = nCacheUsed;
101 :
102 2 : GDALFlushCacheBlock();
103 :
104 2 : if( nCacheUsed == nOldCacheUsed )
105 0 : break;
106 : }
107 40 : }
108 :
109 : /************************************************************************/
110 : /* GDALGetCacheMax() */
111 : /************************************************************************/
112 :
113 : /**
114 : * \brief Get maximum cache memory.
115 : *
116 : * Gets the maximum amount of memory available to the GDALRasterBlock
117 : * caching system for caching GDAL read/write imagery.
118 : *
119 : * The first type this function is called, it will read the GDAL_CACHEMAX
120 : * configuation option to initialize the maximum cache memory.
121 : *
122 : * This function cannot return a value higher than 2 GB. Use
123 : * GDALGetCacheMax64() to get a non-truncated value.
124 : *
125 : * @return maximum in bytes.
126 : */
127 :
128 0 : int CPL_STDCALL GDALGetCacheMax()
129 : {
130 0 : GIntBig nRes = GDALGetCacheMax64();
131 0 : if (nRes > INT_MAX)
132 : {
133 : static int bHasWarned = FALSE;
134 0 : if (!bHasWarned)
135 : {
136 : CPLError(CE_Warning, CPLE_AppDefined,
137 : "Cache max value doesn't fit on a 32 bit integer. "
138 0 : "Call GDALGetCacheMax64() instead");
139 0 : bHasWarned = TRUE;
140 : }
141 0 : nRes = INT_MAX;
142 : }
143 0 : return (int)nRes;
144 : }
145 :
146 : /************************************************************************/
147 : /* GDALGetCacheMax64() */
148 : /************************************************************************/
149 :
150 : /**
151 : * \brief Get maximum cache memory.
152 : *
153 : * Gets the maximum amount of memory available to the GDALRasterBlock
154 : * caching system for caching GDAL read/write imagery.
155 : *
156 : * The first type this function is called, it will read the GDAL_CACHEMAX
157 : * configuation option to initialize the maximum cache memory.
158 : *
159 : * @return maximum in bytes.
160 : *
161 : * @since GDAL 1.8.0
162 : */
163 :
164 1148234 : GIntBig CPL_STDCALL GDALGetCacheMax64()
165 : {
166 1148234 : if( !bCacheMaxInitialized )
167 : {
168 778 : const char* pszCacheMax = CPLGetConfigOption("GDAL_CACHEMAX",NULL);
169 778 : bCacheMaxInitialized = TRUE;
170 778 : if( pszCacheMax != NULL )
171 : {
172 4 : GIntBig nNewCacheMax = (GIntBig)CPLScanUIntBig(pszCacheMax, strlen(pszCacheMax));
173 4 : if( nNewCacheMax < 100000 )
174 : {
175 4 : if (nNewCacheMax < 0)
176 : {
177 : CPLError(CE_Failure, CPLE_NotSupported,
178 0 : "Invalid value for GDAL_CACHEMAX. Using default value.");
179 0 : return nCacheMax;
180 : }
181 4 : nNewCacheMax *= 1024 * 1024;
182 : }
183 4 : nCacheMax = nNewCacheMax;
184 : }
185 : }
186 :
187 1148234 : return nCacheMax;
188 : }
189 :
190 : /************************************************************************/
191 : /* GDALGetCacheUsed() */
192 : /************************************************************************/
193 :
194 : /**
195 : * \brief Get cache memory used.
196 : *
197 : * @return the number of bytes of memory currently in use by the
198 : * GDALRasterBlock memory caching.
199 : */
200 :
201 0 : int CPL_STDCALL GDALGetCacheUsed()
202 : {
203 0 : if (nCacheUsed > INT_MAX)
204 : {
205 : static int bHasWarned = FALSE;
206 0 : if (!bHasWarned)
207 : {
208 : CPLError(CE_Warning, CPLE_AppDefined,
209 : "Cache used value doesn't fit on a 32 bit integer. "
210 0 : "Call GDALGetCacheUsed64() instead");
211 0 : bHasWarned = TRUE;
212 : }
213 0 : return INT_MAX;
214 : }
215 0 : return (int)nCacheUsed;
216 : }
217 :
218 : /************************************************************************/
219 : /* GDALGetCacheUsed64() */
220 : /************************************************************************/
221 :
222 : /**
223 : * \brief Get cache memory used.
224 : *
225 : * @return the number of bytes of memory currently in use by the
226 : * GDALRasterBlock memory caching.
227 : *
228 : * @since GDAL 1.8.0
229 : */
230 :
231 4 : GIntBig CPL_STDCALL GDALGetCacheUsed64()
232 : {
233 4 : return nCacheUsed;
234 : }
235 :
236 : /************************************************************************/
237 : /* GDALFlushCacheBlock() */
238 : /* */
239 : /* The workhorse of cache management! */
240 : /************************************************************************/
241 :
242 : /**
243 : * \brief Try to flush one cached raster block
244 : *
245 : * This function will search the first unlocked raster block and will
246 : * flush it to release the associated memory.
247 : *
248 : * @return TRUE if one block was flushed, FALSE if there are no cached blocks
249 : * or if they are currently locked.
250 : */
251 15936 : int CPL_STDCALL GDALFlushCacheBlock()
252 :
253 : {
254 15936 : return GDALRasterBlock::FlushCacheBlock();
255 : }
256 :
257 : /************************************************************************/
258 : /* ==================================================================== */
259 : /* GDALRasterBlock */
260 : /* ==================================================================== */
261 : /************************************************************************/
262 :
263 : /**
264 : * \class GDALRasterBlock "gdal_priv.h"
265 : *
266 : * GDALRasterBlock objects hold one block of raster data for one band
267 : * that is currently stored in the GDAL raster cache. The cache holds
268 : * some blocks of raster data for zero or more GDALRasterBand objects
269 : * across zero or more GDALDataset objects in a global raster cache with
270 : * a least recently used (LRU) list and an upper cache limit (see
271 : * GDALSetCacheMax()) under which the cache size is normally kept.
272 : *
273 : * Some blocks in the cache may be modified relative to the state on disk
274 : * (they are marked "Dirty") and must be flushed to disk before they can
275 : * be discarded. Other (Clean) blocks may just be discarded if their memory
276 : * needs to be recovered.
277 : *
278 : * In normal situations applications do not interact directly with the
279 : * GDALRasterBlock - instead it it utilized by the RasterIO() interfaces
280 : * to implement caching.
281 : *
282 : * Some driver classes are implemented in a fashion that completely avoids
283 : * use of the GDAL raster cache (and GDALRasterBlock) though this is not very
284 : * common.
285 : */
286 :
287 : /************************************************************************/
288 : /* FlushCacheBlock() */
289 : /* */
290 : /* Note, if we have alot of blocks locked for a long time, this */
291 : /* method is going to get slow because it will have to traverse */
292 : /* the linked list a long ways looking for a flushing */
293 : /* candidate. It might help to re-touch locked blocks to push */
294 : /* them to the top of the list. */
295 : /************************************************************************/
296 :
297 : /**
298 : * \brief Attempt to flush at least one block from the cache.
299 : *
300 : * This static method is normally used to recover memory when a request
301 : * for a new cache block would put cache memory use over the established
302 : * limit.
303 : *
304 : * C++ analog to the C function GDALFlushCacheBlock().
305 : *
306 : * @return TRUE if successful or FALSE if no flushable block is found.
307 : */
308 :
309 15936 : int GDALRasterBlock::FlushCacheBlock()
310 :
311 : {
312 : int nXOff, nYOff;
313 : GDALRasterBand *poBand;
314 :
315 : {
316 15936 : CPLMutexHolderD( &hRBMutex );
317 15936 : GDALRasterBlock *poTarget = (GDALRasterBlock *) poOldest;
318 :
319 31872 : while( poTarget != NULL && poTarget->GetLockCount() > 0 )
320 0 : poTarget = poTarget->poPrevious;
321 :
322 15936 : if( poTarget == NULL )
323 950 : return FALSE;
324 :
325 14986 : poTarget->Detach();
326 :
327 14986 : nXOff = poTarget->GetXOff();
328 14986 : nYOff = poTarget->GetYOff();
329 14986 : poBand = poTarget->GetBand();
330 : }
331 :
332 14986 : CPLErr eErr = poBand->FlushBlock( nXOff, nYOff );
333 14986 : if (eErr != CE_None)
334 : {
335 : /* Save the error for later reporting */
336 0 : poBand->SetFlushBlockErr(eErr);
337 : }
338 :
339 14986 : return TRUE;
340 : }
341 :
342 : /************************************************************************/
343 : /* GDALRasterBlock() */
344 : /************************************************************************/
345 :
346 : /**
347 : * @brief GDALRasterBlock Constructor
348 : *
349 : * Normally only called from GDALRasterBand::GetLockedBlockRef().
350 : *
351 : * @param poBandIn the raster band used as source of raster block
352 : * being constructed.
353 : *
354 : * @param nXOffIn the horizontal block offset, with zero indicating
355 : * the left most block, 1 the next block and so forth.
356 : *
357 : * @param nYOffIn the vertical block offset, with zero indicating
358 : * the top most block, 1 the next block and so forth.
359 : */
360 :
361 817382 : GDALRasterBlock::GDALRasterBlock( GDALRasterBand *poBandIn,
362 817382 : int nXOffIn, int nYOffIn )
363 :
364 : {
365 817382 : CPLAssert( NULL != poBandIn );
366 :
367 817382 : poBand = poBandIn;
368 :
369 817382 : poBand->GetBlockSize( &nXSize, &nYSize );
370 817382 : eType = poBand->GetRasterDataType();
371 817382 : pData = NULL;
372 817382 : bDirty = FALSE;
373 817382 : nLockCount = 0;
374 :
375 817382 : poNext = poPrevious = NULL;
376 :
377 817382 : nXOff = nXOffIn;
378 817382 : nYOff = nYOffIn;
379 817382 : }
380 :
381 : /************************************************************************/
382 : /* ~GDALRasterBlock() */
383 : /************************************************************************/
384 :
385 : /**
386 : * Block destructor.
387 : *
388 : * Normally called from GDALRasterBand::FlushBlock().
389 : */
390 :
391 817382 : GDALRasterBlock::~GDALRasterBlock()
392 :
393 : {
394 817382 : Detach();
395 :
396 817382 : if( pData != NULL )
397 : {
398 : int nSizeInBytes;
399 :
400 817382 : VSIFree( pData );
401 :
402 817382 : nSizeInBytes = (nXSize * nYSize * GDALGetDataTypeSize(eType)+7)/8;
403 :
404 : {
405 817382 : CPLMutexHolderD( &hRBMutex );
406 817382 : nCacheUsed -= nSizeInBytes;
407 : }
408 : }
409 :
410 817382 : CPLAssert( nLockCount == 0 );
411 :
412 : #ifdef ENABLE_DEBUG
413 : Verify();
414 : #endif
415 817382 : }
416 :
417 : /************************************************************************/
418 : /* Detach() */
419 : /************************************************************************/
420 :
421 : /**
422 : * Remove block from cache.
423 : *
424 : * This method removes the current block from the linked list used to keep
425 : * track of all cached blocks in order of age. It does not affect whether
426 : * the block is referenced by a GDALRasterBand nor does it destroy or flush
427 : * the block.
428 : */
429 :
430 1649750 : void GDALRasterBlock::Detach()
431 :
432 : {
433 1649750 : CPLMutexHolderD( &hRBMutex );
434 :
435 1649750 : if( poOldest == this )
436 20540 : poOldest = poPrevious;
437 :
438 1649750 : if( poNewest == this )
439 : {
440 818318 : poNewest = poNext;
441 : }
442 :
443 1649750 : if( poPrevious != NULL )
444 14050 : poPrevious->poNext = poNext;
445 :
446 1649750 : if( poNext != NULL )
447 811828 : poNext->poPrevious = poPrevious;
448 :
449 1649750 : poPrevious = NULL;
450 1649750 : poNext = NULL;
451 1649750 : }
452 :
453 : /************************************************************************/
454 : /* Verify() */
455 : /************************************************************************/
456 :
457 : /**
458 : * Confirms (via assertions) that the block cache linked list is in a
459 : * consistent state.
460 : */
461 :
462 0 : void GDALRasterBlock::Verify()
463 :
464 : {
465 0 : CPLMutexHolderD( &hRBMutex );
466 :
467 : CPLAssert( (poNewest == NULL && poOldest == NULL)
468 0 : || (poNewest != NULL && poOldest != NULL) );
469 :
470 0 : if( poNewest != NULL )
471 : {
472 0 : CPLAssert( poNewest->poPrevious == NULL );
473 0 : CPLAssert( poOldest->poNext == NULL );
474 :
475 0 : for( GDALRasterBlock *poBlock = (GDALRasterBlock *) poNewest;
476 : poBlock != NULL;
477 : poBlock = poBlock->poNext )
478 : {
479 0 : if( poBlock->poPrevious )
480 : {
481 0 : CPLAssert( poBlock->poPrevious->poNext == poBlock );
482 : }
483 :
484 0 : if( poBlock->poNext )
485 : {
486 0 : CPLAssert( poBlock->poNext->poPrevious == poBlock );
487 : }
488 : }
489 0 : }
490 0 : }
491 :
492 : /************************************************************************/
493 : /* Write() */
494 : /************************************************************************/
495 :
496 : /**
497 : * Force writing of the current block, if dirty.
498 : *
499 : * The block is written using GDALRasterBand::IWriteBlock() on it's
500 : * corresponding band object. Even if the write fails the block will
501 : * be marked clean.
502 : *
503 : * @return CE_None otherwise the error returned by IWriteBlock().
504 : */
505 :
506 108682 : CPLErr GDALRasterBlock::Write()
507 :
508 : {
509 108682 : if( !GetDirty() )
510 0 : return CE_None;
511 :
512 108682 : if( poBand == NULL )
513 0 : return CE_Failure;
514 :
515 108682 : MarkClean();
516 :
517 108682 : if (poBand->eFlushBlockErr == CE_None)
518 108682 : return poBand->IWriteBlock( nXOff, nYOff, pData );
519 : else
520 0 : return poBand->eFlushBlockErr;
521 : }
522 :
523 : /************************************************************************/
524 : /* Touch() */
525 : /************************************************************************/
526 :
527 : /**
528 : * Push block to top of LRU (least-recently used) list.
529 : *
530 : * This method is normally called when a block is used to keep track
531 : * that it has been recently used.
532 : */
533 :
534 10612290 : void GDALRasterBlock::Touch()
535 :
536 : {
537 10612290 : CPLMutexHolderD( &hRBMutex );
538 :
539 10612290 : if( poNewest == this )
540 : return;
541 :
542 8765360 : if( poOldest == this )
543 2695132 : poOldest = this->poPrevious;
544 :
545 8765360 : if( poPrevious != NULL )
546 7932992 : poPrevious->poNext = poNext;
547 :
548 8765360 : if( poNext != NULL )
549 5237860 : poNext->poPrevious = poPrevious;
550 :
551 8765360 : poPrevious = NULL;
552 8765360 : poNext = (GDALRasterBlock *) poNewest;
553 :
554 8765360 : if( poNewest != NULL )
555 : {
556 8758870 : CPLAssert( poNewest->poPrevious == NULL );
557 8758870 : poNewest->poPrevious = this;
558 : }
559 8765360 : poNewest = this;
560 :
561 8765360 : if( poOldest == NULL )
562 : {
563 6490 : CPLAssert( poPrevious == NULL && poNext == NULL );
564 6490 : poOldest = this;
565 0 : }
566 : #ifdef ENABLE_DEBUG
567 : Verify();
568 : #endif
569 : }
570 :
571 : /************************************************************************/
572 : /* Internalize() */
573 : /************************************************************************/
574 :
575 : /**
576 : * Allocate memory for block.
577 : *
578 : * This method allocates memory for the block, and attempts to flush other
579 : * blocks, if necessary, to bring the total cache size back within the limits.
580 : * The newly allocated block is touched and will be considered most recently
581 : * used in the LRU list.
582 : *
583 : * @return CE_None on success or CE_Failure if memory allocation fails.
584 : */
585 :
586 817382 : CPLErr GDALRasterBlock::Internalize()
587 :
588 : {
589 817382 : CPLMutexHolderD( &hRBMutex );
590 : void *pNewData;
591 : int nSizeInBytes;
592 817382 : GIntBig nCurCacheMax = GDALGetCacheMax64();
593 :
594 : /* No risk of overflow as it is checked in GDALRasterBand::InitBlockInfo() */
595 817382 : nSizeInBytes = nXSize * nYSize * (GDALGetDataTypeSize(eType) / 8);
596 :
597 817382 : pNewData = VSIMalloc( nSizeInBytes );
598 817382 : if( pNewData == NULL )
599 : {
600 : CPLError( CE_Failure, CPLE_OutOfMemory,
601 : "GDALRasterBlock::Internalize : Out of memory allocating %d bytes.",
602 0 : nSizeInBytes);
603 0 : return( CE_Failure );
604 : }
605 :
606 817382 : if( pData != NULL )
607 0 : memcpy( pNewData, pData, nSizeInBytes );
608 :
609 817382 : pData = pNewData;
610 :
611 : /* -------------------------------------------------------------------- */
612 : /* Flush old blocks if we are nearing our memory limit. */
613 : /* -------------------------------------------------------------------- */
614 817382 : AddLock(); /* don't flush this block! */
615 :
616 817382 : nCacheUsed += nSizeInBytes;
617 1649748 : while( nCacheUsed > nCurCacheMax )
618 : {
619 15934 : GIntBig nOldCacheUsed = nCacheUsed;
620 :
621 15934 : GDALFlushCacheBlock();
622 :
623 15934 : if( nCacheUsed == nOldCacheUsed )
624 950 : break;
625 : }
626 :
627 : /* -------------------------------------------------------------------- */
628 : /* Add this block to the list. */
629 : /* -------------------------------------------------------------------- */
630 817382 : Touch();
631 817382 : DropLock();
632 :
633 817382 : return( CE_None );
634 : }
635 :
636 : /************************************************************************/
637 : /* MarkDirty() */
638 : /************************************************************************/
639 :
640 : /**
641 : * Mark the block as modified.
642 : *
643 : * A dirty block is one that has been modified and will need to be written
644 : * to disk before it can be flushed.
645 : */
646 :
647 3177134 : void GDALRasterBlock::MarkDirty()
648 :
649 : {
650 3177134 : bDirty = TRUE;
651 3177134 : }
652 :
653 :
654 : /************************************************************************/
655 : /* MarkClean() */
656 : /************************************************************************/
657 :
658 : /**
659 : * Mark the block as unmodified.
660 : *
661 : * A dirty block is one that has been modified and will need to be written
662 : * to disk before it can be flushed.
663 : */
664 :
665 117240 : void GDALRasterBlock::MarkClean()
666 :
667 : {
668 117240 : bDirty = FALSE;
669 117240 : }
670 :
671 : /************************************************************************/
672 : /* SafeLockBlock() */
673 : /************************************************************************/
674 :
675 : /**
676 : * \brief Safely lock block.
677 : *
678 : * This method locks a GDALRasterBlock (and touches it) in a thread-safe
679 : * manner. The global block cache mutex is held while locking the block,
680 : * in order to avoid race conditions with other threads that might be
681 : * trying to expire the block at the same time. The block pointer may be
682 : * safely NULL, in which case this method does nothing.
683 : *
684 : * @param ppBlock Pointer to the block pointer to try and lock/touch.
685 : */
686 :
687 9795068 : int GDALRasterBlock::SafeLockBlock( GDALRasterBlock ** ppBlock )
688 :
689 : {
690 9795068 : CPLAssert( NULL != ppBlock );
691 :
692 9795068 : CPLMutexHolderD( &hRBMutex );
693 :
694 9795068 : if( *ppBlock != NULL )
695 : {
696 8977526 : (*ppBlock)->AddLock();
697 8977526 : (*ppBlock)->Touch();
698 :
699 8977526 : return TRUE;
700 : }
701 : else
702 817542 : return FALSE;
703 : }
|
