1 : /******************************************************************************
2 : * $Id: gdalrasterblock.cpp 24412 2012-05-14 17:55:36Z rouault $
3 : *
4 : * Project: GDAL Core
5 : * Purpose: Implementation of GDALRasterBlock class and related global
6 : * raster block cache management.
7 : * Author: Frank Warmerdam, warmerdam@pobox.com
8 : *
9 : **********************************************************************
10 : * Copyright (c) 1998, Frank Warmerdam <warmerdam@pobox.com>
11 : *
12 : * Permission is hereby granted, free of charge, to any person obtaining a
13 : * copy of this software and associated documentation files (the "Software"),
14 : * to deal in the Software without restriction, including without limitation
15 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 : * and/or sell copies of the Software, and to permit persons to whom the
17 : * Software is furnished to do so, subject to the following conditions:
18 : *
19 : * The above copyright notice and this permission notice shall be included
20 : * in all copies or substantial portions of the Software.
21 : *
22 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 : * DEALINGS IN THE SOFTWARE.
29 : ****************************************************************************/
30 :
31 : #include "gdal_priv.h"
32 : #include "cpl_multiproc.h"
33 :
34 : CPL_CVSID("$Id: gdalrasterblock.cpp 24412 2012-05-14 17:55:36Z rouault $");
35 :
36 : static int bCacheMaxInitialized = FALSE;
37 : static GIntBig nCacheMax = 40 * 1024*1024;
38 : static volatile GIntBig nCacheUsed = 0;
39 :
40 : static volatile GDALRasterBlock *poOldest = NULL; /* tail */
41 : static volatile GDALRasterBlock *poNewest = NULL; /* head */
42 :
43 : static void *hRBMutex = NULL;
44 :
45 :
46 : /************************************************************************/
47 : /* GDALSetCacheMax() */
48 : /************************************************************************/
49 :
50 : /**
51 : * \brief Set maximum cache memory.
52 : *
53 : * This function sets the maximum amount of memory that GDAL is permitted
54 : * to use for GDALRasterBlock caching. The unit of the value is bytes.
55 : *
56 : * The maximum value is 2GB, due to the use of a signed 32 bit integer.
57 : * Use GDALSetCacheMax64() to be able to set a higher value.
58 : *
59 : * @param nNewSizeInBytes the maximum number of bytes for caching.
60 : */
61 :
62 8 : void CPL_STDCALL GDALSetCacheMax( int nNewSizeInBytes )
63 :
64 : {
65 8 : GDALSetCacheMax64(nNewSizeInBytes);
66 8 : }
67 :
68 :
69 : /************************************************************************/
70 : /* GDALSetCacheMax64() */
71 : /************************************************************************/
72 :
73 : /**
74 : * \brief Set maximum cache memory.
75 : *
76 : * This function sets the maximum amount of memory that GDAL is permitted
77 : * to use for GDALRasterBlock caching. The unit of the value is bytes.
78 : *
79 : * Note: On 32 bit platforms, the maximum amount of memory that can be addressed
80 : * by a process might be 2 GB or 3 GB, depending on the operating system
81 : * capabilities. This function will not make any attempt to check the
82 : * consistency of the passed value with the effective capabilities of the OS.
83 : *
84 : * @param nNewSizeInBytes the maximum number of bytes for caching.
85 : *
86 : * @since GDAL 1.8.0
87 : */
88 :
89 20 : void CPL_STDCALL GDALSetCacheMax64( GIntBig nNewSizeInBytes )
90 :
91 : {
92 20 : bCacheMaxInitialized = TRUE;
93 20 : nCacheMax = nNewSizeInBytes;
94 :
95 : /* -------------------------------------------------------------------- */
96 : /* Flush blocks till we are under the new limit or till we */
97 : /* can't seem to flush anymore. */
98 : /* -------------------------------------------------------------------- */
99 44 : while( nCacheUsed > nCacheMax )
100 : {
101 4 : GIntBig nOldCacheUsed = nCacheUsed;
102 :
103 4 : GDALFlushCacheBlock();
104 :
105 4 : if( nCacheUsed == nOldCacheUsed )
106 0 : break;
107 : }
108 20 : }
109 :
110 : /************************************************************************/
111 : /* GDALGetCacheMax() */
112 : /************************************************************************/
113 :
114 : /**
115 : * \brief Get maximum cache memory.
116 : *
117 : * Gets the maximum amount of memory available to the GDALRasterBlock
118 : * caching system for caching GDAL read/write imagery.
119 : *
120 : * The first type this function is called, it will read the GDAL_CACHEMAX
121 : * configuation option to initialize the maximum cache memory.
122 : *
123 : * This function cannot return a value higher than 2 GB. Use
124 : * GDALGetCacheMax64() to get a non-truncated value.
125 : *
126 : * @return maximum in bytes.
127 : */
128 :
129 0 : int CPL_STDCALL GDALGetCacheMax()
130 : {
131 0 : GIntBig nRes = GDALGetCacheMax64();
132 0 : if (nRes > INT_MAX)
133 : {
134 : static int bHasWarned = FALSE;
135 0 : if (!bHasWarned)
136 : {
137 : CPLError(CE_Warning, CPLE_AppDefined,
138 : "Cache max value doesn't fit on a 32 bit integer. "
139 0 : "Call GDALGetCacheMax64() instead");
140 0 : bHasWarned = TRUE;
141 : }
142 0 : nRes = INT_MAX;
143 : }
144 0 : return (int)nRes;
145 : }
146 :
147 : /************************************************************************/
148 : /* GDALGetCacheMax64() */
149 : /************************************************************************/
150 :
151 : /**
152 : * \brief Get maximum cache memory.
153 : *
154 : * Gets the maximum amount of memory available to the GDALRasterBlock
155 : * caching system for caching GDAL read/write imagery.
156 : *
157 : * The first type this function is called, it will read the GDAL_CACHEMAX
158 : * configuation option to initialize the maximum cache memory.
159 : *
160 : * @return maximum in bytes.
161 : *
162 : * @since GDAL 1.8.0
163 : */
164 :
165 611522 : GIntBig CPL_STDCALL GDALGetCacheMax64()
166 : {
167 611522 : if( !bCacheMaxInitialized )
168 : {
169 391 : const char* pszCacheMax = CPLGetConfigOption("GDAL_CACHEMAX",NULL);
170 391 : bCacheMaxInitialized = TRUE;
171 391 : if( pszCacheMax != NULL )
172 : {
173 2 : GIntBig nNewCacheMax = (GIntBig)CPLScanUIntBig(pszCacheMax, strlen(pszCacheMax));
174 2 : if( nNewCacheMax < 100000 )
175 : {
176 2 : if (nNewCacheMax < 0)
177 : {
178 : CPLError(CE_Failure, CPLE_NotSupported,
179 0 : "Invalid value for GDAL_CACHEMAX. Using default value.");
180 0 : return nCacheMax;
181 : }
182 2 : nNewCacheMax *= 1024 * 1024;
183 : }
184 2 : nCacheMax = nNewCacheMax;
185 : }
186 : }
187 :
188 611522 : return nCacheMax;
189 : }
190 :
191 : /************************************************************************/
192 : /* GDALGetCacheUsed() */
193 : /************************************************************************/
194 :
195 : /**
196 : * \brief Get cache memory used.
197 : *
198 : * @return the number of bytes of memory currently in use by the
199 : * GDALRasterBlock memory caching.
200 : */
201 :
202 0 : int CPL_STDCALL GDALGetCacheUsed()
203 : {
204 0 : if (nCacheUsed > INT_MAX)
205 : {
206 : static int bHasWarned = FALSE;
207 0 : if (!bHasWarned)
208 : {
209 : CPLError(CE_Warning, CPLE_AppDefined,
210 : "Cache used value doesn't fit on a 32 bit integer. "
211 0 : "Call GDALGetCacheUsed64() instead");
212 0 : bHasWarned = TRUE;
213 : }
214 0 : return INT_MAX;
215 : }
216 0 : return (int)nCacheUsed;
217 : }
218 :
219 : /************************************************************************/
220 : /* GDALGetCacheUsed64() */
221 : /************************************************************************/
222 :
223 : /**
224 : * \brief Get cache memory used.
225 : *
226 : * @return the number of bytes of memory currently in use by the
227 : * GDALRasterBlock memory caching.
228 : *
229 : * @since GDAL 1.8.0
230 : */
231 :
232 2 : GIntBig CPL_STDCALL GDALGetCacheUsed64()
233 : {
234 2 : return nCacheUsed;
235 : }
236 :
237 : /************************************************************************/
238 : /* GDALFlushCacheBlock() */
239 : /* */
240 : /* The workhorse of cache management! */
241 : /************************************************************************/
242 :
243 : /**
244 : * \brief Try to flush one cached raster block
245 : *
246 : * This function will search the first unlocked raster block and will
247 : * flush it to release the associated memory.
248 : *
249 : * @return TRUE if one block was flushed, FALSE if there are no cached blocks
250 : * or if they are currently locked.
251 : */
252 7975 : int CPL_STDCALL GDALFlushCacheBlock()
253 :
254 : {
255 7975 : return GDALRasterBlock::FlushCacheBlock();
256 : }
257 :
258 : /************************************************************************/
259 : /* ==================================================================== */
260 : /* GDALRasterBlock */
261 : /* ==================================================================== */
262 : /************************************************************************/
263 :
264 : /**
265 : * \class GDALRasterBlock "gdal_priv.h"
266 : *
267 : * GDALRasterBlock objects hold one block of raster data for one band
268 : * that is currently stored in the GDAL raster cache. The cache holds
269 : * some blocks of raster data for zero or more GDALRasterBand objects
270 : * across zero or more GDALDataset objects in a global raster cache with
271 : * a least recently used (LRU) list and an upper cache limit (see
272 : * GDALSetCacheMax()) under which the cache size is normally kept.
273 : *
274 : * Some blocks in the cache may be modified relative to the state on disk
275 : * (they are marked "Dirty") and must be flushed to disk before they can
276 : * be discarded. Other (Clean) blocks may just be discarded if their memory
277 : * needs to be recovered.
278 : *
279 : * In normal situations applications do not interact directly with the
280 : * GDALRasterBlock - instead it it utilized by the RasterIO() interfaces
281 : * to implement caching.
282 : *
283 : * Some driver classes are implemented in a fashion that completely avoids
284 : * use of the GDAL raster cache (and GDALRasterBlock) though this is not very
285 : * common.
286 : */
287 :
288 : /************************************************************************/
289 : /* FlushCacheBlock() */
290 : /* */
291 : /* Note, if we have alot of blocks locked for a long time, this */
292 : /* method is going to get slow because it will have to traverse */
293 : /* the linked list a long ways looking for a flushing */
294 : /* candidate. It might help to re-touch locked blocks to push */
295 : /* them to the top of the list. */
296 : /************************************************************************/
297 :
298 : /**
299 : * \brief Attempt to flush at least one block from the cache.
300 : *
301 : * This static method is normally used to recover memory when a request
302 : * for a new cache block would put cache memory use over the established
303 : * limit.
304 : *
305 : * C++ analog to the C function GDALFlushCacheBlock().
306 : *
307 : * @return TRUE if successful or FALSE if no flushable block is found.
308 : */
309 :
310 7975 : int GDALRasterBlock::FlushCacheBlock()
311 :
312 : {
313 : int nXOff, nYOff;
314 : GDALRasterBand *poBand;
315 :
316 : {
317 7975 : CPLMutexHolderD( &hRBMutex );
318 7975 : GDALRasterBlock *poTarget = (GDALRasterBlock *) poOldest;
319 :
320 15950 : while( poTarget != NULL && poTarget->GetLockCount() > 0 )
321 0 : poTarget = poTarget->poPrevious;
322 :
323 7975 : if( poTarget == NULL )
324 476 : return FALSE;
325 :
326 7499 : poTarget->Detach();
327 :
328 7499 : nXOff = poTarget->GetXOff();
329 7499 : nYOff = poTarget->GetYOff();
330 7499 : poBand = poTarget->GetBand();
331 : }
332 :
333 7499 : CPLErr eErr = poBand->FlushBlock( nXOff, nYOff );
334 7499 : if (eErr != CE_None)
335 : {
336 : /* Save the error for later reporting */
337 0 : poBand->SetFlushBlockErr(eErr);
338 : }
339 :
340 7499 : return TRUE;
341 : }
342 :
343 : /************************************************************************/
344 : /* GDALRasterBlock() */
345 : /************************************************************************/
346 :
347 : /**
348 : * @brief GDALRasterBlock Constructor
349 : *
350 : * Normally only called from GDALRasterBand::GetLockedBlockRef().
351 : *
352 : * @param poBandIn the raster band used as source of raster block
353 : * being constructed.
354 : *
355 : * @param nXOffIn the horizontal block offset, with zero indicating
356 : * the left most block, 1 the next block and so forth.
357 : *
358 : * @param nYOffIn the vertical block offset, with zero indicating
359 : * the top most block, 1 the next block and so forth.
360 : */
361 :
362 431232 : GDALRasterBlock::GDALRasterBlock( GDALRasterBand *poBandIn,
363 431232 : int nXOffIn, int nYOffIn )
364 :
365 : {
366 431232 : CPLAssert( NULL != poBandIn );
367 :
368 431232 : poBand = poBandIn;
369 :
370 431232 : poBand->GetBlockSize( &nXSize, &nYSize );
371 431232 : eType = poBand->GetRasterDataType();
372 431232 : pData = NULL;
373 431232 : bDirty = FALSE;
374 431232 : nLockCount = 0;
375 :
376 431232 : poNext = poPrevious = NULL;
377 :
378 431232 : nXOff = nXOffIn;
379 431232 : nYOff = nYOffIn;
380 431232 : }
381 :
382 : /************************************************************************/
383 : /* ~GDALRasterBlock() */
384 : /************************************************************************/
385 :
386 : /**
387 : * Block destructor.
388 : *
389 : * Normally called from GDALRasterBand::FlushBlock().
390 : */
391 :
392 431232 : GDALRasterBlock::~GDALRasterBlock()
393 :
394 : {
395 431232 : Detach();
396 :
397 431232 : if( pData != NULL )
398 : {
399 : int nSizeInBytes;
400 :
401 431232 : VSIFree( pData );
402 :
403 431232 : nSizeInBytes = (nXSize * nYSize * GDALGetDataTypeSize(eType)+7)/8;
404 :
405 : {
406 431232 : CPLMutexHolderD( &hRBMutex );
407 431232 : nCacheUsed -= nSizeInBytes;
408 : }
409 : }
410 :
411 431232 : CPLAssert( nLockCount == 0 );
412 :
413 : #ifdef ENABLE_DEBUG
414 : Verify();
415 : #endif
416 431232 : }
417 :
418 : /************************************************************************/
419 : /* Detach() */
420 : /************************************************************************/
421 :
422 : /**
423 : * Remove block from cache.
424 : *
425 : * This method removes the current block from the linked list used to keep
426 : * track of all cached blocks in order of age. It does not affect whether
427 : * the block is referenced by a GDALRasterBand nor does it destroy or flush
428 : * the block.
429 : */
430 :
431 869963 : void GDALRasterBlock::Detach()
432 :
433 : {
434 869963 : CPLMutexHolderD( &hRBMutex );
435 :
436 869963 : if( poOldest == this )
437 10349 : poOldest = poPrevious;
438 :
439 869963 : if( poNewest == this )
440 : {
441 431700 : poNewest = poNext;
442 : }
443 :
444 869963 : if( poPrevious != NULL )
445 7031 : poPrevious->poNext = poNext;
446 :
447 869963 : if( poNext != NULL )
448 428382 : poNext->poPrevious = poPrevious;
449 :
450 869963 : poPrevious = NULL;
451 869963 : poNext = NULL;
452 869963 : }
453 :
454 : /************************************************************************/
455 : /* Verify() */
456 : /************************************************************************/
457 :
458 : /**
459 : * Confirms (via assertions) that the block cache linked list is in a
460 : * consistent state.
461 : */
462 :
463 0 : void GDALRasterBlock::Verify()
464 :
465 : {
466 0 : CPLMutexHolderD( &hRBMutex );
467 :
468 : CPLAssert( (poNewest == NULL && poOldest == NULL)
469 0 : || (poNewest != NULL && poOldest != NULL) );
470 :
471 0 : if( poNewest != NULL )
472 : {
473 0 : CPLAssert( poNewest->poPrevious == NULL );
474 0 : CPLAssert( poOldest->poNext == NULL );
475 :
476 0 : for( GDALRasterBlock *poBlock = (GDALRasterBlock *) poNewest;
477 : poBlock != NULL;
478 : poBlock = poBlock->poNext )
479 : {
480 0 : if( poBlock->poPrevious )
481 : {
482 0 : CPLAssert( poBlock->poPrevious->poNext == poBlock );
483 : }
484 :
485 0 : if( poBlock->poNext )
486 : {
487 0 : CPLAssert( poBlock->poNext->poPrevious == poBlock );
488 : }
489 : }
490 0 : }
491 0 : }
492 :
493 : /************************************************************************/
494 : /* Write() */
495 : /************************************************************************/
496 :
497 : /**
498 : * Force writing of the current block, if dirty.
499 : *
500 : * The block is written using GDALRasterBand::IWriteBlock() on it's
501 : * corresponding band object. Even if the write fails the block will
502 : * be marked clean.
503 : *
504 : * @return CE_None otherwise the error returned by IWriteBlock().
505 : */
506 :
507 65961 : CPLErr GDALRasterBlock::Write()
508 :
509 : {
510 65961 : if( !GetDirty() )
511 0 : return CE_None;
512 :
513 65961 : if( poBand == NULL )
514 0 : return CE_Failure;
515 :
516 65961 : MarkClean();
517 :
518 65961 : if (poBand->eFlushBlockErr == CE_None)
519 65961 : return poBand->IWriteBlock( nXOff, nYOff, pData );
520 : else
521 0 : return poBand->eFlushBlockErr;
522 : }
523 :
524 : /************************************************************************/
525 : /* Touch() */
526 : /************************************************************************/
527 :
528 : /**
529 : * Push block to top of LRU (least-recently used) list.
530 : *
531 : * This method is normally called when a block is used to keep track
532 : * that it has been recently used.
533 : */
534 :
535 5517772 : void GDALRasterBlock::Touch()
536 :
537 : {
538 5517772 : CPLMutexHolderD( &hRBMutex );
539 :
540 5517772 : if( poNewest == this )
541 : return;
542 :
543 4538907 : if( poOldest == this )
544 1382951 : poOldest = this->poPrevious;
545 :
546 4538907 : if( poPrevious != NULL )
547 4100176 : poPrevious->poNext = poNext;
548 :
549 4538907 : if( poNext != NULL )
550 2717225 : poNext->poPrevious = poPrevious;
551 :
552 4538907 : poPrevious = NULL;
553 4538907 : poNext = (GDALRasterBlock *) poNewest;
554 :
555 4538907 : if( poNewest != NULL )
556 : {
557 4535589 : CPLAssert( poNewest->poPrevious == NULL );
558 4535589 : poNewest->poPrevious = this;
559 : }
560 4538907 : poNewest = this;
561 :
562 4538907 : if( poOldest == NULL )
563 : {
564 3318 : CPLAssert( poPrevious == NULL && poNext == NULL );
565 3318 : poOldest = this;
566 0 : }
567 : #ifdef ENABLE_DEBUG
568 : Verify();
569 : #endif
570 : }
571 :
572 : /************************************************************************/
573 : /* Internalize() */
574 : /************************************************************************/
575 :
576 : /**
577 : * Allocate memory for block.
578 : *
579 : * This method allocates memory for the block, and attempts to flush other
580 : * blocks, if necessary, to bring the total cache size back within the limits.
581 : * The newly allocated block is touched and will be considered most recently
582 : * used in the LRU list.
583 : *
584 : * @return CE_None on success or CE_Failure if memory allocation fails.
585 : */
586 :
587 431232 : CPLErr GDALRasterBlock::Internalize()
588 :
589 : {
590 431232 : CPLMutexHolderD( &hRBMutex );
591 : void *pNewData;
592 : int nSizeInBytes;
593 431232 : GIntBig nCurCacheMax = GDALGetCacheMax64();
594 :
595 : /* No risk of overflow as it is checked in GDALRasterBand::InitBlockInfo() */
596 431232 : nSizeInBytes = nXSize * nYSize * (GDALGetDataTypeSize(eType) / 8);
597 :
598 431232 : pNewData = VSIMalloc( nSizeInBytes );
599 431232 : if( pNewData == NULL )
600 : {
601 : CPLError( CE_Failure, CPLE_OutOfMemory,
602 : "GDALRasterBlock::Internalize : Out of memory allocating %d bytes.",
603 0 : nSizeInBytes);
604 0 : return( CE_Failure );
605 : }
606 :
607 431232 : if( pData != NULL )
608 0 : memcpy( pNewData, pData, nSizeInBytes );
609 :
610 431232 : pData = pNewData;
611 :
612 : /* -------------------------------------------------------------------- */
613 : /* Flush old blocks if we are nearing our memory limit. */
614 : /* -------------------------------------------------------------------- */
615 431232 : AddLock(); /* don't flush this block! */
616 :
617 431232 : nCacheUsed += nSizeInBytes;
618 869959 : while( nCacheUsed > nCurCacheMax )
619 : {
620 7971 : GIntBig nOldCacheUsed = nCacheUsed;
621 :
622 7971 : GDALFlushCacheBlock();
623 :
624 7971 : if( nCacheUsed == nOldCacheUsed )
625 476 : break;
626 : }
627 :
628 : /* -------------------------------------------------------------------- */
629 : /* Add this block to the list. */
630 : /* -------------------------------------------------------------------- */
631 431232 : Touch();
632 431232 : DropLock();
633 :
634 431232 : return( CE_None );
635 : }
636 :
637 : /************************************************************************/
638 : /* MarkDirty() */
639 : /************************************************************************/
640 :
641 : /**
642 : * Mark the block as modified.
643 : *
644 : * A dirty block is one that has been modified and will need to be written
645 : * to disk before it can be flushed.
646 : */
647 :
648 1611171 : void GDALRasterBlock::MarkDirty()
649 :
650 : {
651 1611171 : bDirty = TRUE;
652 1611171 : }
653 :
654 :
655 : /************************************************************************/
656 : /* MarkClean() */
657 : /************************************************************************/
658 :
659 : /**
660 : * Mark the block as unmodified.
661 : *
662 : * A dirty block is one that has been modified and will need to be written
663 : * to disk before it can be flushed.
664 : */
665 :
666 71046 : void GDALRasterBlock::MarkClean()
667 :
668 : {
669 71046 : bDirty = FALSE;
670 71046 : }
671 :
672 : /************************************************************************/
673 : /* SafeLockBlock() */
674 : /************************************************************************/
675 :
676 : /**
677 : * \brief Safely lock block.
678 : *
679 : * This method locks a GDALRasterBlock (and touches it) in a thread-safe
680 : * manner. The global block cache mutex is held while locking the block,
681 : * in order to avoid race conditions with other threads that might be
682 : * trying to expire the block at the same time. The block pointer may be
683 : * safely NULL, in which case this method does nothing.
684 : *
685 : * @param ppBlock Pointer to the block pointer to try and lock/touch.
686 : */
687 :
688 5086904 : int GDALRasterBlock::SafeLockBlock( GDALRasterBlock ** ppBlock )
689 :
690 : {
691 5086904 : CPLAssert( NULL != ppBlock );
692 :
693 5086904 : CPLMutexHolderD( &hRBMutex );
694 :
695 5086904 : if( *ppBlock != NULL )
696 : {
697 4655308 : (*ppBlock)->AddLock();
698 4655308 : (*ppBlock)->Touch();
699 :
700 4655308 : return TRUE;
701 : }
702 : else
703 431596 : return FALSE;
704 : }
|
