1 : /******************************************************************************
2 : * $Id: gdalrasterblock.cpp 22669 2011-07-07 22:37:44Z rouault $
3 : *
4 : * Project: GDAL Core
5 : * Purpose: Implementation of GDALRasterBlock class and related global
6 : * raster block cache management.
7 : * Author: Frank Warmerdam, warmerdam@pobox.com
8 : *
9 : **********************************************************************
10 : * Copyright (c) 1998, Frank Warmerdam <warmerdam@pobox.com>
11 : *
12 : * Permission is hereby granted, free of charge, to any person obtaining a
13 : * copy of this software and associated documentation files (the "Software"),
14 : * to deal in the Software without restriction, including without limitation
15 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 : * and/or sell copies of the Software, and to permit persons to whom the
17 : * Software is furnished to do so, subject to the following conditions:
18 : *
19 : * The above copyright notice and this permission notice shall be included
20 : * in all copies or substantial portions of the Software.
21 : *
22 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 : * DEALINGS IN THE SOFTWARE.
29 : ****************************************************************************/
30 :
31 : #include "gdal_priv.h"
32 : #include "cpl_multiproc.h"
33 :
34 : CPL_CVSID("$Id: gdalrasterblock.cpp 22669 2011-07-07 22:37:44Z rouault $");
35 :
36 : static int bCacheMaxInitialized = FALSE;
37 : static GIntBig nCacheMax = 40 * 1024*1024;
38 : static volatile GIntBig nCacheUsed = 0;
39 :
40 : static volatile GDALRasterBlock *poOldest = NULL; /* tail */
41 : static volatile GDALRasterBlock *poNewest = NULL; /* head */
42 :
43 : static void *hRBMutex = NULL;
44 :
45 :
46 : /************************************************************************/
47 : /* GDALSetCacheMax() */
48 : /************************************************************************/
49 :
50 : /**
51 : * \brief Set maximum cache memory.
52 : *
53 : * This function sets the maximum amount of memory that GDAL is permitted
54 : * to use for GDALRasterBlock caching. The unit of the value is bytes.
55 : *
56 : * The maximum value is 2GB, due to the use of a signed 32 bit integer.
57 : * Use GDALSetCacheMax64() to be able to set a higher value.
58 : *
59 : * @param nNewSizeInBytes the maximum number of bytes for caching.
60 : */
61 :
62 8 : void CPL_STDCALL GDALSetCacheMax( int nNewSizeInBytes )
63 :
64 : {
65 8 : GDALSetCacheMax64(nNewSizeInBytes);
66 8 : }
67 :
68 :
69 : /************************************************************************/
70 : /* GDALSetCacheMax64() */
71 : /************************************************************************/
72 :
73 : /**
74 : * \brief Set maximum cache memory.
75 : *
76 : * This function sets the maximum amount of memory that GDAL is permitted
77 : * to use for GDALRasterBlock caching. The unit of the value is bytes.
78 : *
79 : * Note: On 32 bit platforms, the maximum amount of memory that can be addressed
80 : * by a process might be 2 GB or 3 GB, depending on the operating system
81 : * capabilities. This function will not make any attempt to check the
82 : * consistency of the passed value with the effective capabilities of the OS.
83 : *
84 : * @param nNewSizeInBytes the maximum number of bytes for caching.
85 : *
86 : * @since GDAL 1.8.0
87 : */
88 :
89 20 : void CPL_STDCALL GDALSetCacheMax64( GIntBig nNewSizeInBytes )
90 :
91 : {
92 20 : nCacheMax = nNewSizeInBytes;
93 :
94 : /* -------------------------------------------------------------------- */
95 : /* Flush blocks till we are under the new limit or till we */
96 : /* can't seem to flush anymore. */
97 : /* -------------------------------------------------------------------- */
98 41 : while( nCacheUsed > nCacheMax )
99 : {
100 1 : GIntBig nOldCacheUsed = nCacheUsed;
101 :
102 1 : GDALFlushCacheBlock();
103 :
104 1 : if( nCacheUsed == nOldCacheUsed )
105 0 : break;
106 : }
107 20 : }
108 :
109 : /************************************************************************/
110 : /* GDALGetCacheMax() */
111 : /************************************************************************/
112 :
113 : /**
114 : * \brief Get maximum cache memory.
115 : *
116 : * Gets the maximum amount of memory available to the GDALRasterBlock
117 : * caching system for caching GDAL read/write imagery.
118 : *
119 : * The first type this function is called, it will read the GDAL_CACHEMAX
120 : * configuation option to initialize the maximum cache memory.
121 : *
122 : * This function cannot return a value higher than 2 GB. Use
123 : * GDALGetCacheMax64() to get a non-truncated value.
124 : *
125 : * @return maximum in bytes.
126 : */
127 :
128 0 : int CPL_STDCALL GDALGetCacheMax()
129 : {
130 0 : GIntBig nRes = GDALGetCacheMax64();
131 0 : if (nRes > INT_MAX)
132 : {
133 : static int bHasWarned = FALSE;
134 0 : if (!bHasWarned)
135 : {
136 : CPLError(CE_Warning, CPLE_AppDefined,
137 : "Cache max value doesn't fit on a 32 bit integer. "
138 0 : "Call GDALGetCacheMax64() instead");
139 0 : bHasWarned = TRUE;
140 : }
141 0 : nRes = INT_MAX;
142 : }
143 0 : return (int)nRes;
144 : }
145 :
146 : /************************************************************************/
147 : /* GDALGetCacheMax64() */
148 : /************************************************************************/
149 :
150 : /**
151 : * \brief Get maximum cache memory.
152 : *
153 : * Gets the maximum amount of memory available to the GDALRasterBlock
154 : * caching system for caching GDAL read/write imagery.
155 : *
156 : * The first type this function is called, it will read the GDAL_CACHEMAX
157 : * configuation option to initialize the maximum cache memory.
158 : *
159 : * @return maximum in bytes.
160 : *
161 : * @since GDAL 1.8.0
162 : */
163 :
164 530199 : GIntBig CPL_STDCALL GDALGetCacheMax64()
165 : {
166 530199 : if( !bCacheMaxInitialized )
167 : {
168 388 : const char* pszCacheMax = CPLGetConfigOption("GDAL_CACHEMAX",NULL);
169 388 : bCacheMaxInitialized = TRUE;
170 388 : if( pszCacheMax != NULL )
171 : {
172 2 : GIntBig nNewCacheMax = (GIntBig)CPLScanUIntBig(pszCacheMax, strlen(pszCacheMax));
173 2 : if( nNewCacheMax < 100000 )
174 : {
175 2 : if (nNewCacheMax < 0)
176 : {
177 : CPLError(CE_Failure, CPLE_NotSupported,
178 0 : "Invalid value for GDAL_CACHEMAX. Using default value.");
179 0 : return nCacheMax;
180 : }
181 2 : nNewCacheMax *= 1024 * 1024;
182 : }
183 2 : nCacheMax = nNewCacheMax;
184 : }
185 : }
186 :
187 530199 : return nCacheMax;
188 : }
189 :
190 : /************************************************************************/
191 : /* GDALGetCacheUsed() */
192 : /************************************************************************/
193 :
194 : /**
195 : * \brief Get cache memory used.
196 : *
197 : * @return the number of bytes of memory currently in use by the
198 : * GDALRasterBlock memory caching.
199 : */
200 :
201 0 : int CPL_STDCALL GDALGetCacheUsed()
202 : {
203 0 : if (nCacheUsed > INT_MAX)
204 : {
205 : static int bHasWarned = FALSE;
206 0 : if (!bHasWarned)
207 : {
208 : CPLError(CE_Warning, CPLE_AppDefined,
209 : "Cache used value doesn't fit on a 32 bit integer. "
210 0 : "Call GDALGetCacheUsed64() instead");
211 0 : bHasWarned = TRUE;
212 : }
213 0 : return INT_MAX;
214 : }
215 0 : return (int)nCacheUsed;
216 : }
217 :
218 : /************************************************************************/
219 : /* GDALGetCacheUsed64() */
220 : /************************************************************************/
221 :
222 : /**
223 : * \brief Get cache memory used.
224 : *
225 : * @return the number of bytes of memory currently in use by the
226 : * GDALRasterBlock memory caching.
227 : *
228 : * @since GDAL 1.8.0
229 : */
230 :
231 2 : GIntBig CPL_STDCALL GDALGetCacheUsed64()
232 : {
233 2 : return nCacheUsed;
234 : }
235 :
236 : /************************************************************************/
237 : /* GDALFlushCacheBlock() */
238 : /* */
239 : /* The workhorse of cache management! */
240 : /************************************************************************/
241 :
242 : /**
243 : * \brief Try to flush one cached raster block
244 : *
245 : * This function will search the first unlocked raster block and will
246 : * flush it to release the associated memory.
247 : *
248 : * @return TRUE if one block was flushed, FALSE if there are no cached blocks
249 : * or if they are currently locked.
250 : */
251 7968 : int CPL_STDCALL GDALFlushCacheBlock()
252 :
253 : {
254 7968 : return GDALRasterBlock::FlushCacheBlock();
255 : }
256 :
257 : /************************************************************************/
258 : /* ==================================================================== */
259 : /* GDALRasterBlock */
260 : /* ==================================================================== */
261 : /************************************************************************/
262 :
263 : /**
264 : * \class GDALRasterBlock "gdal_priv.h"
265 : *
266 : * GDALRasterBlock objects hold one block of raster data for one band
267 : * that is currently stored in the GDAL raster cache. The cache holds
268 : * some blocks of raster data for zero or more GDALRasterBand objects
269 : * across zero or more GDALDataset objects in a global raster cache with
270 : * a least recently used (LRU) list and an upper cache limit (see
271 : * GDALSetCacheMax()) under which the cache size is normally kept.
272 : *
273 : * Some blocks in the cache may be modified relative to the state on disk
274 : * (they are marked "Dirty") and must be flushed to disk before they can
275 : * be discarded. Other (Clean) blocks may just be discarded if their memory
276 : * needs to be recovered.
277 : *
278 : * In normal situations applications do not interact directly with the
279 : * GDALRasterBlock - instead it it utilized by the RasterIO() interfaces
280 : * to implement caching.
281 : *
282 : * Some driver classes are implemented in a fashion that completely avoids
283 : * use of the GDAL raster cache (and GDALRasterBlock) though this is not very
284 : * common.
285 : */
286 :
287 : /************************************************************************/
288 : /* FlushCacheBlock() */
289 : /* */
290 : /* Note, if we have alot of blocks locked for a long time, this */
291 : /* method is going to get slow because it will have to traverse */
292 : /* the linked list a long ways looking for a flushing */
293 : /* candidate. It might help to re-touch locked blocks to push */
294 : /* them to the top of the list. */
295 : /************************************************************************/
296 :
297 : /**
298 : * \brief Attempt to flush at least one block from the cache.
299 : *
300 : * This static method is normally used to recover memory when a request
301 : * for a new cache block would put cache memory use over the established
302 : * limit.
303 : *
304 : * C++ analog to the C function GDALFlushCacheBlock().
305 : *
306 : * @return TRUE if successful or FALSE if no flushable block is found.
307 : */
308 :
309 7968 : int GDALRasterBlock::FlushCacheBlock()
310 :
311 : {
312 : int nXOff, nYOff;
313 : GDALRasterBand *poBand;
314 :
315 : {
316 7968 : CPLMutexHolderD( &hRBMutex );
317 7968 : GDALRasterBlock *poTarget = (GDALRasterBlock *) poOldest;
318 :
319 15936 : while( poTarget != NULL && poTarget->GetLockCount() > 0 )
320 0 : poTarget = poTarget->poPrevious;
321 :
322 7968 : if( poTarget == NULL )
323 475 : return FALSE;
324 :
325 7493 : poTarget->Detach();
326 :
327 7493 : nXOff = poTarget->GetXOff();
328 7493 : nYOff = poTarget->GetYOff();
329 7493 : poBand = poTarget->GetBand();
330 : }
331 :
332 7493 : CPLErr eErr = poBand->FlushBlock( nXOff, nYOff );
333 7493 : if (eErr != CE_None)
334 : {
335 : /* Save the error for later reporting */
336 0 : poBand->SetFlushBlockErr(eErr);
337 : }
338 :
339 7493 : return TRUE;
340 : }
341 :
342 : /************************************************************************/
343 : /* GDALRasterBlock() */
344 : /************************************************************************/
345 :
346 : /**
347 : * @brief GDALRasterBlock Constructor
348 : *
349 : * Normally only called from GDALRasterBand::GetLockedBlockRef().
350 : *
351 : * @param poBandIn the raster band used as source of raster block
352 : * being constructed.
353 : *
354 : * @param nXOffIn the horizontal block offset, with zero indicating
355 : * the left most block, 1 the next block and so forth.
356 : *
357 : * @param nYOffIn the vertical block offset, with zero indicating
358 : * the top most block, 1 the next block and so forth.
359 : */
360 :
361 367941 : GDALRasterBlock::GDALRasterBlock( GDALRasterBand *poBandIn,
362 367941 : int nXOffIn, int nYOffIn )
363 :
364 : {
365 367941 : CPLAssert( NULL != poBandIn );
366 :
367 367941 : poBand = poBandIn;
368 :
369 367941 : poBand->GetBlockSize( &nXSize, &nYSize );
370 367941 : eType = poBand->GetRasterDataType();
371 367941 : pData = NULL;
372 367941 : bDirty = FALSE;
373 367941 : nLockCount = 0;
374 :
375 367941 : poNext = poPrevious = NULL;
376 :
377 367941 : nXOff = nXOffIn;
378 367941 : nYOff = nYOffIn;
379 367941 : }
380 :
381 : /************************************************************************/
382 : /* ~GDALRasterBlock() */
383 : /************************************************************************/
384 :
385 : /**
386 : * Block destructor.
387 : *
388 : * Normally called from GDALRasterBand::FlushBlock().
389 : */
390 :
391 367941 : GDALRasterBlock::~GDALRasterBlock()
392 :
393 : {
394 367941 : Detach();
395 :
396 367941 : if( pData != NULL )
397 : {
398 : int nSizeInBytes;
399 :
400 367941 : VSIFree( pData );
401 :
402 367941 : nSizeInBytes = (nXSize * nYSize * GDALGetDataTypeSize(eType)+7)/8;
403 :
404 : {
405 367941 : CPLMutexHolderD( &hRBMutex );
406 367941 : nCacheUsed -= nSizeInBytes;
407 : }
408 : }
409 :
410 367941 : CPLAssert( nLockCount == 0 );
411 :
412 : #ifdef ENABLE_DEBUG
413 : Verify();
414 : #endif
415 367941 : }
416 :
417 : /************************************************************************/
418 : /* Detach() */
419 : /************************************************************************/
420 :
421 : /**
422 : * Remove block from cache.
423 : *
424 : * This method removes the current block from the linked list used to keep
425 : * track of all cached blocks in order of age. It does not affect whether
426 : * the block is referenced by a GDALRasterBand nor does it destroy or flush
427 : * the block.
428 : */
429 :
430 743375 : void GDALRasterBlock::Detach()
431 :
432 : {
433 743375 : CPLMutexHolderD( &hRBMutex );
434 :
435 743375 : if( poOldest == this )
436 10196 : poOldest = poPrevious;
437 :
438 743375 : if( poNewest == this )
439 : {
440 368409 : poNewest = poNext;
441 : }
442 :
443 743375 : if( poPrevious != NULL )
444 7025 : poPrevious->poNext = poNext;
445 :
446 743375 : if( poNext != NULL )
447 365238 : poNext->poPrevious = poPrevious;
448 :
449 743375 : poPrevious = NULL;
450 743375 : poNext = NULL;
451 743375 : }
452 :
453 : /************************************************************************/
454 : /* Verify() */
455 : /************************************************************************/
456 :
457 : /**
458 : * Confirms (via assertions) that the block cache linked list is in a
459 : * consistent state.
460 : */
461 :
462 0 : void GDALRasterBlock::Verify()
463 :
464 : {
465 0 : CPLMutexHolderD( &hRBMutex );
466 :
467 : CPLAssert( (poNewest == NULL && poOldest == NULL)
468 0 : || (poNewest != NULL && poOldest != NULL) );
469 :
470 0 : if( poNewest != NULL )
471 : {
472 0 : CPLAssert( poNewest->poPrevious == NULL );
473 0 : CPLAssert( poOldest->poNext == NULL );
474 :
475 0 : for( GDALRasterBlock *poBlock = (GDALRasterBlock *) poNewest;
476 : poBlock != NULL;
477 : poBlock = poBlock->poNext )
478 : {
479 0 : if( poBlock->poPrevious )
480 : {
481 0 : CPLAssert( poBlock->poPrevious->poNext == poBlock );
482 : }
483 :
484 0 : if( poBlock->poNext )
485 : {
486 0 : CPLAssert( poBlock->poNext->poPrevious == poBlock );
487 : }
488 : }
489 0 : }
490 0 : }
491 :
492 : /************************************************************************/
493 : /* Write() */
494 : /************************************************************************/
495 :
496 : /**
497 : * Force writing of the current block, if dirty.
498 : *
499 : * The block is written using GDALRasterBand::IWriteBlock() on it's
500 : * corresponding band object. Even if the write fails the block will
501 : * be marked clean.
502 : *
503 : * @return CE_None otherwise the error returned by IWriteBlock().
504 : */
505 :
506 53407 : CPLErr GDALRasterBlock::Write()
507 :
508 : {
509 53407 : if( !GetDirty() )
510 0 : return CE_None;
511 :
512 53407 : if( poBand == NULL )
513 0 : return CE_Failure;
514 :
515 53407 : MarkClean();
516 :
517 53407 : if (poBand->eFlushBlockErr == CE_None)
518 53407 : return poBand->IWriteBlock( nXOff, nYOff, pData );
519 : else
520 0 : return poBand->eFlushBlockErr;
521 : }
522 :
523 : /************************************************************************/
524 : /* Touch() */
525 : /************************************************************************/
526 :
527 : /**
528 : * Push block to top of LRU (least-recently used) list.
529 : *
530 : * This method is normally called when a block is used to keep track
531 : * that it has been recently used.
532 : */
533 :
534 5139356 : void GDALRasterBlock::Touch()
535 :
536 : {
537 5139356 : CPLMutexHolderD( &hRBMutex );
538 :
539 5139356 : if( poNewest == this )
540 : return;
541 :
542 4264814 : if( poOldest == this )
543 1333254 : poOldest = this->poPrevious;
544 :
545 4264814 : if( poPrevious != NULL )
546 3889380 : poPrevious->poNext = poNext;
547 :
548 4264814 : if( poNext != NULL )
549 2556126 : poNext->poPrevious = poPrevious;
550 :
551 4264814 : poPrevious = NULL;
552 4264814 : poNext = (GDALRasterBlock *) poNewest;
553 :
554 4264814 : if( poNewest != NULL )
555 : {
556 4261643 : CPLAssert( poNewest->poPrevious == NULL );
557 4261643 : poNewest->poPrevious = this;
558 : }
559 4264814 : poNewest = this;
560 :
561 4264814 : if( poOldest == NULL )
562 : {
563 3171 : CPLAssert( poPrevious == NULL && poNext == NULL );
564 3171 : poOldest = this;
565 0 : }
566 : #ifdef ENABLE_DEBUG
567 : Verify();
568 : #endif
569 : }
570 :
571 : /************************************************************************/
572 : /* Internalize() */
573 : /************************************************************************/
574 :
575 : /**
576 : * Allocate memory for block.
577 : *
578 : * This method allocates memory for the block, and attempts to flush other
579 : * blocks, if necessary, to bring the total cache size back within the limits.
580 : * The newly allocated block is touched and will be considered most recently
581 : * used in the LRU list.
582 : *
583 : * @return CE_None on success or CE_Failure if memory allocation fails.
584 : */
585 :
586 367941 : CPLErr GDALRasterBlock::Internalize()
587 :
588 : {
589 367941 : CPLMutexHolderD( &hRBMutex );
590 : void *pNewData;
591 : int nSizeInBytes;
592 367941 : GIntBig nCurCacheMax = GDALGetCacheMax64();
593 :
594 : /* No risk of overflow as it is checked in GDALRasterBand::InitBlockInfo() */
595 367941 : nSizeInBytes = nXSize * nYSize * (GDALGetDataTypeSize(eType) / 8);
596 :
597 367941 : pNewData = VSIMalloc( nSizeInBytes );
598 367941 : if( pNewData == NULL )
599 : {
600 : CPLError( CE_Failure, CPLE_OutOfMemory,
601 : "GDALRasterBlock::Internalize : Out of memory allocating %d bytes.",
602 0 : nSizeInBytes);
603 0 : return( CE_Failure );
604 : }
605 :
606 367941 : if( pData != NULL )
607 0 : memcpy( pNewData, pData, nSizeInBytes );
608 :
609 367941 : pData = pNewData;
610 :
611 : /* -------------------------------------------------------------------- */
612 : /* Flush old blocks if we are nearing our memory limit. */
613 : /* -------------------------------------------------------------------- */
614 367941 : AddLock(); /* don't flush this block! */
615 :
616 367941 : nCacheUsed += nSizeInBytes;
617 743374 : while( nCacheUsed > nCurCacheMax )
618 : {
619 7967 : GIntBig nOldCacheUsed = nCacheUsed;
620 :
621 7967 : GDALFlushCacheBlock();
622 :
623 7967 : if( nCacheUsed == nOldCacheUsed )
624 475 : break;
625 : }
626 :
627 : /* -------------------------------------------------------------------- */
628 : /* Add this block to the list. */
629 : /* -------------------------------------------------------------------- */
630 367941 : Touch();
631 367941 : DropLock();
632 :
633 367941 : return( CE_None );
634 : }
635 :
636 : /************************************************************************/
637 : /* MarkDirty() */
638 : /************************************************************************/
639 :
640 : /**
641 : * Mark the block as modified.
642 : *
643 : * A dirty block is one that has been modified and will need to be written
644 : * to disk before it can be flushed.
645 : */
646 :
647 1587633 : void GDALRasterBlock::MarkDirty()
648 :
649 : {
650 1587633 : bDirty = TRUE;
651 1587633 : }
652 :
653 :
654 : /************************************************************************/
655 : /* MarkClean() */
656 : /************************************************************************/
657 :
658 : /**
659 : * Mark the block as unmodified.
660 : *
661 : * A dirty block is one that has been modified and will need to be written
662 : * to disk before it can be flushed.
663 : */
664 :
665 57683 : void GDALRasterBlock::MarkClean()
666 :
667 : {
668 57683 : bDirty = FALSE;
669 57683 : }
670 :
671 : /************************************************************************/
672 : /* SafeLockBlock() */
673 : /************************************************************************/
674 :
675 : /**
676 : * \brief Safely lock block.
677 : *
678 : * This method locks a GDALRasterBlock (and touches it) in a thread-safe
679 : * manner. The global block cache mutex is held while locking the block,
680 : * in order to avoid race conditions with other threads that might be
681 : * trying to expire the block at the same time. The block pointer may be
682 : * safely NULL, in which case this method does nothing.
683 : *
684 : * @param ppBlock Pointer to the block pointer to try and lock/touch.
685 : */
686 :
687 4771455 : int GDALRasterBlock::SafeLockBlock( GDALRasterBlock ** ppBlock )
688 :
689 : {
690 4771455 : CPLAssert( NULL != ppBlock );
691 :
692 4771455 : CPLMutexHolderD( &hRBMutex );
693 :
694 4771455 : if( *ppBlock != NULL )
695 : {
696 4403474 : (*ppBlock)->AddLock();
697 4403474 : (*ppBlock)->Touch();
698 :
699 4403474 : return TRUE;
700 : }
701 : else
702 367981 : return FALSE;
703 : }
|
