1 : /******************************************************************************
2 : *
3 : * Purpose: Implementation of the SysVirtualFile class.
4 : *
5 : * This class is used to manage access to a single virtual file stored in
6 : * SysBData segments based on a block map stored in the SysBMDir segment
7 : * (and managed by SysBlockMap class).
8 : *
9 : * The virtual files are allocated in 8K chunks (block_size) in segments.
10 : * To minimize IO requests, other overhead, we keep one such 8K block in
11 : * our working cache for the virtual file stream.
12 : *
13 : * This class is primarily used by the CTiledChannel class for access to
14 : * tiled images.
15 : *
16 : ******************************************************************************
17 : * Copyright (c) 2009
18 : * PCI Geomatics, 50 West Wilmot Street, Richmond Hill, Ont, Canada
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining a
21 : * copy of this software and associated documentation files (the "Software"),
22 : * to deal in the Software without restriction, including without limitation
23 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
24 : * and/or sell copies of the Software, and to permit persons to whom the
25 : * Software is furnished to do so, subject to the following conditions:
26 : *
27 : * The above copyright notice and this permission notice shall be included
28 : * in all copies or substantial portions of the Software.
29 : *
30 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
31 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
32 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
33 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
34 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
35 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
36 : * DEALINGS IN THE SOFTWARE.
37 : ****************************************************************************/
38 : #include "pcidsk_config.h"
39 : #include "pcidsk_types.h"
40 : #include "pcidsk_buffer.h"
41 : #include "pcidsk_exception.h"
42 : #include "pcidsk_utils.h"
43 : #include "core/sysvirtualfile.h"
44 : #include "core/cpcidskfile.h"
45 : #include "core/mutexholder.h"
46 : #include "segment/sysblockmap.h"
47 : #include <cassert>
48 : #include <cstring>
49 : #if 0
50 : #include <cstdio>
51 : #endif
52 :
53 : using namespace PCIDSK;
54 :
55 :
56 : const int SysVirtualFile::block_size = SYSVIRTUALFILE_BLOCKSIZE;
57 :
58 : /************************************************************************/
59 : /* SysVirtualFile() */
60 : /************************************************************************/
61 :
62 14 : SysVirtualFile::SysVirtualFile( CPCIDSKFile *file, int start_block,
63 : uint64 image_length,
64 : SysBlockMap *sysblockmap,
65 14 : int image_index )
66 :
67 : {
68 14 : io_handle = NULL;
69 14 : io_mutex = NULL;
70 :
71 14 : file_length = image_length;
72 14 : this->file = file;
73 14 : this->sysblockmap = sysblockmap;
74 14 : this->image_index = image_index;
75 :
76 14 : loaded_block = -1;
77 14 : loaded_block_dirty = false;
78 :
79 14 : last_bm_index = -1;
80 :
81 14 : next_bm_entry_to_load = start_block;
82 :
83 14 : regular_blocks = false;
84 14 : blocks_loaded = 0;
85 14 : }
86 :
87 : /************************************************************************/
88 : /* ~SysVirtualFile() */
89 : /************************************************************************/
90 :
91 14 : SysVirtualFile::~SysVirtualFile()
92 :
93 : {
94 14 : Synchronize();
95 14 : }
96 :
97 : /************************************************************************/
98 : /* Synchronize() */
99 : /************************************************************************/
100 :
101 34 : void SysVirtualFile::Synchronize()
102 :
103 : {
104 34 : FlushDirtyBlock();
105 34 : }
106 :
107 : /************************************************************************/
108 : /* GetBlockSegment() */
109 : /************************************************************************/
110 :
111 24 : uint16 SysVirtualFile::GetBlockSegment( int requested_block )
112 :
113 : {
114 24 : if( requested_block < 0 )
115 : ThrowPCIDSKException( "SysVirtualFile::GetBlockSegment(%d) - illegal request.",
116 0 : requested_block );
117 :
118 24 : if( requested_block >= blocks_loaded )
119 0 : LoadBMEntrysTo( requested_block );
120 :
121 24 : if( regular_blocks )
122 : // regular blocks are all in one segment.
123 0 : return xblock_segment[0];
124 : else
125 24 : return xblock_segment[requested_block];
126 : }
127 :
128 : /************************************************************************/
129 : /* GetBlockIndexInSegment() */
130 : /************************************************************************/
131 :
132 24 : int SysVirtualFile::GetBlockIndexInSegment( int requested_block )
133 :
134 : {
135 24 : if( requested_block < 0 )
136 : ThrowPCIDSKException( "SysVirtualFile::GetBlockIndexInSegment(%d) - illegal request.",
137 0 : requested_block );
138 :
139 24 : if( requested_block >= blocks_loaded )
140 0 : LoadBMEntrysTo( requested_block );
141 :
142 24 : if( regular_blocks )
143 : // regular blocks all follow the first block in order.
144 0 : return xblock_index[0] + requested_block;
145 : else
146 24 : return xblock_index[requested_block];
147 : }
148 :
149 :
150 : /************************************************************************/
151 : /* SetBlockInfo() */
152 : /************************************************************************/
153 :
154 15 : void SysVirtualFile::SetBlockInfo( int requested_block,
155 : uint16 new_block_segment,
156 : int new_block_index )
157 :
158 : {
159 15 : if( requested_block < 0 )
160 : ThrowPCIDSKException( "SysVirtualFile::SetBlockSegment(%d) - illegal request.",
161 0 : requested_block );
162 :
163 : // this should always be the case.
164 15 : assert( requested_block == blocks_loaded );
165 :
166 : // Initialization case.
167 15 : if( requested_block == 0 && blocks_loaded == 0 )
168 : {
169 14 : xblock_segment.push_back( new_block_segment );
170 14 : xblock_index.push_back( new_block_index );
171 14 : blocks_loaded = 1;
172 14 : return;
173 : }
174 :
175 1 : if( !regular_blocks )
176 : {
177 1 : xblock_segment.push_back( new_block_segment );
178 1 : xblock_index.push_back( new_block_index );
179 1 : blocks_loaded++;
180 1 : return;
181 : }
182 :
183 : // Are things still regular?
184 0 : if( new_block_segment == xblock_segment[0]
185 : && new_block_index == xblock_index[0] + requested_block )
186 : {
187 0 : blocks_loaded++;
188 0 : return;
189 : }
190 :
191 : // Ah, we see they are now irregular. We need to build up the
192 : // segment/index arrays and proceed to populate them.
193 0 : Debug( file->GetInterfaces()->Debug,
194 : "SysVirtualFile - Discovered stream is irregulr. %d/%d follows %d/%d at block %d.\n",
195 : new_block_segment, new_block_index,
196 : xblock_segment[0], xblock_index[0],
197 0 : requested_block );
198 :
199 0 : regular_blocks = false;
200 0 : while( (int) xblock_segment.size() < blocks_loaded )
201 : {
202 0 : xblock_segment.push_back( xblock_segment[0] );
203 0 : xblock_index.push_back( xblock_index[xblock_index.size()-1]+1 );
204 : }
205 :
206 0 : xblock_segment.push_back( new_block_segment );
207 0 : xblock_index.push_back( new_block_index );
208 0 : blocks_loaded++;
209 : }
210 :
211 : /************************************************************************/
212 : /* WriteToFile() */
213 : /************************************************************************/
214 :
215 : void
216 20 : SysVirtualFile::WriteToFile( const void *buffer, uint64 offset, uint64 size )
217 :
218 : {
219 20 : uint64 buffer_offset = 0;
220 :
221 20 : if(io_handle == NULL || io_mutex == NULL)
222 4 : file->GetIODetails( &io_handle, &io_mutex );
223 :
224 20 : MutexHolder oMutex(*io_mutex);
225 :
226 61 : while( buffer_offset < size )
227 : {
228 21 : int request_block = (int) ((offset + buffer_offset) / block_size);
229 21 : int offset_in_block = (int) ((offset + buffer_offset) % block_size);
230 21 : int amount_to_copy = block_size - offset_in_block;
231 :
232 42 : if (offset_in_block != 0 || (size - buffer_offset) < (uint64)block_size) {
233 : // we need to read in the block for update
234 21 : LoadBlock( request_block );
235 21 : if( amount_to_copy > (int) (size - buffer_offset) )
236 20 : amount_to_copy = (int) (size - buffer_offset);
237 :
238 : // fill in the block
239 : memcpy( block_data + offset_in_block,
240 : ((uint8 *) buffer) + buffer_offset,
241 21 : amount_to_copy );
242 :
243 21 : loaded_block_dirty = true;
244 : } else {
245 0 : int num_full_blocks = (int) ((size - buffer_offset) / block_size);
246 :
247 0 : WriteBlocks(request_block, num_full_blocks, (uint8*)buffer + buffer_offset);
248 :
249 0 : amount_to_copy = num_full_blocks * block_size;
250 : }
251 :
252 21 : buffer_offset += amount_to_copy;
253 : }
254 :
255 20 : if( offset+size > file_length )
256 : {
257 12 : file_length = offset+size;
258 12 : sysblockmap->SetVirtualFileSize( image_index, file_length );
259 20 : }
260 20 : }
261 :
262 : /************************************************************************/
263 : /* ReadFromFile() */
264 : /************************************************************************/
265 :
266 35 : void SysVirtualFile::ReadFromFile( void *buffer, uint64 offset, uint64 size )
267 :
268 : {
269 35 : if(io_handle == NULL || io_mutex == NULL)
270 10 : file->GetIODetails( &io_handle, &io_mutex );
271 :
272 35 : MutexHolder oMutex(*io_mutex);
273 :
274 35 : uint64 buffer_offset = 0;
275 : #if 0
276 : printf("Requesting region at %llu of size %llu\n", offset, size);
277 : #endif
278 106 : while( buffer_offset < size )
279 : {
280 36 : int request_block = (int) ((offset + buffer_offset) / block_size);
281 36 : int offset_in_block = (int) ((offset + buffer_offset) % block_size);
282 36 : int amount_to_copy = block_size - offset_in_block;
283 :
284 :
285 :
286 72 : if (offset_in_block != 0 || (size - buffer_offset) < (uint64)block_size) {
287 : // Deal with the case where we need to load a partial block. Hopefully
288 : // this doesn't happen often
289 36 : LoadBlock( request_block );
290 36 : if( amount_to_copy > (int) (size - buffer_offset) )
291 35 : amount_to_copy = (int) (size - buffer_offset);
292 : memcpy( ((uint8 *) buffer) + buffer_offset,
293 36 : block_data + offset_in_block, amount_to_copy );
294 : } else {
295 : // Use the bulk loading of blocks. First, compute the range
296 : // of full blocks we need to load
297 0 : int num_full_blocks = (int) ((size - buffer_offset)/block_size);
298 :
299 0 : LoadBlocks(request_block, num_full_blocks, ((uint8*)buffer) + buffer_offset);
300 0 : amount_to_copy = num_full_blocks * block_size;
301 : }
302 :
303 :
304 36 : buffer_offset += amount_to_copy;
305 35 : }
306 35 : }
307 :
308 : /************************************************************************/
309 : /* LoadBlock() */
310 : /************************************************************************/
311 : /**
312 : * Loads the requested_block block from the system virtual file. Extends
313 : * the file if necessary
314 : */
315 57 : void SysVirtualFile::LoadBlock( int requested_block )
316 :
317 : {
318 : /* -------------------------------------------------------------------- */
319 : /* Do we already have this block? */
320 : /* -------------------------------------------------------------------- */
321 57 : if( requested_block == loaded_block )
322 39 : return;
323 :
324 : /* -------------------------------------------------------------------- */
325 : /* Do we need to grow the virtual file by one block? */
326 : /* -------------------------------------------------------------------- */
327 18 : GrowVirtualFile(requested_block);
328 :
329 : /* -------------------------------------------------------------------- */
330 : /* Does this block exist in the virtual file? */
331 : /* -------------------------------------------------------------------- */
332 18 : if( requested_block < 0 || requested_block >= blocks_loaded )
333 : ThrowPCIDSKException( "SysVirtualFile::LoadBlock(%d) - block out of range.",
334 0 : requested_block );
335 :
336 : /* -------------------------------------------------------------------- */
337 : /* Do we have a dirty block loaded that needs to be saved? */
338 : /* -------------------------------------------------------------------- */
339 18 : FlushDirtyBlock();
340 :
341 : /* -------------------------------------------------------------------- */
342 : /* Load the requested block. */
343 : /* -------------------------------------------------------------------- */
344 18 : LoadBMEntrysTo( requested_block );
345 : PCIDSKSegment *data_seg_obj =
346 18 : file->GetSegment( GetBlockSegment( requested_block ) );
347 :
348 : data_seg_obj->ReadFromFile( block_data,
349 : block_size * (uint64) GetBlockIndexInSegment( requested_block ),
350 18 : block_size );
351 :
352 18 : loaded_block = requested_block;
353 18 : loaded_block_dirty = false;
354 : }
355 :
356 : /************************************************************************/
357 : /* FlushDirtyBlock() */
358 : /************************************************************************/
359 : /**
360 : * If the block currently loaded is dirty, flush it to the file
361 : */
362 52 : void SysVirtualFile::FlushDirtyBlock(void)
363 : {
364 52 : if (loaded_block_dirty) {
365 6 : if(io_handle == NULL || io_mutex == NULL)
366 0 : file->GetIODetails( &io_handle, &io_mutex );
367 :
368 6 : MutexHolder oMutex(*io_mutex);
369 :
370 : PCIDSKSegment *data_seg_obj =
371 6 : file->GetSegment( GetBlockSegment( loaded_block ) );
372 :
373 : data_seg_obj->WriteToFile( block_data,
374 : block_size * (uint64) GetBlockIndexInSegment( loaded_block ),
375 6 : block_size );
376 6 : loaded_block_dirty = false;
377 : }
378 52 : }
379 :
380 : /************************************************************************/
381 : /* GrowVirtualFile() */
382 : /************************************************************************/
383 18 : void SysVirtualFile::GrowVirtualFile(std::ptrdiff_t requested_block)
384 : {
385 18 : LoadBMEntrysTo( requested_block );
386 :
387 18 : if( requested_block == blocks_loaded )
388 : {
389 5 : if(io_handle == NULL || io_mutex == NULL)
390 0 : file->GetIODetails( &io_handle, &io_mutex );
391 :
392 5 : MutexHolder oMutex(*io_mutex);
393 :
394 : int new_seg;
395 : int offset;
396 :
397 : offset =
398 5 : sysblockmap->GrowVirtualFile( image_index, last_bm_index, new_seg);
399 5 : SetBlockInfo( requested_block, (uint16) new_seg, offset );
400 : }
401 18 : }
402 :
403 : /************************************************************************/
404 : /* WriteBlocks() */
405 : /************************************************************************/
406 : /**
407 : * \brief Writes a group of blocks
408 : * Attempts to create a group of blocks (if the SysVirtualFile
409 : * is not already large enough to hold them) and then write them
410 : * out contiguously
411 : */
412 0 : void SysVirtualFile::WriteBlocks(int first_block,
413 : int block_count,
414 : void* const buffer)
415 : {
416 0 : if(io_handle == NULL || io_mutex == NULL)
417 0 : file->GetIODetails( &io_handle, &io_mutex );
418 :
419 0 : MutexHolder oMutex(*io_mutex);
420 :
421 0 : FlushDirtyBlock();
422 : // Iterate through all the blocks to be written, first, then
423 : // grow the virtual file
424 0 : for (unsigned int i = 0; i <= (unsigned int) block_count; i++) {
425 0 : GrowVirtualFile(first_block + i);
426 : }
427 :
428 : // Using a similar algorithm to the LoadBlocks() function,
429 : // attempt to coalesce writes
430 0 : std::size_t buffer_off = 0;
431 0 : std::size_t blocks_written = 0;
432 0 : std::size_t current_first_block = first_block;
433 0 : while (blocks_written < (std::size_t) block_count) {
434 0 : LoadBMEntrysTo( current_first_block+1 );
435 :
436 0 : unsigned int cur_segment = GetBlockSegment( current_first_block );
437 0 : unsigned int cur_block = current_first_block;
438 0 : while (cur_block < (unsigned int)block_count + first_block &&
439 : (unsigned int) GetBlockSegment(cur_block + 1) == cur_segment)
440 : {
441 0 : cur_block++;
442 0 : LoadBMEntrysTo( current_first_block+1 );
443 : }
444 :
445 : // Find largest span of contiguous blocks we can write
446 0 : uint64 write_start = GetBlockIndexInSegment(current_first_block);
447 0 : uint64 write_cur = write_start * block_size;
448 0 : unsigned int count_to_write = 1;
449 0 : while (write_cur + block_size ==
450 : (uint64)GetBlockIndexInSegment(count_to_write + current_first_block - 1) * block_size &&
451 : count_to_write < (cur_block - current_first_block))
452 : {
453 0 : write_cur += block_size;
454 0 : count_to_write++;
455 : }
456 :
457 : PCIDSKSegment *data_seg_obj =
458 0 : file->GetSegment( cur_segment );
459 :
460 0 : std::size_t bytes_to_write = count_to_write * block_size;
461 :
462 : data_seg_obj->WriteToFile((uint8*)buffer + buffer_off,
463 : block_size * write_start,
464 0 : bytes_to_write);
465 :
466 0 : buffer_off += bytes_to_write;
467 0 : blocks_written += count_to_write;
468 0 : current_first_block += count_to_write;
469 0 : }
470 0 : }
471 :
472 : /************************************************************************/
473 : /* LoadBlocks() */
474 : /************************************************************************/
475 : /**
476 : * \brief Load a group of blocks
477 : * Attempts to coalesce reading of groups of blocks into a single
478 : * filesystem I/O operation. Does not cache the loaded block, nor
479 : * does it modify the state of the SysVirtualFile, other than to
480 : * flush the loaded block if it is dirty.
481 : */
482 0 : void SysVirtualFile::LoadBlocks(int requested_block_start,
483 : int requested_block_count,
484 : void* const buffer)
485 : {
486 0 : if(io_handle == NULL || io_mutex == NULL)
487 0 : file->GetIODetails( &io_handle, &io_mutex );
488 :
489 0 : MutexHolder oMutex(*io_mutex);
490 :
491 0 : FlushDirtyBlock();
492 :
493 0 : unsigned int blocks_read = 0;
494 0 : unsigned int current_start = requested_block_start;
495 :
496 0 : std::size_t buffer_off = 0;
497 :
498 0 : while (blocks_read < (unsigned int)requested_block_count) {
499 : // Coalesce blocks that are in the same segment
500 0 : LoadBMEntrysTo( current_start+1 );
501 0 : unsigned int cur_segment = GetBlockSegment(current_start); // segment of current
502 : // first block
503 0 : unsigned int cur_block = current_start; // starting block ID
504 0 : while (cur_block < (unsigned int)requested_block_count + requested_block_start &&
505 : GetBlockSegment(cur_block + 1) == cur_segment) {
506 : // this block is in the same segment as the previous one we
507 : // wanted to read.
508 0 : cur_block++;
509 0 : LoadBMEntrysTo( cur_block+1 );
510 : }
511 :
512 : // now attempt to determine if the region of blocks (from current_start
513 : // to cur_block are contiguous
514 0 : uint64 read_start = GetBlockIndexInSegment(current_start);
515 0 : uint64 read_cur = read_start * block_size;
516 0 : unsigned int count_to_read = 1; // we'll read at least one block
517 0 : while (read_cur + block_size ==
518 : (uint64)GetBlockIndexInSegment(count_to_read + current_start) * block_size && // compare count of blocks * offset with stored offset
519 : count_to_read < (cur_block - current_start) ) // make sure we don't try to read more blocks than we determined fell in
520 : // this segment
521 : {
522 0 : read_cur += block_size;
523 0 : count_to_read++;
524 : }
525 :
526 : #if 0
527 : // Check if we need to grow the virtual file for each of these blocks
528 : for (unsigned int i = 0 ; i < count_to_read; i++) {
529 : GrowVirtualFile(i + current_start);
530 : }
531 :
532 : printf("Coalescing the read of %d blocks\n", count_to_read);
533 : #endif
534 :
535 : // Perform the actual read
536 : PCIDSKSegment *data_seg_obj =
537 0 : file->GetSegment( cur_segment );
538 :
539 0 : std::size_t data_size = block_size * count_to_read;
540 :
541 : #if 0
542 : printf("Reading %d bytes at offset %d in buffer\n", data_size, buffer_off);
543 : #endif
544 :
545 : data_seg_obj->ReadFromFile( ((uint8*)buffer) + buffer_off,
546 : block_size * read_start,
547 0 : data_size );
548 :
549 0 : buffer_off += data_size; // increase buffer offset
550 :
551 : // Increment the current start by the number of blocks we jsut read
552 0 : current_start += count_to_read;
553 0 : blocks_read += count_to_read;
554 0 : }
555 0 : }
556 :
557 : /************************************************************************/
558 : /* LoadBMEntryTo() */
559 : /* */
560 : /* We load the blockmap "as needed". This method fills in */
561 : /* blockmap entries up to the target block, if not already */
562 : /* loaded. Passing in a target block_index of -1 loads the */
563 : /* whole blockmap. It is harmless to request more blocks than */
564 : /* are available. */
565 : /************************************************************************/
566 :
567 36 : void SysVirtualFile::LoadBMEntrysTo( int target_index )
568 :
569 : {
570 36 : if( target_index > 0 )
571 : {
572 4 : target_index += 200 - (target_index%200);
573 : }
574 :
575 82 : while( (target_index == -1 || blocks_loaded <= target_index )
576 : && next_bm_entry_to_load != -1 )
577 : {
578 : uint16 segment;
579 : int block;
580 :
581 10 : last_bm_index = next_bm_entry_to_load;
582 : next_bm_entry_to_load =
583 : sysblockmap->GetNextBlockMapEntry(
584 10 : next_bm_entry_to_load, segment, block );
585 :
586 10 : SetBlockInfo( blocks_loaded, segment, block );
587 : }
588 36 : }
589 :
|
