1 : /******************************************************************************
2 : *
3 : * Purpose: Implementation of the SysVirtualFile class.
4 : *
5 : * This class is used to manage access to a single virtual file stored in
6 : * SysBData segments based on a block map stored in the SysBMDir segment
7 : * (and managed by SysBlockMap class).
8 : *
9 : * The virtual files are allocated in 8K chunks (block_size) in segments.
10 : * To minimize IO requests, other overhead, we keep one such 8K block in
11 : * our working cache for the virtual file stream.
12 : *
13 : * This class is primarily used by the CTiledChannel class for access to
14 : * tiled images.
15 : *
16 : ******************************************************************************
17 : * Copyright (c) 2009
18 : * PCI Geomatics, 50 West Wilmot Street, Richmond Hill, Ont, Canada
19 : *
20 : * Permission is hereby granted, free of charge, to any person obtaining a
21 : * copy of this software and associated documentation files (the "Software"),
22 : * to deal in the Software without restriction, including without limitation
23 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
24 : * and/or sell copies of the Software, and to permit persons to whom the
25 : * Software is furnished to do so, subject to the following conditions:
26 : *
27 : * The above copyright notice and this permission notice shall be included
28 : * in all copies or substantial portions of the Software.
29 : *
30 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
31 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
32 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
33 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
34 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
35 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
36 : * DEALINGS IN THE SOFTWARE.
37 : ****************************************************************************/
38 : #include "pcidsk_config.h"
39 : #include "pcidsk_types.h"
40 : #include "pcidsk_buffer.h"
41 : #include "pcidsk_exception.h"
42 : #include "core/sysvirtualfile.h"
43 : #include "core/cpcidskfile.h"
44 : #include "segment/sysblockmap.h"
45 : #include <cassert>
46 : #include <cstring>
47 : #if 0
48 : #include <cstdio>
49 : #endif
50 :
51 : using namespace PCIDSK;
52 :
53 :
54 : const int SysVirtualFile::block_size = SYSVIRTUALFILE_BLOCKSIZE;
55 :
56 : /************************************************************************/
57 : /* SysVirtualFile() */
58 : /************************************************************************/
59 :
60 : SysVirtualFile::SysVirtualFile( CPCIDSKFile *file, int start_block,
61 : uint64 image_length,
62 : PCIDSKBuffer &block_map_data,
63 : SysBlockMap *sysblockmap,
64 3 : int image_index )
65 :
66 : {
67 3 : file_length = image_length;
68 3 : this->file = file;
69 3 : this->sysblockmap = sysblockmap;
70 3 : this->image_index = image_index;
71 :
72 3 : loaded_block = -1;
73 3 : loaded_block_dirty = false;
74 :
75 3 : last_bm_index = -1;
76 :
77 3 : int next_block = start_block;
78 :
79 : // perhaps we should defer all this work till the first request is made?
80 9 : while( next_block != -1 )
81 : {
82 3 : int offset = 512 + next_block * 28;
83 :
84 3 : block_segment.push_back( block_map_data.GetInt( offset+0, 4 ) );
85 3 : block_index.push_back( block_map_data.GetInt( offset+4, 8 ) );
86 :
87 3 : last_bm_index = next_block;
88 3 : next_block = block_map_data.GetInt( offset + 20, 8 );
89 : }
90 :
91 3 : assert( block_index.size() * block_size >= file_length );
92 3 : }
93 :
94 : /************************************************************************/
95 : /* ~SysVirtualFile() */
96 : /************************************************************************/
97 :
98 3 : SysVirtualFile::~SysVirtualFile()
99 :
100 : {
101 3 : Synchronize();
102 3 : }
103 :
104 : /************************************************************************/
105 : /* Synchronize() */
106 : /************************************************************************/
107 :
108 3 : void SysVirtualFile::Synchronize()
109 :
110 : {
111 3 : if( loaded_block_dirty )
112 : {
113 : PCIDSKSegment *data_seg_obj =
114 0 : file->GetSegment( block_segment[loaded_block] );
115 :
116 : data_seg_obj->WriteToFile( block_data,
117 : block_size * (uint64) block_index[loaded_block],
118 0 : block_size );
119 0 : loaded_block_dirty = false;
120 : }
121 3 : }
122 :
123 : /************************************************************************/
124 : /* WriteToFile() */
125 : /************************************************************************/
126 :
127 : void
128 0 : SysVirtualFile::WriteToFile( const void *buffer, uint64 offset, uint64 size )
129 :
130 : {
131 0 : uint64 buffer_offset = 0;
132 :
133 0 : while( buffer_offset < size )
134 : {
135 0 : int request_block = (int) ((offset + buffer_offset) / block_size);
136 0 : int offset_in_block = (int) ((offset + buffer_offset) % block_size);
137 0 : int amount_to_copy = block_size - offset_in_block;
138 :
139 0 : if (offset_in_block != 0 || (size - buffer_offset) < (uint64)block_size) {
140 : // we need to read in the block for update
141 0 : LoadBlock( request_block );
142 0 : if( amount_to_copy > (int) (size - buffer_offset) )
143 0 : amount_to_copy = (int) (size - buffer_offset);
144 :
145 : // fill in the block
146 : memcpy( block_data + offset_in_block,
147 : ((uint8 *) buffer) + buffer_offset,
148 0 : amount_to_copy );
149 :
150 0 : loaded_block_dirty = true;
151 : } else {
152 0 : int num_full_blocks = (size - buffer_offset) / block_size;
153 :
154 0 : WriteBlocks(request_block, num_full_blocks, (uint8*)buffer + buffer_offset);
155 :
156 0 : amount_to_copy = num_full_blocks * block_size;
157 : }
158 :
159 0 : buffer_offset += amount_to_copy;
160 : }
161 :
162 0 : if( offset+size > file_length )
163 : {
164 0 : file_length = offset+size;
165 0 : sysblockmap->SetVirtualFileSize( image_index, file_length );
166 : }
167 0 : }
168 :
169 : /************************************************************************/
170 : /* ReadFromFile() */
171 : /************************************************************************/
172 :
173 7 : void SysVirtualFile::ReadFromFile( void *buffer, uint64 offset, uint64 size )
174 :
175 : {
176 7 : uint64 buffer_offset = 0;
177 : #if 0
178 : printf("Requesting region at %llu of size %llu\n", offset, size);
179 : #endif
180 21 : while( buffer_offset < size )
181 : {
182 7 : int request_block = (int) ((offset + buffer_offset) / block_size);
183 7 : int offset_in_block = (int) ((offset + buffer_offset) % block_size);
184 7 : int amount_to_copy = block_size - offset_in_block;
185 :
186 :
187 :
188 14 : if (offset_in_block != 0 || (size - buffer_offset) < (uint64)block_size) {
189 : // Deal with the case where we need to load a partial block. Hopefully
190 : // this doesn't happen often
191 7 : LoadBlock( request_block );
192 7 : if( amount_to_copy > (int) (size - buffer_offset) )
193 7 : amount_to_copy = (int) (size - buffer_offset);
194 : #if 0
195 : printf("Requested block: %d Offset: %d copying %d bytes\n",
196 : request_block, offset_in_block, amount_to_copy);
197 : #endif
198 : memcpy( ((uint8 *) buffer) + buffer_offset,
199 7 : block_data + offset_in_block, amount_to_copy );
200 : } else {
201 : // Use the bulk loading of blocks. First, compute the range
202 : // of full blocks we need to load
203 0 : int num_full_blocks = (size - buffer_offset)/block_size;
204 :
205 : #if 0
206 : printf("Attempting a coalesced read of %d blocks (from %d) in buffer at %d\n",
207 : num_full_blocks, request_block, buffer_offset);
208 : #endif
209 :
210 0 : LoadBlocks(request_block, num_full_blocks, ((uint8*)buffer) + buffer_offset);
211 0 : amount_to_copy = num_full_blocks * block_size;
212 : }
213 :
214 :
215 7 : buffer_offset += amount_to_copy;
216 : }
217 7 : }
218 :
219 : /************************************************************************/
220 : /* LoadBlock() */
221 : /************************************************************************/
222 : /**
223 : * Loads the requested_block block from the system virtual file. Extends
224 : * the file if necessary
225 : */
226 7 : void SysVirtualFile::LoadBlock( int requested_block )
227 :
228 : {
229 : /* -------------------------------------------------------------------- */
230 : /* Do we already have this block? */
231 : /* -------------------------------------------------------------------- */
232 7 : if( requested_block == loaded_block )
233 4 : return;
234 :
235 : /* -------------------------------------------------------------------- */
236 : /* Do we need to grow the virtual file by one block? */
237 : /* -------------------------------------------------------------------- */
238 3 : GrowVirtualFile(requested_block);
239 :
240 : /* -------------------------------------------------------------------- */
241 : /* Does this block exist in the virtual file? */
242 : /* -------------------------------------------------------------------- */
243 3 : if( requested_block < 0 || requested_block >= (int) block_index.size() )
244 : ThrowPCIDSKException( "SysVirtualFile::LoadBlock(%d) - block out of range.",
245 0 : requested_block );
246 :
247 : /* -------------------------------------------------------------------- */
248 : /* Do we have a dirty block loaded that needs to be saved? */
249 : /* -------------------------------------------------------------------- */
250 3 : FlushDirtyBlock();
251 :
252 : /* -------------------------------------------------------------------- */
253 : /* Load the requested block. */
254 : /* -------------------------------------------------------------------- */
255 : PCIDSKSegment *data_seg_obj =
256 3 : file->GetSegment( block_segment[requested_block] );
257 :
258 : data_seg_obj->ReadFromFile( block_data,
259 : block_size * (uint64) block_index[requested_block],
260 3 : block_size );
261 :
262 3 : loaded_block = requested_block;
263 3 : loaded_block_dirty = false;
264 : }
265 :
266 : /************************************************************************/
267 : /* FlushDirtyBlock() */
268 : /************************************************************************/
269 : /**
270 : * If the block currently loaded is dirty, flush it to the file
271 : */
272 3 : void SysVirtualFile::FlushDirtyBlock(void)
273 : {
274 3 : if (loaded_block_dirty) {
275 : PCIDSKSegment *data_seg_obj =
276 0 : file->GetSegment( block_segment[loaded_block] );
277 :
278 : data_seg_obj->WriteToFile( block_data,
279 : block_size * (uint64) block_index[loaded_block],
280 0 : block_size );
281 0 : loaded_block_dirty = false;
282 : }
283 3 : }
284 :
285 3 : void SysVirtualFile::GrowVirtualFile(std::ptrdiff_t requested_block)
286 : {
287 3 : if( requested_block == (int) block_index.size() )
288 : {
289 : int new_seg;
290 :
291 : block_index.push_back(
292 : sysblockmap->GrowVirtualFile( image_index,
293 0 : last_bm_index, new_seg ) );
294 0 : block_segment.push_back( new_seg );
295 : }
296 3 : }
297 :
298 : /**
299 : * \brief Writes a group of blocks
300 : * Attempts to create a group of blocks (if the SysVirtualFile
301 : * is not already large enough to hold them) and then write them
302 : * out contiguously
303 : */
304 : void SysVirtualFile::WriteBlocks(int first_block,
305 : int block_count,
306 0 : void* const buffer)
307 : {
308 0 : FlushDirtyBlock();
309 : // Iterate through all the blocks to be written, first, then
310 : // grow the virtual file
311 0 : for (unsigned int i = 0; i <= (unsigned int) block_count; i++) {
312 0 : GrowVirtualFile(first_block + i);
313 : }
314 :
315 : // Using a similar algorithm to the LoadBlocks() function,
316 : // attempt to coalesce writes
317 0 : std::size_t buffer_off = 0;
318 0 : std::size_t blocks_written = 0;
319 0 : std::size_t current_first_block = first_block;
320 0 : while (blocks_written < (std::size_t) block_count) {
321 0 : unsigned int cur_segment = block_segment[current_first_block];
322 0 : unsigned int cur_block = current_first_block;
323 0 : while (cur_block < (unsigned int)block_count + first_block &&
324 : (unsigned int)block_segment[cur_block + 1] == cur_segment)
325 : {
326 0 : cur_block++;
327 : }
328 :
329 : // Find largest span of contiguous blocks we can write
330 0 : uint64 write_start = block_index[current_first_block];
331 0 : uint64 write_cur = write_start * block_size;
332 0 : unsigned int count_to_write = 1;
333 0 : while (write_cur + block_size ==
334 : (uint64)block_index[count_to_write + current_first_block - 1] * block_size &&
335 : count_to_write < (cur_block - current_first_block))
336 : {
337 0 : write_cur += block_size;
338 0 : count_to_write++;
339 : }
340 :
341 : PCIDSKSegment *data_seg_obj =
342 0 : file->GetSegment( cur_segment );
343 :
344 0 : std::size_t bytes_to_write = count_to_write * block_size;
345 :
346 : data_seg_obj->WriteToFile((uint8*)buffer + buffer_off,
347 : block_size * write_start,
348 0 : bytes_to_write);
349 :
350 0 : buffer_off += bytes_to_write;
351 0 : blocks_written += count_to_write;
352 0 : current_first_block += count_to_write;
353 : }
354 0 : }
355 :
356 : /**
357 : * \brief Load a group of blocks
358 : * Attempts to coalesce reading of groups of blocks into a single
359 : * filesystem I/O operation. Does not cache the loaded block, nor
360 : * does it modify the state of the SysVirtualFile, other than to
361 : * flush the loaded block if it is dirty.
362 : */
363 : void SysVirtualFile::LoadBlocks(int requested_block_start,
364 : int requested_block_count,
365 0 : void* const buffer)
366 : {
367 0 : FlushDirtyBlock();
368 0 : unsigned int blocks_read = 0;
369 0 : unsigned int current_start = requested_block_start;
370 :
371 0 : std::size_t buffer_off = 0;
372 :
373 0 : while (blocks_read < (unsigned int)requested_block_count) {
374 : // Coalesce blocks that are in the same segment
375 0 : unsigned int cur_segment = block_segment[current_start]; // segment of current
376 : // first block
377 0 : unsigned int cur_block = current_start; // starting block ID
378 0 : while (cur_block < (unsigned int)requested_block_count + requested_block_start &&
379 : (unsigned int)block_segment[cur_block + 1] == cur_segment) {
380 : // this block is in the same segment as the previous one we
381 : // wanted to read.
382 0 : cur_block++;
383 : }
384 :
385 : // now attempt to determine if the region of blocks (from current_start
386 : // to cur_block are contiguous
387 0 : uint64 read_start = block_index[current_start];
388 0 : uint64 read_cur = read_start * block_size;
389 0 : unsigned int count_to_read = 1; // we'll read at least one block
390 0 : while (read_cur + block_size ==
391 : (uint64)block_index[count_to_read + current_start] * block_size && // compare count of blocks * offset with stored offset
392 : count_to_read < (cur_block - current_start) ) // make sure we don't try to read more blocks than we determined fell in
393 : // this segment
394 : {
395 0 : read_cur += block_size;
396 0 : count_to_read++;
397 : }
398 :
399 : #if 0
400 :
401 : // Check if we need to grow the virtual file for each of these blocks
402 : for (unsigned int i = 0 ; i < count_to_read; i++) {
403 : GrowVirtualFile(i + current_start);
404 : }
405 :
406 :
407 : printf("Coalescing the read of %d blocks\n", count_to_read);
408 : #endif
409 :
410 : // Perform the actual read
411 : PCIDSKSegment *data_seg_obj =
412 0 : file->GetSegment( cur_segment );
413 :
414 0 : std::size_t data_size = block_size * count_to_read;
415 :
416 : #if 0
417 : printf("Reading %d bytes at offset %d in buffer\n", data_size, buffer_off);
418 : #endif
419 :
420 : data_seg_obj->ReadFromFile( ((uint8*)buffer) + buffer_off,
421 : block_size * read_start,
422 0 : data_size );
423 :
424 0 : buffer_off += data_size; // increase buffer offset
425 :
426 : // Increment the current start by the number of blocks we jsut read
427 0 : current_start += count_to_read;
428 0 : blocks_read += count_to_read;
429 : }
430 0 : }
431 :
|
