1 : /******************************************************************************
2 : * $Id: cpl_vsil_curl_streaming.cpp 25787 2013-03-23 13:59:41Z rouault $
3 : *
4 : * Project: CPL - Common Portability Library
5 : * Purpose: Implement VSI large file api for HTTP/FTP files in streaming mode
6 : * Author: Even Rouault <even dot rouault at mines dash paris.org>
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2012, Even Rouault <even dot rouault at mines dash paris.org>
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_string.h"
32 : #include "cpl_multiproc.h"
33 : #include "cpl_hash_set.h"
34 : #include "cpl_time.h"
35 :
36 : CPL_CVSID("$Id: cpl_vsil_curl_streaming.cpp 25787 2013-03-23 13:59:41Z rouault $");
37 :
38 : #if !defined(HAVE_CURL) || defined(CPL_MULTIPROC_STUB)
39 :
40 : void VSIInstallCurlStreamingFileHandler(void)
41 : {
42 : /* not supported */
43 : }
44 :
45 : #else
46 :
47 : #include <curl/curl.h>
48 :
49 : void VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL);
50 :
51 : #include <map>
52 :
53 : #define ENABLE_DEBUG 0
54 :
55 : #define N_MAX_REGIONS 10
56 :
57 : #define BKGND_BUFFER_SIZE (1024 * 1024)
58 :
59 :
60 : /************************************************************************/
61 : /* RingBuffer */
62 : /************************************************************************/
63 :
64 : class RingBuffer
65 : {
66 : GByte* pabyBuffer;
67 : size_t nCapacity;
68 : size_t nOffset;
69 : size_t nLength;
70 :
71 : public:
72 : RingBuffer(size_t nCapacity = BKGND_BUFFER_SIZE);
73 : ~RingBuffer();
74 :
75 929 : size_t GetCapacity() const { return nCapacity; }
76 6253 : size_t GetSize() const { return nLength; }
77 :
78 : void Reset();
79 : void Write(void* pBuffer, size_t nSize);
80 : void Read(void* pBuffer, size_t nSize);
81 : };
82 :
83 48 : RingBuffer::RingBuffer(size_t nCapacityIn)
84 : {
85 48 : pabyBuffer = (GByte*)CPLMalloc(nCapacityIn);
86 48 : nCapacity = nCapacityIn;
87 48 : nOffset = 0;
88 48 : nLength = 0;
89 48 : }
90 :
91 48 : RingBuffer::~RingBuffer()
92 : {
93 48 : CPLFree(pabyBuffer);
94 48 : }
95 :
96 97 : void RingBuffer::Reset()
97 : {
98 97 : nOffset = 0;
99 97 : nLength = 0;
100 97 : }
101 :
102 927 : void RingBuffer::Write(void* pBuffer, size_t nSize)
103 : {
104 927 : CPLAssert(nLength + nSize <= nCapacity);
105 :
106 927 : size_t nEndOffset = (nOffset + nLength) % nCapacity;
107 927 : size_t nSz = MIN(nSize, nCapacity - nEndOffset);
108 927 : memcpy(pabyBuffer + nEndOffset, pBuffer, nSz);
109 927 : if (nSz < nSize)
110 2 : memcpy(pabyBuffer, (GByte*)pBuffer + nSz, nSize - nSz);
111 :
112 927 : nLength += nSize;
113 927 : }
114 :
115 1622 : void RingBuffer::Read(void* pBuffer, size_t nSize)
116 : {
117 1622 : CPLAssert(nSize <= nLength);
118 :
119 1622 : if (pBuffer)
120 : {
121 1622 : size_t nSz = MIN(nSize, nCapacity - nOffset);
122 1622 : memcpy(pBuffer, pabyBuffer + nOffset, nSz);
123 1622 : if (nSz < nSize)
124 1 : memcpy((GByte*)pBuffer + nSz, pabyBuffer, nSize - nSz);
125 : }
126 :
127 1622 : nOffset = (nOffset + nSize) % nCapacity;
128 1622 : nLength -= nSize;
129 1622 : }
130 :
131 : /************************************************************************/
132 :
133 : typedef enum
134 : {
135 : EXIST_UNKNOWN = -1,
136 : EXIST_NO,
137 : EXIST_YES,
138 : } ExistStatus;
139 :
140 : typedef struct
141 : {
142 : ExistStatus eExists;
143 : int bHastComputedFileSize;
144 : vsi_l_offset fileSize;
145 : int bIsDirectory;
146 : #ifdef notdef
147 : unsigned int nChecksumOfFirst1024Bytes;
148 : #endif
149 : } CachedFileProp;
150 :
151 : /************************************************************************/
152 : /* VSICurlStreamingFSHandler */
153 : /************************************************************************/
154 :
155 : class VSICurlStreamingFSHandler : public VSIFilesystemHandler
156 : {
157 : void *hMutex;
158 :
159 : std::map<CPLString, CachedFileProp*> cacheFileSize;
160 :
161 : public:
162 : VSICurlStreamingFSHandler();
163 : ~VSICurlStreamingFSHandler();
164 :
165 : virtual VSIVirtualHandle *Open( const char *pszFilename,
166 : const char *pszAccess);
167 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
168 :
169 : void AcquireMutex();
170 : void ReleaseMutex();
171 :
172 : CachedFileProp* GetCachedFileProp(const char* pszURL);
173 : };
174 :
175 : /************************************************************************/
176 : /* VSICurlStreamingHandle */
177 : /************************************************************************/
178 :
179 : class VSICurlStreamingHandle : public VSIVirtualHandle
180 : {
181 : private:
182 : VSICurlStreamingFSHandler* poFS;
183 :
184 : char* pszURL;
185 :
186 : #ifdef notdef
187 : unsigned int nRecomputedChecksumOfFirst1024Bytes;
188 : #endif
189 : vsi_l_offset curOffset;
190 : vsi_l_offset fileSize;
191 : int bHastComputedFileSize;
192 : ExistStatus eExists;
193 : int bIsDirectory;
194 :
195 : int bCanTrustCandidateFileSize;
196 : int bHasCandidateFileSize;
197 : vsi_l_offset nCandidateFileSize;
198 :
199 : int bEOF;
200 :
201 : size_t nCachedSize;
202 : GByte *pCachedData;
203 :
204 : CURL* hCurlHandle;
205 :
206 : volatile int bDownloadInProgress;
207 : volatile int bDownloadStopped;
208 : volatile int bAskDownloadEnd;
209 : vsi_l_offset nRingBufferFileOffset;
210 : void *hThread;
211 : void *hRingBufferMutex;
212 : void *hCondProducer;
213 : void *hCondConsumer;
214 : RingBuffer oRingBuffer;
215 : void StartDownload();
216 : void StopDownload();
217 : void PutRingBufferInCache();
218 :
219 : GByte *pabyHeaderData;
220 : size_t nHeaderSize;
221 : vsi_l_offset nBodySize;
222 : int nHTTPCode;
223 :
224 : void AcquireMutex();
225 : void ReleaseMutex();
226 :
227 : void AddRegion( vsi_l_offset nFileOffsetStart,
228 : size_t nSize,
229 : GByte *pData );
230 : public:
231 :
232 : VSICurlStreamingHandle(VSICurlStreamingFSHandler* poFS, const char* pszURL);
233 : ~VSICurlStreamingHandle();
234 :
235 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
236 : virtual vsi_l_offset Tell();
237 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
238 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
239 : virtual int Eof();
240 : virtual int Flush();
241 : virtual int Close();
242 :
243 : void DownloadInThread();
244 : int ReceivedBytes(GByte *buffer, size_t count, size_t nmemb);
245 : int ReceivedBytesHeader(GByte *buffer, size_t count, size_t nmemb);
246 :
247 0 : int IsKnownFileSize() const { return bHastComputedFileSize; }
248 : vsi_l_offset GetFileSize();
249 : int Exists();
250 0 : int IsDirectory() const { return bIsDirectory; }
251 : };
252 :
253 : /************************************************************************/
254 : /* VSICurlStreamingHandle() */
255 : /************************************************************************/
256 :
257 48 : VSICurlStreamingHandle::VSICurlStreamingHandle(VSICurlStreamingFSHandler* poFS, const char* pszURL)
258 : {
259 48 : this->poFS = poFS;
260 48 : this->pszURL = CPLStrdup(pszURL);
261 :
262 : #ifdef notdef
263 : nRecomputedChecksumOfFirst1024Bytes = 0;
264 : #endif
265 48 : curOffset = 0;
266 :
267 48 : poFS->AcquireMutex();
268 48 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
269 48 : eExists = cachedFileProp->eExists;
270 48 : fileSize = cachedFileProp->fileSize;
271 48 : bHastComputedFileSize = cachedFileProp->bHastComputedFileSize;
272 48 : bIsDirectory = cachedFileProp->bIsDirectory;
273 48 : poFS->ReleaseMutex();
274 :
275 48 : bCanTrustCandidateFileSize = TRUE;
276 48 : bHasCandidateFileSize = FALSE;
277 48 : nCandidateFileSize = 0;
278 :
279 48 : nCachedSize = 0;
280 48 : pCachedData = NULL;
281 :
282 48 : bEOF = FALSE;
283 :
284 48 : hCurlHandle = NULL;
285 :
286 48 : hThread = NULL;
287 48 : hRingBufferMutex = CPLCreateMutex();
288 48 : ReleaseMutex();
289 48 : hCondProducer = CPLCreateCond();
290 48 : hCondConsumer = CPLCreateCond();
291 :
292 48 : bDownloadInProgress = FALSE;
293 48 : bDownloadStopped = FALSE;
294 48 : bAskDownloadEnd = FALSE;
295 48 : nRingBufferFileOffset = 0;
296 :
297 48 : pabyHeaderData = NULL;
298 48 : nHeaderSize = 0;
299 48 : nBodySize = 0;
300 48 : nHTTPCode = 0;
301 48 : }
302 :
303 : /************************************************************************/
304 : /* ~VSICurlStreamingHandle() */
305 : /************************************************************************/
306 :
307 48 : VSICurlStreamingHandle::~VSICurlStreamingHandle()
308 : {
309 48 : StopDownload();
310 :
311 48 : CPLFree(pszURL);
312 48 : if (hCurlHandle != NULL)
313 0 : curl_easy_cleanup(hCurlHandle);
314 :
315 48 : CPLFree(pCachedData);
316 :
317 48 : CPLFree(pabyHeaderData);
318 :
319 48 : CPLDestroyMutex( hRingBufferMutex );
320 48 : CPLDestroyCond( hCondProducer );
321 48 : CPLDestroyCond( hCondConsumer );
322 48 : }
323 :
324 : /************************************************************************/
325 : /* AcquireMutex() */
326 : /************************************************************************/
327 :
328 4127 : void VSICurlStreamingHandle::AcquireMutex()
329 : {
330 4127 : CPLAcquireMutex(hRingBufferMutex, 1000.0);
331 4127 : }
332 :
333 : /************************************************************************/
334 : /* ReleaseMutex() */
335 : /************************************************************************/
336 :
337 4175 : void VSICurlStreamingHandle::ReleaseMutex()
338 : {
339 4175 : CPLReleaseMutex(hRingBufferMutex);
340 4175 : }
341 :
342 : /************************************************************************/
343 : /* Seek() */
344 : /************************************************************************/
345 :
346 143 : int VSICurlStreamingHandle::Seek( vsi_l_offset nOffset, int nWhence )
347 : {
348 143 : if( curOffset >= BKGND_BUFFER_SIZE )
349 : {
350 : if (ENABLE_DEBUG)
351 : CPLDebug("VSICURL", "Invalidating cache and file size due to Seek() beyond caching zone");
352 1 : CPLFree(pCachedData);
353 1 : pCachedData = NULL;
354 1 : nCachedSize = 0;
355 1 : AcquireMutex();
356 1 : bHastComputedFileSize = FALSE;
357 1 : fileSize = 0;
358 1 : ReleaseMutex();
359 : }
360 :
361 143 : if (nWhence == SEEK_SET)
362 : {
363 142 : curOffset = nOffset;
364 : }
365 1 : else if (nWhence == SEEK_CUR)
366 : {
367 0 : curOffset = curOffset + nOffset;
368 : }
369 : else
370 : {
371 1 : curOffset = GetFileSize() + nOffset;
372 : }
373 143 : bEOF = FALSE;
374 143 : return 0;
375 : }
376 :
377 : typedef struct
378 : {
379 : char* pBuffer;
380 : size_t nSize;
381 : int bIsHTTP;
382 : int bIsInHeader;
383 : int nHTTPCode;
384 : int bDownloadHeaderOnly;
385 : } WriteFuncStruct;
386 :
387 : /************************************************************************/
388 : /* VSICURLStreamingInitWriteFuncStruct() */
389 : /************************************************************************/
390 :
391 0 : static void VSICURLStreamingInitWriteFuncStruct(WriteFuncStruct *psStruct)
392 : {
393 0 : psStruct->pBuffer = NULL;
394 0 : psStruct->nSize = 0;
395 0 : psStruct->bIsHTTP = FALSE;
396 0 : psStruct->bIsInHeader = TRUE;
397 0 : psStruct->nHTTPCode = 0;
398 0 : psStruct->bDownloadHeaderOnly = FALSE;
399 0 : }
400 :
401 : /************************************************************************/
402 : /* VSICurlStreamingHandleWriteFuncForHeader() */
403 : /************************************************************************/
404 :
405 0 : static int VSICurlStreamingHandleWriteFuncForHeader(void *buffer, size_t count, size_t nmemb, void *req)
406 : {
407 0 : WriteFuncStruct* psStruct = (WriteFuncStruct*) req;
408 0 : size_t nSize = count * nmemb;
409 :
410 : char* pNewBuffer = (char*) VSIRealloc(psStruct->pBuffer,
411 0 : psStruct->nSize + nSize + 1);
412 0 : if (pNewBuffer)
413 : {
414 0 : psStruct->pBuffer = pNewBuffer;
415 0 : memcpy(psStruct->pBuffer + psStruct->nSize, buffer, nSize);
416 0 : psStruct->pBuffer[psStruct->nSize + nSize] = '\0';
417 0 : if (psStruct->bIsHTTP && psStruct->bIsInHeader)
418 : {
419 0 : char* pszLine = psStruct->pBuffer + psStruct->nSize;
420 0 : if (EQUALN(pszLine, "HTTP/1.0 ", 9) ||
421 : EQUALN(pszLine, "HTTP/1.1 ", 9))
422 0 : psStruct->nHTTPCode = atoi(pszLine + 9);
423 :
424 0 : if (pszLine[0] == '\r' || pszLine[0] == '\n')
425 : {
426 0 : if (psStruct->bDownloadHeaderOnly)
427 : {
428 : /* If moved permanently/temporarily, go on. Otherwise stop now*/
429 0 : if (!(psStruct->nHTTPCode == 301 || psStruct->nHTTPCode == 302))
430 0 : return 0;
431 : }
432 : else
433 : {
434 0 : psStruct->bIsInHeader = FALSE;
435 : }
436 : }
437 : }
438 0 : psStruct->nSize += nSize;
439 0 : return nmemb;
440 : }
441 : else
442 : {
443 0 : return 0;
444 : }
445 : }
446 :
447 :
448 : /************************************************************************/
449 : /* GetFileSize() */
450 : /************************************************************************/
451 :
452 1 : vsi_l_offset VSICurlStreamingHandle::GetFileSize()
453 : {
454 : WriteFuncStruct sWriteFuncData;
455 : WriteFuncStruct sWriteFuncHeaderData;
456 :
457 1 : AcquireMutex();
458 1 : if (bHastComputedFileSize)
459 : {
460 1 : vsi_l_offset nRet = fileSize;
461 1 : ReleaseMutex();
462 1 : return nRet;
463 : }
464 0 : ReleaseMutex();
465 :
466 : #if LIBCURL_VERSION_NUM < 0x070B00
467 : /* Curl 7.10.X doesn't manage to unset the CURLOPT_RANGE that would have been */
468 : /* previously set, so we have to reinit the connection handle */
469 : if (hCurlHandle)
470 : {
471 : curl_easy_cleanup(hCurlHandle);
472 : hCurlHandle = curl_easy_init();
473 : }
474 : #endif
475 :
476 0 : CURL* hLocalHandle = curl_easy_init();
477 :
478 0 : VSICurlSetOptions(hLocalHandle, pszURL);
479 :
480 0 : VSICURLStreamingInitWriteFuncStruct(&sWriteFuncHeaderData);
481 :
482 : /* HACK for mbtiles driver: proper fix would be to auto-detect servers that don't accept HEAD */
483 : /* http://a.tiles.mapbox.com/v3/ doesn't accept HEAD, so let's start a GET */
484 : /* and interrupt is as soon as the header is found */
485 0 : if (strstr(pszURL, ".tiles.mapbox.com/") != NULL)
486 : {
487 0 : curl_easy_setopt(hLocalHandle, CURLOPT_HEADERDATA, &sWriteFuncHeaderData);
488 0 : curl_easy_setopt(hLocalHandle, CURLOPT_HEADERFUNCTION, VSICurlStreamingHandleWriteFuncForHeader);
489 :
490 0 : sWriteFuncHeaderData.bIsHTTP = strncmp(pszURL, "http", 4) == 0;
491 0 : sWriteFuncHeaderData.bDownloadHeaderOnly = TRUE;
492 : }
493 : else
494 : {
495 0 : curl_easy_setopt(hLocalHandle, CURLOPT_NOBODY, 1);
496 0 : curl_easy_setopt(hLocalHandle, CURLOPT_HTTPGET, 0);
497 0 : curl_easy_setopt(hLocalHandle, CURLOPT_HEADER, 1);
498 : }
499 :
500 : /* We need that otherwise OSGEO4W's libcurl issue a dummy range request */
501 : /* when doing a HEAD when recycling connections */
502 0 : curl_easy_setopt(hLocalHandle, CURLOPT_RANGE, NULL);
503 :
504 : /* Bug with older curl versions (<=7.16.4) and FTP. See http://curl.haxx.se/mail/lib-2007-08/0312.html */
505 0 : VSICURLStreamingInitWriteFuncStruct(&sWriteFuncData);
506 0 : curl_easy_setopt(hLocalHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
507 0 : curl_easy_setopt(hLocalHandle, CURLOPT_WRITEFUNCTION, VSICurlStreamingHandleWriteFuncForHeader);
508 :
509 : char szCurlErrBuf[CURL_ERROR_SIZE+1];
510 0 : szCurlErrBuf[0] = '\0';
511 0 : curl_easy_setopt(hLocalHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
512 :
513 0 : double dfSize = 0;
514 0 : curl_easy_perform(hLocalHandle);
515 :
516 0 : AcquireMutex();
517 :
518 0 : eExists = EXIST_UNKNOWN;
519 0 : bHastComputedFileSize = TRUE;
520 :
521 0 : if (strncmp(pszURL, "ftp", 3) == 0)
522 : {
523 0 : if (sWriteFuncData.pBuffer != NULL &&
524 : strncmp(sWriteFuncData.pBuffer, "Content-Length: ", strlen( "Content-Length: ")) == 0)
525 : {
526 0 : const char* pszBuffer = sWriteFuncData.pBuffer + strlen("Content-Length: ");
527 0 : eExists = EXIST_YES;
528 0 : fileSize = CPLScanUIntBig(pszBuffer, sWriteFuncData.nSize - strlen("Content-Length: "));
529 0 : if (ENABLE_DEBUG)
530 : CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB,
531 : pszURL, fileSize);
532 : }
533 : }
534 :
535 0 : if (eExists != EXIST_YES)
536 : {
537 0 : CURLcode code = curl_easy_getinfo(hLocalHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dfSize );
538 0 : if (code == 0)
539 : {
540 0 : eExists = EXIST_YES;
541 0 : if (dfSize < 0)
542 0 : fileSize = 0;
543 : else
544 0 : fileSize = (GUIntBig)dfSize;
545 : }
546 : else
547 : {
548 0 : eExists = EXIST_NO;
549 0 : fileSize = 0;
550 : CPLError(CE_Failure, CPLE_AppDefined,
551 0 : "VSICurlStreamingHandle::GetFileSize failed");
552 : }
553 :
554 0 : long response_code = 0;
555 0 : curl_easy_getinfo(hLocalHandle, CURLINFO_HTTP_CODE, &response_code);
556 0 : if (response_code != 200)
557 : {
558 0 : eExists = EXIST_NO;
559 0 : fileSize = 0;
560 : }
561 :
562 : /* Try to guess if this is a directory. Generally if this is a directory, */
563 : /* curl will retry with an URL with slash added */
564 0 : char *pszEffectiveURL = NULL;
565 0 : curl_easy_getinfo(hLocalHandle, CURLINFO_EFFECTIVE_URL, &pszEffectiveURL);
566 0 : if (pszEffectiveURL != NULL &&
567 : strncmp(pszURL, pszEffectiveURL, strlen(pszURL)) == 0 &&
568 0 : pszEffectiveURL[strlen(pszURL)] == '/')
569 : {
570 0 : eExists = EXIST_YES;
571 0 : fileSize = 0;
572 0 : bIsDirectory = TRUE;
573 : }
574 :
575 0 : if (ENABLE_DEBUG)
576 : CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d",
577 : pszURL, fileSize, (int)response_code);
578 : }
579 :
580 0 : CPLFree(sWriteFuncData.pBuffer);
581 0 : CPLFree(sWriteFuncHeaderData.pBuffer);
582 :
583 0 : poFS->AcquireMutex();
584 0 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
585 0 : cachedFileProp->bHastComputedFileSize = TRUE;
586 : #ifdef notdef
587 : cachedFileProp->nChecksumOfFirst1024Bytes = nRecomputedChecksumOfFirst1024Bytes;
588 : #endif
589 0 : cachedFileProp->fileSize = fileSize;
590 0 : cachedFileProp->eExists = eExists;
591 0 : cachedFileProp->bIsDirectory = bIsDirectory;
592 0 : poFS->ReleaseMutex();
593 :
594 0 : vsi_l_offset nRet = fileSize;
595 0 : ReleaseMutex();
596 :
597 0 : if (hCurlHandle == NULL)
598 0 : hCurlHandle = hLocalHandle;
599 : else
600 0 : curl_easy_cleanup(hLocalHandle);
601 :
602 0 : return nRet;
603 : }
604 :
605 : /************************************************************************/
606 : /* Exists() */
607 : /************************************************************************/
608 :
609 48 : int VSICurlStreamingHandle::Exists()
610 : {
611 48 : if (eExists == EXIST_UNKNOWN)
612 : {
613 : /* Consider that only the files whose extension ends up with one that is */
614 : /* listed in CPL_VSIL_CURL_ALLOWED_EXTENSIONS exist on the server */
615 : /* This can speeds up dramatically open experience, in case the server */
616 : /* cannot return a file list */
617 : /* For example : */
618 : /* gdalinfo --config CPL_VSIL_CURL_ALLOWED_EXTENSIONS ".tif" /vsicurl_streaming/http://igskmncngs506.cr.usgs.gov/gmted/Global_tiles_GMTED/075darcsec/bln/W030/30N030W_20101117_gmted_bln075.tif */
619 : const char* pszAllowedExtensions =
620 28 : CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", NULL);
621 28 : if (pszAllowedExtensions)
622 : {
623 0 : char** papszExtensions = CSLTokenizeString2( pszAllowedExtensions, ", ", 0 );
624 0 : int nURLLen = strlen(pszURL);
625 0 : int bFound = FALSE;
626 0 : for(int i=0;papszExtensions[i] != NULL;i++)
627 : {
628 0 : int nExtensionLen = strlen(papszExtensions[i]);
629 0 : if (nURLLen > nExtensionLen &&
630 0 : EQUAL(pszURL + nURLLen - nExtensionLen, papszExtensions[i]))
631 : {
632 0 : bFound = TRUE;
633 0 : break;
634 : }
635 : }
636 :
637 0 : if (!bFound)
638 : {
639 0 : eExists = EXIST_NO;
640 0 : fileSize = 0;
641 :
642 0 : poFS->AcquireMutex();
643 0 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
644 0 : cachedFileProp->bHastComputedFileSize = TRUE;
645 0 : cachedFileProp->fileSize = fileSize;
646 0 : cachedFileProp->eExists = eExists;
647 0 : poFS->ReleaseMutex();
648 :
649 0 : CSLDestroy(papszExtensions);
650 :
651 0 : return 0;
652 : }
653 :
654 0 : CSLDestroy(papszExtensions);
655 : }
656 :
657 : char chFirstByte;
658 28 : int bExists = (Read(&chFirstByte, 1, 1) == 1);
659 :
660 28 : AcquireMutex();
661 28 : poFS->AcquireMutex();
662 28 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
663 28 : cachedFileProp->eExists = eExists = bExists ? EXIST_YES : EXIST_NO;
664 28 : poFS->ReleaseMutex();
665 28 : ReleaseMutex();
666 :
667 28 : Seek(0, SEEK_SET);
668 : }
669 :
670 48 : return eExists == EXIST_YES;
671 : }
672 :
673 : /************************************************************************/
674 : /* Tell() */
675 : /************************************************************************/
676 :
677 8 : vsi_l_offset VSICurlStreamingHandle::Tell()
678 : {
679 8 : return curOffset;
680 : }
681 :
682 : /************************************************************************/
683 : /* ReceivedBytes() */
684 : /************************************************************************/
685 :
686 928 : int VSICurlStreamingHandle::ReceivedBytes(GByte *buffer, size_t count, size_t nmemb)
687 : {
688 928 : size_t nSize = count * nmemb;
689 928 : nBodySize += nSize;
690 :
691 : if (ENABLE_DEBUG)
692 : CPLDebug("VSICURL", "Receiving %d bytes...", (int)nSize);
693 :
694 928 : if( bHasCandidateFileSize && bCanTrustCandidateFileSize && !bHastComputedFileSize )
695 : {
696 4 : poFS->AcquireMutex();
697 4 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
698 4 : cachedFileProp->fileSize = fileSize = nCandidateFileSize;
699 4 : cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
700 : if (ENABLE_DEBUG)
701 : CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
702 4 : poFS->ReleaseMutex();
703 : }
704 :
705 928 : AcquireMutex();
706 928 : if (eExists == EXIST_UNKNOWN)
707 : {
708 0 : poFS->AcquireMutex();
709 0 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
710 0 : cachedFileProp->eExists = eExists = EXIST_YES;
711 0 : poFS->ReleaseMutex();
712 : }
713 928 : else if (eExists == EXIST_NO)
714 : {
715 1 : ReleaseMutex();
716 1 : return 0;
717 : }
718 :
719 0 : while(TRUE)
720 : {
721 927 : size_t nFree = oRingBuffer.GetCapacity() - oRingBuffer.GetSize();
722 927 : if (nSize <= nFree)
723 : {
724 926 : oRingBuffer.Write(buffer, nSize);
725 :
726 : /* Signal to the consumer that we have added bytes to the buffer */
727 926 : CPLCondSignal(hCondProducer);
728 :
729 926 : if (bAskDownloadEnd)
730 : {
731 : if (ENABLE_DEBUG)
732 : CPLDebug("VSICURL", "Download interruption asked");
733 :
734 3 : ReleaseMutex();
735 3 : return 0;
736 : }
737 : break;
738 : }
739 : else
740 : {
741 1 : oRingBuffer.Write(buffer, nFree);
742 1 : buffer += nFree;
743 1 : nSize -= nFree;
744 :
745 : /* Signal to the consumer that we have added bytes to the buffer */
746 1 : CPLCondSignal(hCondProducer);
747 :
748 : if (ENABLE_DEBUG)
749 : CPLDebug("VSICURL", "Waiting for reader to consume some bytes...");
750 :
751 3 : while(oRingBuffer.GetSize() == oRingBuffer.GetCapacity() && !bAskDownloadEnd)
752 : {
753 1 : CPLCondWait(hCondConsumer, hRingBufferMutex);
754 : }
755 :
756 1 : if (bAskDownloadEnd)
757 : {
758 : if (ENABLE_DEBUG)
759 : CPLDebug("VSICURL", "Download interruption asked");
760 :
761 1 : ReleaseMutex();
762 1 : return 0;
763 : }
764 : }
765 : }
766 :
767 923 : ReleaseMutex();
768 :
769 923 : return nmemb;
770 : }
771 :
772 : /************************************************************************/
773 : /* VSICurlStreamingHandleReceivedBytes() */
774 : /************************************************************************/
775 :
776 928 : static int VSICurlStreamingHandleReceivedBytes(void *buffer, size_t count, size_t nmemb, void *req)
777 : {
778 928 : return ((VSICurlStreamingHandle*)req)->ReceivedBytes((GByte*)buffer, count, nmemb);
779 : }
780 :
781 :
782 : /************************************************************************/
783 : /* VSICurlStreamingHandleReceivedBytesHeader() */
784 : /************************************************************************/
785 :
786 : #define HEADER_SIZE 32768
787 :
788 393 : int VSICurlStreamingHandle::ReceivedBytesHeader(GByte *buffer, size_t count, size_t nmemb)
789 : {
790 393 : size_t nSize = count * nmemb;
791 : if (ENABLE_DEBUG)
792 : CPLDebug("VSICURL", "Receiving %d bytes for header...", (int)nSize);
793 :
794 : /* Reset buffer if we have followed link after a redirect */
795 393 : if (nSize >=9 && (nHTTPCode == 301 || nHTTPCode == 302) &&
796 : (EQUALN((const char*)buffer, "HTTP/1.0 ", 9) ||
797 : EQUALN((const char*)buffer, "HTTP/1.1 ", 9)))
798 : {
799 0 : nHeaderSize = 0;
800 0 : nHTTPCode = 0;
801 : }
802 :
803 393 : if (nHeaderSize < HEADER_SIZE)
804 : {
805 393 : size_t nSz = MIN(nSize, HEADER_SIZE - nHeaderSize);
806 393 : memcpy(pabyHeaderData + nHeaderSize, buffer, nSz);
807 393 : pabyHeaderData[nHeaderSize + nSz] = '\0';
808 393 : nHeaderSize += nSz;
809 :
810 : //CPLDebug("VSICURL", "Header : %s", pabyHeaderData);
811 :
812 393 : AcquireMutex();
813 :
814 393 : if (eExists == EXIST_UNKNOWN && nHTTPCode == 0 &&
815 : strchr((const char*)pabyHeaderData, '\n') != NULL &&
816 : (EQUALN((const char*)pabyHeaderData, "HTTP/1.0 ", 9) ||
817 : EQUALN((const char*)pabyHeaderData, "HTTP/1.1 ", 9)))
818 : {
819 28 : nHTTPCode = atoi((const char*)pabyHeaderData + 9);
820 : if (ENABLE_DEBUG)
821 : CPLDebug("VSICURL", "HTTP code = %d", nHTTPCode);
822 :
823 : /* If moved permanently/temporarily, go on */
824 28 : if( !(nHTTPCode == 301 || nHTTPCode == 302) )
825 : {
826 28 : poFS->AcquireMutex();
827 28 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
828 28 : cachedFileProp->eExists = eExists = (nHTTPCode == 200) ? EXIST_YES : EXIST_NO;
829 28 : poFS->ReleaseMutex();
830 : }
831 : }
832 :
833 393 : if ( !(nHTTPCode == 301 || nHTTPCode == 302) && !bHastComputedFileSize)
834 : {
835 : /* Caution: when gzip compression is enabled, the content-length is the compressed */
836 : /* size, which we are not interested in, so we must not take it into account. */
837 :
838 231 : const char* pszContentLength = strstr((const char*)pabyHeaderData, "Content-Length: ");
839 231 : const char* pszEndOfLine = pszContentLength ? strchr(pszContentLength, '\n') : NULL;
840 231 : if( bCanTrustCandidateFileSize && pszEndOfLine != NULL )
841 : {
842 32 : const char* pszVal = pszContentLength + strlen("Content-Length: ");
843 32 : bHasCandidateFileSize = TRUE;
844 32 : nCandidateFileSize = CPLScanUIntBig(pszVal, pszEndOfLine - pszVal);
845 32 : if (ENABLE_DEBUG)
846 : CPLDebug("VSICURL", "Has found candidate file size = " CPL_FRMT_GUIB, nCandidateFileSize);
847 : }
848 :
849 231 : const char* pszContentEncoding = strstr((const char*)pabyHeaderData, "Content-Encoding: ");
850 231 : pszEndOfLine = pszContentEncoding ? strchr(pszContentEncoding, '\n') : NULL;
851 231 : if( bHasCandidateFileSize && pszEndOfLine != NULL )
852 : {
853 66 : const char* pszVal = pszContentEncoding + strlen("Content-Encoding: ");
854 66 : if( strncmp(pszVal, "gzip", 4) == 0 )
855 : {
856 : if (ENABLE_DEBUG)
857 : CPLDebug("VSICURL", "GZip compression enabled --> cannot trust candidate file size");
858 66 : bCanTrustCandidateFileSize = FALSE;
859 : }
860 : }
861 : }
862 :
863 393 : ReleaseMutex();
864 : }
865 :
866 393 : return nmemb;
867 : }
868 :
869 : /************************************************************************/
870 : /* VSICurlStreamingHandleReceivedBytesHeader() */
871 : /************************************************************************/
872 :
873 393 : static int VSICurlStreamingHandleReceivedBytesHeader(void *buffer, size_t count, size_t nmemb, void *req)
874 : {
875 393 : return ((VSICurlStreamingHandle*)req)->ReceivedBytesHeader((GByte*)buffer, count, nmemb);
876 : }
877 : /************************************************************************/
878 : /* DownloadInThread() */
879 : /************************************************************************/
880 :
881 48 : void VSICurlStreamingHandle::DownloadInThread()
882 : {
883 48 : VSICurlSetOptions(hCurlHandle, pszURL);
884 :
885 : static int bHasCheckVersion = FALSE;
886 : static int bSupportGZip = FALSE;
887 48 : if (!bHasCheckVersion)
888 : {
889 3 : bSupportGZip = strstr(curl_version(), "zlib/") != NULL;
890 3 : bHasCheckVersion = TRUE;
891 : }
892 48 : if (bSupportGZip && CSLTestBoolean(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
893 : {
894 48 : curl_easy_setopt(hCurlHandle, CURLOPT_ENCODING, "gzip");
895 : }
896 :
897 48 : if (pabyHeaderData == NULL)
898 47 : pabyHeaderData = (GByte*) CPLMalloc(HEADER_SIZE + 1);
899 48 : nHeaderSize = 0;
900 48 : nBodySize = 0;
901 48 : nHTTPCode = 0;
902 :
903 48 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, this);
904 48 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, VSICurlStreamingHandleReceivedBytesHeader);
905 :
906 48 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, this);
907 48 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlStreamingHandleReceivedBytes);
908 :
909 : char szCurlErrBuf[CURL_ERROR_SIZE+1];
910 48 : szCurlErrBuf[0] = '\0';
911 48 : curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
912 :
913 48 : CURLcode eRet = curl_easy_perform(hCurlHandle);
914 :
915 48 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, NULL);
916 48 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, NULL);
917 48 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, NULL);
918 48 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, NULL);
919 :
920 48 : AcquireMutex();
921 48 : if (!bAskDownloadEnd && eRet == 0 && !bHastComputedFileSize)
922 : {
923 23 : poFS->AcquireMutex();
924 23 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
925 23 : cachedFileProp->fileSize = fileSize = nBodySize;
926 23 : cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
927 : if (ENABLE_DEBUG)
928 : CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
929 23 : poFS->ReleaseMutex();
930 : }
931 :
932 48 : bDownloadInProgress = FALSE;
933 48 : bDownloadStopped = TRUE;
934 :
935 : /* Signal to the consumer that the download has ended */
936 48 : CPLCondSignal(hCondProducer);
937 48 : ReleaseMutex();
938 48 : }
939 :
940 48 : static void VSICurlDownloadInThread(void* pArg)
941 : {
942 48 : ((VSICurlStreamingHandle*)pArg)->DownloadInThread();
943 48 : }
944 :
945 : /************************************************************************/
946 : /* StartDownload() */
947 : /************************************************************************/
948 :
949 64 : void VSICurlStreamingHandle::StartDownload()
950 : {
951 64 : if (bDownloadInProgress || bDownloadStopped)
952 16 : return;
953 :
954 : //if (ENABLE_DEBUG)
955 48 : CPLDebug("VSICURL", "Start download for %s", pszURL);
956 :
957 48 : if (hCurlHandle == NULL)
958 48 : hCurlHandle = curl_easy_init();
959 48 : oRingBuffer.Reset();
960 48 : bDownloadInProgress = TRUE;
961 48 : nRingBufferFileOffset = 0;
962 48 : hThread = CPLCreateJoinableThread(VSICurlDownloadInThread, this);
963 : }
964 :
965 : /************************************************************************/
966 : /* StopDownload() */
967 : /************************************************************************/
968 :
969 49 : void VSICurlStreamingHandle::StopDownload()
970 : {
971 49 : if (hThread)
972 : {
973 : //if (ENABLE_DEBUG)
974 48 : CPLDebug("VSICURL", "Stop download for %s", pszURL);
975 :
976 48 : AcquireMutex();
977 : /* Signal to the producer that we ask for download interruption */
978 48 : bAskDownloadEnd = TRUE;
979 48 : CPLCondSignal(hCondConsumer);
980 :
981 : /* Wait for the producer to have finished */
982 101 : while(bDownloadInProgress)
983 5 : CPLCondWait(hCondProducer, hRingBufferMutex);
984 :
985 48 : bAskDownloadEnd = FALSE;
986 :
987 48 : ReleaseMutex();
988 :
989 48 : CPLJoinThread(hThread);
990 48 : hThread = NULL;
991 :
992 48 : curl_easy_cleanup(hCurlHandle);
993 48 : hCurlHandle = NULL;
994 : }
995 :
996 49 : oRingBuffer.Reset();
997 49 : bDownloadStopped = FALSE;
998 49 : }
999 :
1000 : /************************************************************************/
1001 : /* PutRingBufferInCache() */
1002 : /************************************************************************/
1003 :
1004 139 : void VSICurlStreamingHandle::PutRingBufferInCache()
1005 : {
1006 139 : if (nRingBufferFileOffset >= BKGND_BUFFER_SIZE)
1007 0 : return;
1008 :
1009 139 : AcquireMutex();
1010 :
1011 : /* Cache any remaining bytes available in the ring buffer */
1012 139 : size_t nBufSize = oRingBuffer.GetSize();
1013 139 : if ( nBufSize > 0 )
1014 : {
1015 54 : if (nRingBufferFileOffset + nBufSize > BKGND_BUFFER_SIZE)
1016 0 : nBufSize = (size_t) (BKGND_BUFFER_SIZE - nRingBufferFileOffset);
1017 54 : GByte* pabyTmp = (GByte*) CPLMalloc(nBufSize);
1018 54 : oRingBuffer.Read(pabyTmp, nBufSize);
1019 :
1020 : /* Signal to the producer that we have ingested some bytes */
1021 54 : CPLCondSignal(hCondConsumer);
1022 :
1023 54 : AddRegion(nRingBufferFileOffset, nBufSize, pabyTmp);
1024 54 : nRingBufferFileOffset += nBufSize;
1025 54 : CPLFree(pabyTmp);
1026 : }
1027 :
1028 139 : ReleaseMutex();
1029 : }
1030 :
1031 : /************************************************************************/
1032 : /* Read() */
1033 : /************************************************************************/
1034 :
1035 189 : size_t VSICurlStreamingHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
1036 : {
1037 189 : GByte* pabyBuffer = (GByte*)pBuffer;
1038 189 : size_t nBufferRequestSize = nSize * nMemb;
1039 189 : if (nBufferRequestSize == 0)
1040 0 : return 0;
1041 189 : size_t nRemaining = nBufferRequestSize;
1042 :
1043 189 : AcquireMutex();
1044 189 : int bHastComputedFileSizeLocal = bHastComputedFileSize;
1045 189 : vsi_l_offset fileSizeLocal = fileSize;
1046 189 : ReleaseMutex();
1047 :
1048 189 : if (bHastComputedFileSizeLocal && curOffset >= fileSizeLocal)
1049 : {
1050 1 : CPLDebug("VSICURL", "Read attempt beyond end of file");
1051 1 : bEOF = TRUE;
1052 : }
1053 189 : if (bEOF)
1054 1 : return 0;
1055 :
1056 188 : if (curOffset < nRingBufferFileOffset)
1057 139 : PutRingBufferInCache();
1058 :
1059 : if (ENABLE_DEBUG)
1060 : CPLDebug("VSICURL", "Read [" CPL_FRMT_GUIB ", " CPL_FRMT_GUIB "[ in %s",
1061 : curOffset, curOffset + nBufferRequestSize, pszURL);
1062 :
1063 : #ifdef notdef
1064 : if( pCachedData != NULL && nCachedSize >= 1024 &&
1065 : nRecomputedChecksumOfFirst1024Bytes == 0 )
1066 : {
1067 : for(size_t i = 0; i < 1024 / sizeof(int); i ++)
1068 : {
1069 : int nVal;
1070 : memcpy(&nVal, pCachedData + i * sizeof(int), sizeof(int));
1071 : nRecomputedChecksumOfFirst1024Bytes += nVal;
1072 : }
1073 :
1074 : if( bHastComputedFileSizeLocal )
1075 : {
1076 : poFS->AcquireMutex();
1077 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
1078 : if( cachedFileProp->nChecksumOfFirst1024Bytes == 0 )
1079 : {
1080 : cachedFileProp->nChecksumOfFirst1024Bytes = nRecomputedChecksumOfFirst1024Bytes;
1081 : }
1082 : else if( nRecomputedChecksumOfFirst1024Bytes != cachedFileProp->nChecksumOfFirst1024Bytes )
1083 : {
1084 : CPLDebug("VSICURL", "Invalidating previously cached file size. First bytes of file have changed!");
1085 : AcquireMutex();
1086 : bHastComputedFileSize = FALSE;
1087 : cachedFileProp->bHastComputedFileSize = FALSE;
1088 : cachedFileProp->nChecksumOfFirst1024Bytes = 0;
1089 : ReleaseMutex();
1090 : }
1091 : poFS->ReleaseMutex();
1092 : }
1093 : }
1094 : #endif
1095 :
1096 : /* Can we use the cache ? */
1097 188 : if( pCachedData != NULL && curOffset < nCachedSize )
1098 : {
1099 138 : size_t nSz = MIN(nRemaining, (size_t)(nCachedSize - curOffset));
1100 : if (ENABLE_DEBUG)
1101 : CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
1102 : (int)curOffset, (int)(curOffset + nSz), pszURL);
1103 138 : memcpy(pabyBuffer, pCachedData + curOffset, nSz);
1104 138 : pabyBuffer += nSz;
1105 138 : curOffset += nSz;
1106 138 : nRemaining -= nSz;
1107 : }
1108 :
1109 : /* Is the request partially covered by the cache and going beyond file size ? */
1110 188 : if ( pCachedData != NULL && bHastComputedFileSizeLocal &&
1111 : curOffset <= nCachedSize &&
1112 : curOffset + nRemaining > fileSizeLocal &&
1113 : fileSize == nCachedSize )
1114 : {
1115 103 : size_t nSz = (size_t) (nCachedSize - curOffset);
1116 : if (ENABLE_DEBUG && nSz != 0)
1117 : CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
1118 : (int)curOffset, (int)(curOffset + nSz), pszURL);
1119 103 : memcpy(pabyBuffer, pCachedData + curOffset, nSz);
1120 103 : pabyBuffer += nSz;
1121 103 : curOffset += nSz;
1122 103 : nRemaining -= nSz;
1123 103 : bEOF = TRUE;
1124 : }
1125 :
1126 : /* Has a Seek() being done since the last Read() ? */
1127 188 : if (!bEOF && nRemaining > 0 && curOffset != nRingBufferFileOffset)
1128 : {
1129 : /* Backward seek : we need to restart the download from the start */
1130 3 : if (curOffset < nRingBufferFileOffset)
1131 1 : StopDownload();
1132 :
1133 3 : StartDownload();
1134 :
1135 : #define SKIP_BUFFER_SIZE 32768
1136 3 : GByte* pabyTmp = (GByte*)CPLMalloc(SKIP_BUFFER_SIZE);
1137 :
1138 3 : CPLAssert(curOffset >= nRingBufferFileOffset);
1139 3 : vsi_l_offset nBytesToSkip = curOffset - nRingBufferFileOffset;
1140 1278 : while(nBytesToSkip > 0)
1141 : {
1142 1272 : vsi_l_offset nBytesToRead = nBytesToSkip;
1143 :
1144 1272 : AcquireMutex();
1145 1272 : if (nBytesToRead > oRingBuffer.GetSize())
1146 1269 : nBytesToRead = oRingBuffer.GetSize();
1147 1272 : if (nBytesToRead > SKIP_BUFFER_SIZE)
1148 0 : nBytesToRead = SKIP_BUFFER_SIZE;
1149 1272 : oRingBuffer.Read(pabyTmp, (size_t)nBytesToRead);
1150 :
1151 : /* Signal to the producer that we have ingested some bytes */
1152 1272 : CPLCondSignal(hCondConsumer);
1153 1272 : ReleaseMutex();
1154 :
1155 1272 : if (nBytesToRead)
1156 637 : AddRegion(nRingBufferFileOffset, (size_t)nBytesToRead, pabyTmp);
1157 :
1158 1272 : nBytesToSkip -= nBytesToRead;
1159 1272 : nRingBufferFileOffset += nBytesToRead;
1160 :
1161 1272 : if (nBytesToRead == 0 && nBytesToSkip != 0)
1162 : {
1163 : if (ENABLE_DEBUG)
1164 : CPLDebug("VSICURL", "Waiting for writer to produce some bytes...");
1165 :
1166 635 : AcquireMutex();
1167 1905 : while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
1168 635 : CPLCondWait(hCondProducer, hRingBufferMutex);
1169 635 : int bBufferEmpty = (oRingBuffer.GetSize() == 0);
1170 635 : ReleaseMutex();
1171 :
1172 635 : if (bBufferEmpty && !bDownloadInProgress)
1173 0 : break;
1174 : }
1175 : }
1176 :
1177 3 : CPLFree(pabyTmp);
1178 :
1179 3 : if (nBytesToSkip != 0)
1180 : {
1181 0 : bEOF = TRUE;
1182 0 : return 0;
1183 : }
1184 : }
1185 :
1186 188 : if (!bEOF && nRemaining > 0)
1187 : {
1188 61 : StartDownload();
1189 61 : CPLAssert(curOffset == nRingBufferFileOffset);
1190 : }
1191 :
1192 : /* Fill the destination buffer from the ring buffer */
1193 667 : while(!bEOF && nRemaining > 0)
1194 : {
1195 296 : AcquireMutex();
1196 296 : size_t nToRead = oRingBuffer.GetSize();
1197 296 : if (nToRead > nRemaining)
1198 56 : nToRead = nRemaining;
1199 296 : oRingBuffer.Read(pabyBuffer, nToRead);
1200 :
1201 : /* Signal to the producer that we have ingested some bytes */
1202 296 : CPLCondSignal(hCondConsumer);
1203 296 : ReleaseMutex();
1204 :
1205 296 : if (nToRead)
1206 147 : AddRegion(curOffset, nToRead, pabyBuffer);
1207 :
1208 296 : nRemaining -= nToRead;
1209 296 : pabyBuffer += nToRead;
1210 296 : curOffset += nToRead;
1211 296 : nRingBufferFileOffset += nToRead;
1212 :
1213 296 : if (nToRead == 0 && nRemaining != 0)
1214 : {
1215 : if (ENABLE_DEBUG)
1216 : CPLDebug("VSICURL", "Waiting for writer to produce some bytes...");
1217 :
1218 149 : AcquireMutex();
1219 443 : while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
1220 145 : CPLCondWait(hCondProducer, hRingBufferMutex);
1221 149 : int bBufferEmpty = (oRingBuffer.GetSize() == 0);
1222 149 : ReleaseMutex();
1223 :
1224 149 : if (bBufferEmpty && !bDownloadInProgress)
1225 5 : break;
1226 : }
1227 : }
1228 :
1229 : if (ENABLE_DEBUG)
1230 : CPLDebug("VSICURL", "Read(%d) = %d",
1231 : (int)nBufferRequestSize, (int)(nBufferRequestSize - nRemaining));
1232 188 : size_t nRet = (nBufferRequestSize - nRemaining) / nSize;
1233 188 : if (nRet < nMemb)
1234 108 : bEOF = TRUE;
1235 :
1236 188 : return nRet;
1237 : }
1238 :
1239 : /************************************************************************/
1240 : /* AddRegion() */
1241 : /************************************************************************/
1242 :
1243 838 : void VSICurlStreamingHandle::AddRegion( vsi_l_offset nFileOffsetStart,
1244 : size_t nSize,
1245 : GByte *pData )
1246 : {
1247 838 : if (nFileOffsetStart >= BKGND_BUFFER_SIZE)
1248 1 : return;
1249 :
1250 837 : if (pCachedData == NULL)
1251 47 : pCachedData = (GByte*) CPLMalloc(BKGND_BUFFER_SIZE);
1252 :
1253 837 : if (nFileOffsetStart <= nCachedSize &&
1254 : nFileOffsetStart + nSize > nCachedSize)
1255 : {
1256 836 : size_t nSz = MIN(nSize, (size_t) (BKGND_BUFFER_SIZE - nFileOffsetStart));
1257 : if (ENABLE_DEBUG)
1258 : CPLDebug("VSICURL", "Writing [%d, %d[ in cache for %s",
1259 : (int)nFileOffsetStart, (int)(nFileOffsetStart + nSz), pszURL);
1260 836 : memcpy(pCachedData + nFileOffsetStart, pData, nSz);
1261 836 : nCachedSize = (size_t) (nFileOffsetStart + nSz);
1262 : }
1263 : }
1264 : /************************************************************************/
1265 : /* Write() */
1266 : /************************************************************************/
1267 :
1268 0 : size_t VSICurlStreamingHandle::Write( const void *pBuffer, size_t nSize, size_t nMemb )
1269 : {
1270 0 : return 0;
1271 : }
1272 :
1273 : /************************************************************************/
1274 : /* Eof() */
1275 : /************************************************************************/
1276 :
1277 :
1278 162 : int VSICurlStreamingHandle::Eof()
1279 : {
1280 162 : return bEOF;
1281 : }
1282 :
1283 : /************************************************************************/
1284 : /* Flush() */
1285 : /************************************************************************/
1286 :
1287 0 : int VSICurlStreamingHandle::Flush()
1288 : {
1289 0 : return 0;
1290 : }
1291 :
1292 : /************************************************************************/
1293 : /* Close() */
1294 : /************************************************************************/
1295 :
1296 46 : int VSICurlStreamingHandle::Close()
1297 : {
1298 46 : return 0;
1299 : }
1300 :
1301 :
1302 : /************************************************************************/
1303 : /* VSICurlStreamingFSHandler() */
1304 : /************************************************************************/
1305 :
1306 757 : VSICurlStreamingFSHandler::VSICurlStreamingFSHandler()
1307 : {
1308 757 : hMutex = CPLCreateMutex();
1309 757 : CPLReleaseMutex(hMutex);
1310 757 : }
1311 :
1312 : /************************************************************************/
1313 : /* ~VSICurlStreamingFSHandler() */
1314 : /************************************************************************/
1315 :
1316 704 : VSICurlStreamingFSHandler::~VSICurlStreamingFSHandler()
1317 : {
1318 704 : std::map<CPLString, CachedFileProp*>::const_iterator iterCacheFileSize;
1319 :
1320 732 : for( iterCacheFileSize = cacheFileSize.begin();
1321 : iterCacheFileSize != cacheFileSize.end();
1322 : iterCacheFileSize++ )
1323 : {
1324 28 : CPLFree(iterCacheFileSize->second);
1325 : }
1326 :
1327 704 : CPLDestroyMutex( hMutex );
1328 704 : hMutex = NULL;
1329 704 : }
1330 :
1331 : /************************************************************************/
1332 : /* AcquireMutex() */
1333 : /************************************************************************/
1334 :
1335 131 : void VSICurlStreamingFSHandler::AcquireMutex()
1336 : {
1337 131 : CPLAcquireMutex(hMutex, 1000.0);
1338 131 : }
1339 :
1340 : /************************************************************************/
1341 : /* ReleaseMutex() */
1342 : /************************************************************************/
1343 :
1344 131 : void VSICurlStreamingFSHandler::ReleaseMutex()
1345 : {
1346 131 : CPLReleaseMutex(hMutex);
1347 131 : }
1348 :
1349 : /************************************************************************/
1350 : /* GetCachedFileProp() */
1351 : /************************************************************************/
1352 :
1353 : /* Should be called under the FS Lock */
1354 :
1355 131 : CachedFileProp* VSICurlStreamingFSHandler::GetCachedFileProp(const char* pszURL)
1356 : {
1357 131 : CachedFileProp* cachedFileProp = cacheFileSize[pszURL];
1358 131 : if (cachedFileProp == NULL)
1359 : {
1360 28 : cachedFileProp = (CachedFileProp*) CPLMalloc(sizeof(CachedFileProp));
1361 28 : cachedFileProp->eExists = EXIST_UNKNOWN;
1362 28 : cachedFileProp->bHastComputedFileSize = FALSE;
1363 28 : cachedFileProp->fileSize = 0;
1364 28 : cachedFileProp->bIsDirectory = FALSE;
1365 : #ifdef notdef
1366 : cachedFileProp->nChecksumOfFirst1024Bytes = 0;
1367 : #endif
1368 28 : cacheFileSize[pszURL] = cachedFileProp;
1369 : }
1370 :
1371 131 : return cachedFileProp;
1372 : }
1373 :
1374 : /************************************************************************/
1375 : /* Open() */
1376 : /************************************************************************/
1377 :
1378 48 : VSIVirtualHandle* VSICurlStreamingFSHandler::Open( const char *pszFilename,
1379 : const char *pszAccess )
1380 : {
1381 48 : if (strchr(pszAccess, 'w') != NULL ||
1382 : strchr(pszAccess, '+') != NULL)
1383 : {
1384 : CPLError(CE_Failure, CPLE_AppDefined,
1385 0 : "Only read-only mode is supported for /vsicurl_streaming");
1386 0 : return NULL;
1387 : }
1388 :
1389 : VSICurlStreamingHandle* poHandle = new VSICurlStreamingHandle(
1390 48 : this, pszFilename + strlen("/vsicurl_streaming/"));
1391 : /* If we didn't get a filelist, check that the file really exists */
1392 48 : if (!poHandle->Exists())
1393 : {
1394 2 : delete poHandle;
1395 2 : poHandle = NULL;
1396 : }
1397 :
1398 48 : if( CSLTestBoolean( CPLGetConfigOption( "VSI_CACHE", "FALSE" ) ) )
1399 0 : return VSICreateCachedFile( poHandle );
1400 : else
1401 48 : return poHandle;
1402 : }
1403 :
1404 : /************************************************************************/
1405 : /* Stat() */
1406 : /************************************************************************/
1407 :
1408 0 : int VSICurlStreamingFSHandler::Stat( const char *pszFilename,
1409 : VSIStatBufL *pStatBuf,
1410 : int nFlags )
1411 : {
1412 0 : CPLString osFilename(pszFilename);
1413 :
1414 0 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
1415 :
1416 0 : VSICurlStreamingHandle oHandle( this, osFilename + strlen("/vsicurl_streaming/"));
1417 :
1418 0 : if ( oHandle.IsKnownFileSize() ||
1419 : ((nFlags & VSI_STAT_SIZE_FLAG) && !oHandle.IsDirectory() &&
1420 : CSLTestBoolean(CPLGetConfigOption("CPL_VSIL_CURL_SLOW_GET_SIZE", "YES"))) )
1421 0 : pStatBuf->st_size = oHandle.GetFileSize();
1422 :
1423 0 : int nRet = (oHandle.Exists()) ? 0 : -1;
1424 0 : pStatBuf->st_mode = oHandle.IsDirectory() ? S_IFDIR : S_IFREG;
1425 0 : return nRet;
1426 : }
1427 :
1428 : /************************************************************************/
1429 : /* VSIInstallCurlFileHandler() */
1430 : /************************************************************************/
1431 :
1432 : /**
1433 : * \brief Install /vsicurl_streaming/ HTTP/FTP file system handler (requires libcurl)
1434 : *
1435 : * A special file handler is installed that allows on-the-fly reading of files
1436 : * streamed through HTTP/FTP web protocols (typically dynamically generated files),
1437 : * without downloading the entire file.
1438 : *
1439 : * Although this file handler is able seek to random offsets in the file, this will not
1440 : * be efficient. If you need efficient random access and that the server supports range
1441 : * dowloading, you should use the /vsicurl/ file system handler instead.
1442 : *
1443 : * Recognized filenames are of the form /vsicurl_streaming/http://path/to/remote/ressource or
1444 : * /vsicurl_streaming/ftp://path/to/remote/ressource where path/to/remote/ressource is the
1445 : * URL of a remote ressource.
1446 : *
1447 : * The GDAL_HTTP_PROXY, GDAL_HTTP_PROXYUSERPWD and GDAL_PROXY_AUTH configuration options can be
1448 : * used to define a proxy server. The syntax to use is the one of Curl CURLOPT_PROXY,
1449 : * CURLOPT_PROXYUSERPWD and CURLOPT_PROXYAUTH options.
1450 : *
1451 : * The file can be cached in RAM by setting the configuration option
1452 : * VSI_CACHE to TRUE. The cache size defaults to 25 MB, but can be modified by setting
1453 : * the configuration option VSI_CACHE_SIZE (in bytes).
1454 : *
1455 : * VSIStatL() will return the size in st_size member and file
1456 : * nature- file or directory - in st_mode member (the later only reliable with FTP
1457 : * resources for now).
1458 : *
1459 : * @since GDAL 1.10
1460 : */
1461 757 : void VSIInstallCurlStreamingFileHandler(void)
1462 : {
1463 757 : VSIFileManager::InstallHandler( "/vsicurl_streaming/", new VSICurlStreamingFSHandler );
1464 757 : }
1465 :
1466 : #ifdef AS_PLUGIN
1467 : CPL_C_START
1468 : void CPL_DLL GDALRegisterMe();
1469 : void GDALRegisterMe()
1470 : {
1471 : VSIInstallCurlStreamingFileHandler();
1472 : }
1473 : CPL_C_END
1474 : #endif
1475 :
1476 : #endif /* !defined(HAVE_CURL) || defined(CPL_MULTIPROC_STUB) */
|
