1 : /******************************************************************************
2 : * $Id: cpl_vsil_curl_streaming.cpp 25311 2012-12-15 12:48:14Z rouault $
3 : *
4 : * Project: CPL - Common Portability Library
5 : * Purpose: Implement VSI large file api for HTTP/FTP files in streaming mode
6 : * Author: Even Rouault <even dot rouault at mines dash paris.org>
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2012, Even Rouault <even dot rouault at mines dash paris.org>
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_string.h"
32 : #include "cpl_multiproc.h"
33 : #include "cpl_hash_set.h"
34 : #include "cpl_time.h"
35 :
36 : CPL_CVSID("$Id: cpl_vsil_curl_streaming.cpp 25311 2012-12-15 12:48:14Z rouault $");
37 :
38 : #if !defined(HAVE_CURL) || defined(CPL_MULTIPROC_STUB)
39 :
40 : void VSIInstallCurlStreamingFileHandler(void)
41 : {
42 : /* not supported */
43 : }
44 :
45 : #else
46 :
47 : #include <curl/curl.h>
48 :
49 : #include <map>
50 :
51 : #define ENABLE_DEBUG 0
52 :
53 : #define N_MAX_REGIONS 10
54 :
55 : #define BKGND_BUFFER_SIZE (1024 * 1024)
56 :
57 :
58 : /************************************************************************/
59 : /* RingBuffer */
60 : /************************************************************************/
61 :
62 : class RingBuffer
63 : {
64 : GByte* pabyBuffer;
65 : size_t nCapacity;
66 : size_t nOffset;
67 : size_t nLength;
68 :
69 : public:
70 : RingBuffer(size_t nCapacity = BKGND_BUFFER_SIZE);
71 : ~RingBuffer();
72 :
73 4820 : size_t GetCapacity() const { return nCapacity; }
74 26566 : size_t GetSize() const { return nLength; }
75 :
76 : void Reset();
77 : void Write(void* pBuffer, size_t nSize);
78 : void Read(void* pBuffer, size_t nSize);
79 : };
80 :
81 41 : RingBuffer::RingBuffer(size_t nCapacityIn)
82 : {
83 41 : pabyBuffer = (GByte*)CPLMalloc(nCapacityIn);
84 41 : nCapacity = nCapacityIn;
85 41 : nOffset = 0;
86 41 : nLength = 0;
87 41 : }
88 :
89 41 : RingBuffer::~RingBuffer()
90 : {
91 41 : CPLFree(pabyBuffer);
92 41 : }
93 :
94 68 : void RingBuffer::Reset()
95 : {
96 68 : nOffset = 0;
97 68 : nLength = 0;
98 68 : }
99 :
100 4820 : void RingBuffer::Write(void* pBuffer, size_t nSize)
101 : {
102 4820 : CPLAssert(nLength + nSize <= nCapacity);
103 :
104 4820 : size_t nEndOffset = (nOffset + nLength) % nCapacity;
105 4820 : size_t nSz = MIN(nSize, nCapacity - nEndOffset);
106 4820 : memcpy(pabyBuffer + nEndOffset, pBuffer, nSz);
107 4820 : if (nSz < nSize)
108 4 : memcpy(pabyBuffer, (GByte*)pBuffer + nSz, nSize - nSz);
109 :
110 4820 : nLength += nSize;
111 4820 : }
112 :
113 7070 : void RingBuffer::Read(void* pBuffer, size_t nSize)
114 : {
115 7070 : CPLAssert(nSize <= nLength);
116 :
117 7070 : if (pBuffer)
118 : {
119 7070 : size_t nSz = MIN(nSize, nCapacity - nOffset);
120 7070 : memcpy(pBuffer, pabyBuffer + nOffset, nSz);
121 7070 : if (nSz < nSize)
122 2 : memcpy((GByte*)pBuffer + nSz, pabyBuffer, nSize - nSz);
123 : }
124 :
125 7070 : nOffset = (nOffset + nSize) % nCapacity;
126 7070 : nLength -= nSize;
127 7070 : }
128 :
129 : /************************************************************************/
130 :
131 : typedef enum
132 : {
133 : EXIST_UNKNOWN = -1,
134 : EXIST_NO,
135 : EXIST_YES,
136 : } ExistStatus;
137 :
138 : typedef struct
139 : {
140 : ExistStatus eExists;
141 : int bHastComputedFileSize;
142 : vsi_l_offset fileSize;
143 : int bIsDirectory;
144 : } CachedFileProp;
145 :
146 : typedef struct
147 : {
148 : unsigned long pszURLHash;
149 : size_t nSize;
150 : GByte *pData;
151 : } CachedRegion;
152 :
153 : /************************************************************************/
154 : /* VSICurlStreamingFSHandler */
155 : /************************************************************************/
156 :
157 : class VSICurlStreamingFSHandler : public VSIFilesystemHandler
158 : {
159 : void *hMutex;
160 :
161 : CachedRegion **papsRegions;
162 : int nRegions;
163 :
164 : std::map<CPLString, CachedFileProp*> cacheFileSize;
165 :
166 : public:
167 : VSICurlStreamingFSHandler();
168 : ~VSICurlStreamingFSHandler();
169 :
170 : virtual VSIVirtualHandle *Open( const char *pszFilename,
171 : const char *pszAccess);
172 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
173 :
174 : void AcquireMutex();
175 : void ReleaseMutex();
176 :
177 : CachedRegion* GetRegion(const char* pszURL);
178 :
179 : void AddRegion(const char* pszURL,
180 : vsi_l_offset nFileOffsetStart,
181 : size_t nSize,
182 : GByte *pData);
183 :
184 : CachedFileProp* GetCachedFileProp(const char* pszURL);
185 : };
186 :
187 : /************************************************************************/
188 : /* VSICurlStreamingHandle */
189 : /************************************************************************/
190 :
191 : class VSICurlStreamingHandle : public VSIVirtualHandle
192 : {
193 : private:
194 : VSICurlStreamingFSHandler* poFS;
195 :
196 : char* pszURL;
197 :
198 : vsi_l_offset curOffset;
199 : vsi_l_offset fileSize;
200 : int bHastComputedFileSize;
201 : ExistStatus eExists;
202 : int bIsDirectory;
203 :
204 : vsi_l_offset lastDownloadedOffset;
205 : int nBlocksToDownload;
206 : int bEOF;
207 :
208 : CURL* hCurlHandle;
209 :
210 : volatile int bDownloadInProgress;
211 : volatile int bDownloadStopped;
212 : volatile int bAskDownloadEnd;
213 : vsi_l_offset nRingBufferFileOffset;
214 : void *hThread;
215 : void *hRingBufferMutex;
216 : void *hCondProducer;
217 : void *hCondConsumer;
218 : RingBuffer oRingBuffer;
219 : void StartDownload();
220 : void StopDownload();
221 : void PutRingBufferInCache();
222 :
223 : GByte *pabyHeaderData;
224 : size_t nHeaderSize;
225 : vsi_l_offset nBodySize;
226 : int nHTTPCode;
227 :
228 : void AcquireMutex();
229 : void ReleaseMutex();
230 :
231 : public:
232 :
233 : VSICurlStreamingHandle(VSICurlStreamingFSHandler* poFS, const char* pszURL);
234 : ~VSICurlStreamingHandle();
235 :
236 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
237 : virtual vsi_l_offset Tell();
238 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
239 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
240 : virtual int Eof();
241 : virtual int Flush();
242 : virtual int Close();
243 :
244 : void DownloadInThread();
245 : int ReceivedBytes(GByte *buffer, size_t count, size_t nmemb);
246 : int ReceivedBytesHeader(GByte *buffer, size_t count, size_t nmemb);
247 :
248 0 : int IsKnownFileSize() const { return bHastComputedFileSize; }
249 : vsi_l_offset GetFileSize();
250 : int Exists();
251 0 : int IsDirectory() const { return bIsDirectory; }
252 : };
253 :
254 : /************************************************************************/
255 : /* VSICurlStreamingHandle() */
256 : /************************************************************************/
257 :
258 41 : VSICurlStreamingHandle::VSICurlStreamingHandle(VSICurlStreamingFSHandler* poFS, const char* pszURL)
259 : {
260 41 : this->poFS = poFS;
261 41 : this->pszURL = CPLStrdup(pszURL);
262 :
263 41 : curOffset = 0;
264 :
265 41 : poFS->AcquireMutex();
266 41 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
267 41 : eExists = cachedFileProp->eExists;
268 41 : fileSize = cachedFileProp->fileSize;
269 41 : bHastComputedFileSize = cachedFileProp->bHastComputedFileSize;
270 41 : bIsDirectory = cachedFileProp->bIsDirectory;
271 41 : poFS->ReleaseMutex();
272 :
273 41 : lastDownloadedOffset = -1;
274 41 : nBlocksToDownload = 1;
275 41 : bEOF = FALSE;
276 :
277 41 : hCurlHandle = NULL;
278 :
279 41 : hThread = NULL;
280 41 : hRingBufferMutex = CPLCreateMutex();
281 41 : ReleaseMutex();
282 41 : hCondProducer = CPLCreateCond();
283 41 : hCondConsumer = CPLCreateCond();
284 :
285 41 : bDownloadInProgress = FALSE;
286 41 : bDownloadStopped = FALSE;
287 41 : bAskDownloadEnd = FALSE;
288 41 : nRingBufferFileOffset = 0;
289 :
290 41 : pabyHeaderData = NULL;
291 41 : nHeaderSize = 0;
292 41 : nBodySize = 0;
293 41 : nHTTPCode = 0;
294 41 : }
295 :
296 : /************************************************************************/
297 : /* ~VSICurlStreamingHandle() */
298 : /************************************************************************/
299 :
300 41 : VSICurlStreamingHandle::~VSICurlStreamingHandle()
301 : {
302 41 : StopDownload();
303 :
304 41 : CPLFree(pszURL);
305 41 : if (hCurlHandle != NULL)
306 0 : curl_easy_cleanup(hCurlHandle);
307 :
308 41 : CPLFree(pabyHeaderData);
309 :
310 41 : CPLDestroyMutex( hRingBufferMutex );
311 41 : CPLDestroyCond( hCondProducer );
312 41 : CPLDestroyCond( hCondConsumer );
313 41 : }
314 :
315 : /************************************************************************/
316 : /* AcquireMutex() */
317 : /************************************************************************/
318 :
319 15955 : void VSICurlStreamingHandle::AcquireMutex()
320 : {
321 15955 : CPLAcquireMutex(hRingBufferMutex, 1000.0);
322 15955 : }
323 :
324 : /************************************************************************/
325 : /* ReleaseMutex() */
326 : /************************************************************************/
327 :
328 15996 : void VSICurlStreamingHandle::ReleaseMutex()
329 : {
330 15996 : CPLReleaseMutex(hRingBufferMutex);
331 15995 : }
332 :
333 : /************************************************************************/
334 : /* Seek() */
335 : /************************************************************************/
336 :
337 124 : int VSICurlStreamingHandle::Seek( vsi_l_offset nOffset, int nWhence )
338 : {
339 124 : if (nWhence == SEEK_SET)
340 : {
341 123 : curOffset = nOffset;
342 : }
343 1 : else if (nWhence == SEEK_CUR)
344 : {
345 0 : curOffset = curOffset + nOffset;
346 : }
347 : else
348 : {
349 1 : curOffset = GetFileSize() + nOffset;
350 : }
351 124 : bEOF = FALSE;
352 124 : return 0;
353 : }
354 :
355 : /************************************************************************/
356 : /* VSICurlSetOptions() */
357 : /************************************************************************/
358 :
359 26 : static void VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL)
360 : {
361 26 : curl_easy_setopt(hCurlHandle, CURLOPT_URL, pszURL);
362 26 : if (CSLTestBoolean(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")))
363 0 : curl_easy_setopt(hCurlHandle, CURLOPT_VERBOSE, 1);
364 :
365 : /* Set Proxy parameters */
366 26 : const char* pszProxy = CPLGetConfigOption("GDAL_HTTP_PROXY", NULL);
367 26 : if (pszProxy)
368 0 : curl_easy_setopt(hCurlHandle,CURLOPT_PROXY,pszProxy);
369 :
370 26 : const char* pszProxyUserPwd = CPLGetConfigOption("GDAL_HTTP_PROXYUSERPWD", NULL);
371 26 : if (pszProxyUserPwd)
372 0 : curl_easy_setopt(hCurlHandle,CURLOPT_PROXYUSERPWD,pszProxyUserPwd);
373 :
374 : /* Enable following redirections. Requires libcurl 7.10.1 at least */
375 26 : curl_easy_setopt(hCurlHandle, CURLOPT_FOLLOWLOCATION, 1);
376 26 : curl_easy_setopt(hCurlHandle, CURLOPT_MAXREDIRS, 10);
377 :
378 : /* 7.16 */
379 : #if LIBCURL_VERSION_NUM >= 0x071000
380 26 : long option = CURLFTPMETHOD_SINGLECWD;
381 26 : curl_easy_setopt(hCurlHandle, CURLOPT_FTP_FILEMETHOD, option);
382 : #endif
383 :
384 : /* 7.12.3 */
385 : #if LIBCURL_VERSION_NUM > 0x070C03
386 : /* ftp://ftp2.cits.rncan.gc.ca/pub/cantopo/250k_tif/ doesn't like EPSV command */
387 26 : curl_easy_setopt(hCurlHandle, CURLOPT_FTP_USE_EPSV, 0);
388 : #endif
389 :
390 : /* NOSIGNAL should be set to true for timeout to work in multithread
391 : environments on Unix, requires libcurl 7.10 or more recent.
392 : (this force avoiding the use of sgnal handlers) */
393 :
394 : /* 7.10 */
395 : #if LIBCURL_VERSION_NUM >= 0x070A00
396 26 : curl_easy_setopt(hCurlHandle, CURLOPT_NOSIGNAL, 1);
397 : #endif
398 :
399 26 : curl_easy_setopt(hCurlHandle, CURLOPT_NOBODY, 0);
400 26 : curl_easy_setopt(hCurlHandle, CURLOPT_HTTPGET, 1);
401 26 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADER, 0);
402 :
403 : /* 7.16.4 */
404 : #if LIBCURL_VERSION_NUM <= 0x071004
405 : curl_easy_setopt(hCurlHandle, CURLOPT_FTPLISTONLY, 0);
406 : #elif LIBCURL_VERSION_NUM > 0x071004
407 26 : curl_easy_setopt(hCurlHandle, CURLOPT_DIRLISTONLY, 0);
408 : #endif
409 :
410 26 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, NULL);
411 26 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, NULL);
412 26 : }
413 :
414 :
415 : typedef struct
416 : {
417 : char* pBuffer;
418 : size_t nSize;
419 : int bIsHTTP;
420 : int bIsInHeader;
421 : int nHTTPCode;
422 : int bDownloadHeaderOnly;
423 : } WriteFuncStruct;
424 :
425 : /************************************************************************/
426 : /* VSICURLStreamingInitWriteFuncStruct() */
427 : /************************************************************************/
428 :
429 0 : static void VSICURLStreamingInitWriteFuncStruct(WriteFuncStruct *psStruct)
430 : {
431 0 : psStruct->pBuffer = NULL;
432 0 : psStruct->nSize = 0;
433 0 : psStruct->bIsHTTP = FALSE;
434 0 : psStruct->bIsInHeader = TRUE;
435 0 : psStruct->nHTTPCode = 0;
436 0 : psStruct->bDownloadHeaderOnly = FALSE;
437 0 : }
438 :
439 : /************************************************************************/
440 : /* VSICurlStreamingHandleWriteFuncForHeader() */
441 : /************************************************************************/
442 :
443 0 : static int VSICurlStreamingHandleWriteFuncForHeader(void *buffer, size_t count, size_t nmemb, void *req)
444 : {
445 0 : WriteFuncStruct* psStruct = (WriteFuncStruct*) req;
446 0 : size_t nSize = count * nmemb;
447 :
448 : char* pNewBuffer = (char*) VSIRealloc(psStruct->pBuffer,
449 0 : psStruct->nSize + nSize + 1);
450 0 : if (pNewBuffer)
451 : {
452 0 : psStruct->pBuffer = pNewBuffer;
453 0 : memcpy(psStruct->pBuffer + psStruct->nSize, buffer, nSize);
454 0 : psStruct->pBuffer[psStruct->nSize + nSize] = '\0';
455 0 : if (psStruct->bIsHTTP && psStruct->bIsInHeader)
456 : {
457 0 : char* pszLine = psStruct->pBuffer + psStruct->nSize;
458 0 : if (EQUALN(pszLine, "HTTP/1.0 ", 9) ||
459 : EQUALN(pszLine, "HTTP/1.1 ", 9))
460 0 : psStruct->nHTTPCode = atoi(pszLine + 9);
461 :
462 0 : if (pszLine[0] == '\r' || pszLine[0] == '\n')
463 : {
464 0 : if (psStruct->bDownloadHeaderOnly)
465 : {
466 : /* If moved permanently/temporarily, go on. Otherwise stop now*/
467 0 : if (!(psStruct->nHTTPCode == 301 || psStruct->nHTTPCode == 302))
468 0 : return 0;
469 : }
470 : else
471 : {
472 0 : psStruct->bIsInHeader = FALSE;
473 : }
474 : }
475 : }
476 0 : psStruct->nSize += nSize;
477 0 : return nmemb;
478 : }
479 : else
480 : {
481 0 : return 0;
482 : }
483 : }
484 :
485 :
486 : /************************************************************************/
487 : /* GetFileSize() */
488 : /************************************************************************/
489 :
490 1 : vsi_l_offset VSICurlStreamingHandle::GetFileSize()
491 : {
492 : WriteFuncStruct sWriteFuncData;
493 : WriteFuncStruct sWriteFuncHeaderData;
494 :
495 1 : AcquireMutex();
496 1 : if (bHastComputedFileSize)
497 : {
498 1 : vsi_l_offset nRet = fileSize;
499 1 : ReleaseMutex();
500 1 : return nRet;
501 : }
502 0 : ReleaseMutex();
503 :
504 : #if LIBCURL_VERSION_NUM < 0x070B00
505 : /* Curl 7.10.X doesn't manage to unset the CURLOPT_RANGE that would have been */
506 : /* previously set, so we have to reinit the connection handle */
507 : if (hCurlHandle)
508 : {
509 : curl_easy_cleanup(hCurlHandle);
510 : hCurlHandle = curl_easy_init();
511 : }
512 : #endif
513 :
514 0 : CURL* hLocalHandle = curl_easy_init();
515 :
516 0 : VSICurlSetOptions(hLocalHandle, pszURL);
517 :
518 0 : VSICURLStreamingInitWriteFuncStruct(&sWriteFuncHeaderData);
519 :
520 : /* HACK for mbtiles driver: proper fix would be to auto-detect servers that don't accept HEAD */
521 : /* http://a.tiles.mapbox.com/v3/ doesn't accept HEAD, so let's start a GET */
522 : /* and interrupt is as soon as the header is found */
523 0 : if (strstr(pszURL, ".tiles.mapbox.com/") != NULL)
524 : {
525 0 : curl_easy_setopt(hLocalHandle, CURLOPT_HEADERDATA, &sWriteFuncHeaderData);
526 0 : curl_easy_setopt(hLocalHandle, CURLOPT_HEADERFUNCTION, VSICurlStreamingHandleWriteFuncForHeader);
527 :
528 0 : sWriteFuncHeaderData.bIsHTTP = strncmp(pszURL, "http", 4) == 0;
529 0 : sWriteFuncHeaderData.bDownloadHeaderOnly = TRUE;
530 : }
531 : else
532 : {
533 0 : curl_easy_setopt(hLocalHandle, CURLOPT_NOBODY, 1);
534 0 : curl_easy_setopt(hLocalHandle, CURLOPT_HTTPGET, 0);
535 0 : curl_easy_setopt(hLocalHandle, CURLOPT_HEADER, 1);
536 : }
537 :
538 : /* We need that otherwise OSGEO4W's libcurl issue a dummy range request */
539 : /* when doing a HEAD when recycling connections */
540 0 : curl_easy_setopt(hLocalHandle, CURLOPT_RANGE, NULL);
541 :
542 : /* Bug with older curl versions (<=7.16.4) and FTP. See http://curl.haxx.se/mail/lib-2007-08/0312.html */
543 0 : VSICURLStreamingInitWriteFuncStruct(&sWriteFuncData);
544 0 : curl_easy_setopt(hLocalHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
545 0 : curl_easy_setopt(hLocalHandle, CURLOPT_WRITEFUNCTION, VSICurlStreamingHandleWriteFuncForHeader);
546 :
547 : char szCurlErrBuf[CURL_ERROR_SIZE+1];
548 0 : szCurlErrBuf[0] = '\0';
549 0 : curl_easy_setopt(hLocalHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
550 :
551 0 : double dfSize = 0;
552 0 : curl_easy_perform(hLocalHandle);
553 :
554 0 : AcquireMutex();
555 :
556 0 : eExists = EXIST_UNKNOWN;
557 0 : bHastComputedFileSize = TRUE;
558 :
559 0 : if (strncmp(pszURL, "ftp", 3) == 0)
560 : {
561 0 : if (sWriteFuncData.pBuffer != NULL &&
562 : strncmp(sWriteFuncData.pBuffer, "Content-Length: ", strlen( "Content-Length: ")) == 0)
563 : {
564 0 : const char* pszBuffer = sWriteFuncData.pBuffer + strlen("Content-Length: ");
565 0 : eExists = EXIST_YES;
566 0 : fileSize = CPLScanUIntBig(pszBuffer, sWriteFuncData.nSize - strlen("Content-Length: "));
567 0 : if (ENABLE_DEBUG)
568 : CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB,
569 : pszURL, fileSize);
570 : }
571 : }
572 :
573 0 : if (eExists != EXIST_YES)
574 : {
575 0 : CURLcode code = curl_easy_getinfo(hLocalHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dfSize );
576 0 : if (code == 0)
577 : {
578 0 : eExists = EXIST_YES;
579 0 : if (dfSize < 0)
580 0 : fileSize = 0;
581 : else
582 0 : fileSize = (GUIntBig)dfSize;
583 : }
584 : else
585 : {
586 0 : eExists = EXIST_NO;
587 0 : fileSize = 0;
588 : CPLError(CE_Failure, CPLE_AppDefined,
589 0 : "VSICurlStreamingHandle::GetFileSize failed");
590 : }
591 :
592 0 : long response_code = 0;
593 0 : curl_easy_getinfo(hLocalHandle, CURLINFO_HTTP_CODE, &response_code);
594 0 : if (response_code != 200)
595 : {
596 0 : eExists = EXIST_NO;
597 0 : fileSize = 0;
598 : }
599 :
600 : /* Try to guess if this is a directory. Generally if this is a directory, */
601 : /* curl will retry with an URL with slash added */
602 0 : char *pszEffectiveURL = NULL;
603 0 : curl_easy_getinfo(hLocalHandle, CURLINFO_EFFECTIVE_URL, &pszEffectiveURL);
604 0 : if (pszEffectiveURL != NULL &&
605 : strncmp(pszURL, pszEffectiveURL, strlen(pszURL)) == 0 &&
606 0 : pszEffectiveURL[strlen(pszURL)] == '/')
607 : {
608 0 : eExists = EXIST_YES;
609 0 : fileSize = 0;
610 0 : bIsDirectory = TRUE;
611 : }
612 :
613 0 : if (ENABLE_DEBUG)
614 : CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d",
615 : pszURL, fileSize, (int)response_code);
616 : }
617 :
618 0 : CPLFree(sWriteFuncData.pBuffer);
619 0 : CPLFree(sWriteFuncHeaderData.pBuffer);
620 :
621 0 : poFS->AcquireMutex();
622 0 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
623 0 : cachedFileProp->bHastComputedFileSize = TRUE;
624 0 : cachedFileProp->fileSize = fileSize;
625 0 : cachedFileProp->eExists = eExists;
626 0 : cachedFileProp->bIsDirectory = bIsDirectory;
627 0 : poFS->ReleaseMutex();
628 :
629 0 : vsi_l_offset nRet = fileSize;
630 0 : ReleaseMutex();
631 :
632 0 : if (hCurlHandle == NULL)
633 0 : hCurlHandle = hLocalHandle;
634 : else
635 0 : curl_easy_cleanup(hLocalHandle);
636 :
637 0 : return nRet;
638 : }
639 :
640 : /************************************************************************/
641 : /* Exists() */
642 : /************************************************************************/
643 :
644 41 : int VSICurlStreamingHandle::Exists()
645 : {
646 41 : if (eExists == EXIST_UNKNOWN)
647 : {
648 : /* Consider that only the files whose extension ends up with one that is */
649 : /* listed in CPL_VSIL_CURL_ALLOWED_EXTENSIONS exist on the server */
650 : /* This can speeds up dramatically open experience, in case the server */
651 : /* cannot return a file list */
652 : /* For example : */
653 : /* gdalinfo --config CPL_VSIL_CURL_ALLOWED_EXTENSIONS ".tif" /vsicurl_streaming/http://igskmncngs506.cr.usgs.gov/gmted/Global_tiles_GMTED/075darcsec/bln/W030/30N030W_20101117_gmted_bln075.tif */
654 : const char* pszAllowedExtensions =
655 24 : CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", NULL);
656 24 : if (pszAllowedExtensions)
657 : {
658 0 : char** papszExtensions = CSLTokenizeString2( pszAllowedExtensions, ", ", 0 );
659 0 : int nURLLen = strlen(pszURL);
660 0 : int bFound = FALSE;
661 0 : for(int i=0;papszExtensions[i] != NULL;i++)
662 : {
663 0 : int nExtensionLen = strlen(papszExtensions[i]);
664 0 : if (nURLLen > nExtensionLen &&
665 0 : EQUAL(pszURL + nURLLen - nExtensionLen, papszExtensions[i]))
666 : {
667 0 : bFound = TRUE;
668 0 : break;
669 : }
670 : }
671 :
672 0 : if (!bFound)
673 : {
674 0 : eExists = EXIST_NO;
675 0 : fileSize = 0;
676 :
677 0 : poFS->AcquireMutex();
678 0 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
679 0 : cachedFileProp->bHastComputedFileSize = TRUE;
680 0 : cachedFileProp->fileSize = fileSize;
681 0 : cachedFileProp->eExists = eExists;
682 0 : poFS->ReleaseMutex();
683 :
684 0 : CSLDestroy(papszExtensions);
685 :
686 0 : return 0;
687 : }
688 :
689 0 : CSLDestroy(papszExtensions);
690 : }
691 :
692 : char chFirstByte;
693 24 : int bExists = (Read(&chFirstByte, 1, 1) == 1);
694 :
695 24 : AcquireMutex();
696 24 : poFS->AcquireMutex();
697 24 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
698 24 : cachedFileProp->eExists = eExists = bExists ? EXIST_YES : EXIST_NO;
699 24 : poFS->ReleaseMutex();
700 24 : ReleaseMutex();
701 :
702 24 : Seek(0, SEEK_SET);
703 : }
704 :
705 41 : return eExists == EXIST_YES;
706 : }
707 :
708 : /************************************************************************/
709 : /* Tell() */
710 : /************************************************************************/
711 :
712 8 : vsi_l_offset VSICurlStreamingHandle::Tell()
713 : {
714 8 : return curOffset;
715 : }
716 :
717 : /************************************************************************/
718 : /* ReceivedBytes() */
719 : /************************************************************************/
720 :
721 4820 : int VSICurlStreamingHandle::ReceivedBytes(GByte *buffer, size_t count, size_t nmemb)
722 : {
723 4820 : size_t nSize = count * nmemb;
724 4820 : nBodySize += nSize;
725 :
726 : if (ENABLE_DEBUG)
727 : CPLDebug("VSICURL", "Receiving %d bytes...", (int)nSize);
728 :
729 4820 : AcquireMutex();
730 4820 : if (eExists == EXIST_UNKNOWN)
731 : {
732 0 : poFS->AcquireMutex();
733 0 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
734 0 : cachedFileProp->eExists = eExists = EXIST_YES;
735 0 : poFS->ReleaseMutex();
736 : }
737 4820 : else if (eExists == EXIST_NO)
738 : {
739 0 : ReleaseMutex();
740 0 : return 0;
741 : }
742 :
743 0 : while(TRUE)
744 : {
745 4820 : size_t nFree = oRingBuffer.GetCapacity() - oRingBuffer.GetSize();
746 4820 : if (nSize <= nFree)
747 : {
748 4820 : oRingBuffer.Write(buffer, nSize);
749 :
750 : /* Signal to the consumer that we have added bytes to the buffer */
751 4820 : CPLCondSignal(hCondProducer);
752 :
753 4820 : if (bAskDownloadEnd)
754 : {
755 : if (ENABLE_DEBUG)
756 : CPLDebug("VSICURL", "Download interruption asked");
757 :
758 3 : ReleaseMutex();
759 3 : return 0;
760 : }
761 : break;
762 : }
763 : else
764 : {
765 0 : oRingBuffer.Write(buffer, nFree);
766 0 : buffer += nFree;
767 0 : nSize -= nFree;
768 :
769 : /* Signal to the consumer that we have added bytes to the buffer */
770 0 : CPLCondSignal(hCondProducer);
771 :
772 : if (ENABLE_DEBUG)
773 : CPLDebug("VSICURL", "Waiting for reader to consume some bytes...");
774 :
775 0 : while(oRingBuffer.GetSize() == oRingBuffer.GetCapacity() && !bAskDownloadEnd)
776 : {
777 0 : CPLCondWait(hCondConsumer, hRingBufferMutex);
778 : }
779 :
780 0 : if (bAskDownloadEnd)
781 : {
782 : if (ENABLE_DEBUG)
783 : CPLDebug("VSICURL", "Download interruption asked");
784 :
785 0 : ReleaseMutex();
786 0 : return 0;
787 : }
788 : }
789 : }
790 :
791 4817 : ReleaseMutex();
792 :
793 4817 : return nmemb;
794 : }
795 :
796 : /************************************************************************/
797 : /* VSICurlStreamingHandleReceivedBytes() */
798 : /************************************************************************/
799 :
800 4820 : static int VSICurlStreamingHandleReceivedBytes(void *buffer, size_t count, size_t nmemb, void *req)
801 : {
802 4820 : return ((VSICurlStreamingHandle*)req)->ReceivedBytes((GByte*)buffer, count, nmemb);
803 : }
804 :
805 :
806 : /************************************************************************/
807 : /* VSICurlStreamingHandleReceivedBytesHeader() */
808 : /************************************************************************/
809 :
810 : #define HEADER_SIZE 32768
811 :
812 192 : int VSICurlStreamingHandle::ReceivedBytesHeader(GByte *buffer, size_t count, size_t nmemb)
813 : {
814 192 : size_t nSize = count * nmemb;
815 : if (ENABLE_DEBUG)
816 : CPLDebug("VSICURL", "Receiving %d bytes for header...", (int)nSize);
817 :
818 : /* Reset buffer if we have followed link after a redirect */
819 192 : if (nSize >=9 && (nHTTPCode == 301 || nHTTPCode == 302) &&
820 : (EQUALN((const char*)buffer, "HTTP/1.0 ", 9) ||
821 : EQUALN((const char*)buffer, "HTTP/1.1 ", 9)))
822 : {
823 0 : nHeaderSize = 0;
824 0 : nHTTPCode = 0;
825 : }
826 :
827 192 : if (nHeaderSize < HEADER_SIZE)
828 : {
829 192 : size_t nSz = MIN(nSize, HEADER_SIZE - nHeaderSize);
830 192 : memcpy(pabyHeaderData + nHeaderSize, buffer, nSz);
831 192 : pabyHeaderData[nHeaderSize + nSz] = '\0';
832 192 : nHeaderSize += nSz;
833 :
834 : //CPLDebug("VSICURL", "Header : %s", pabyHeaderData);
835 :
836 192 : AcquireMutex();
837 :
838 192 : if (eExists == EXIST_UNKNOWN && nHTTPCode == 0 &&
839 : strchr((const char*)pabyHeaderData, '\n') != NULL &&
840 : (EQUALN((const char*)pabyHeaderData, "HTTP/1.0 ", 9) ||
841 : EQUALN((const char*)pabyHeaderData, "HTTP/1.1 ", 9)))
842 : {
843 24 : nHTTPCode = atoi((const char*)pabyHeaderData + 9);
844 : if (ENABLE_DEBUG)
845 : CPLDebug("VSICURL", "HTTP code = %d", nHTTPCode);
846 :
847 : /* If moved permanently/temporarily, go on */
848 24 : if( !(nHTTPCode == 301 || nHTTPCode == 302) )
849 : {
850 24 : poFS->AcquireMutex();
851 24 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
852 24 : cachedFileProp->eExists = eExists = (nHTTPCode == 200) ? EXIST_YES : EXIST_NO;
853 24 : poFS->ReleaseMutex();
854 : }
855 : }
856 :
857 192 : if ( !(nHTTPCode == 301 || nHTTPCode == 302) && !bHastComputedFileSize)
858 : {
859 144 : const char* pszContentLength = strstr((const char*)pabyHeaderData, "Content-Length: ");
860 144 : const char* pszEndOfLine = pszContentLength ? strchr(pszContentLength, '\n') : NULL;
861 144 : if (pszEndOfLine != NULL)
862 : {
863 16 : poFS->AcquireMutex();
864 16 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
865 : cachedFileProp->fileSize = fileSize = CPLScanUIntBig(pszContentLength + 16,
866 16 : pszEndOfLine - (pszContentLength + 16));
867 16 : cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
868 : if (ENABLE_DEBUG)
869 : CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
870 16 : poFS->ReleaseMutex();
871 : }
872 : }
873 :
874 192 : ReleaseMutex();
875 : }
876 :
877 192 : return nmemb;
878 : }
879 :
880 : /************************************************************************/
881 : /* VSICurlStreamingHandleReceivedBytesHeader() */
882 : /************************************************************************/
883 :
884 192 : static int VSICurlStreamingHandleReceivedBytesHeader(void *buffer, size_t count, size_t nmemb, void *req)
885 : {
886 192 : return ((VSICurlStreamingHandle*)req)->ReceivedBytesHeader((GByte*)buffer, count, nmemb);
887 : }
888 : /************************************************************************/
889 : /* DownloadInThread() */
890 : /************************************************************************/
891 :
892 26 : void VSICurlStreamingHandle::DownloadInThread()
893 : {
894 26 : VSICurlSetOptions(hCurlHandle, pszURL);
895 :
896 26 : if (pabyHeaderData == NULL)
897 25 : pabyHeaderData = (GByte*) CPLMalloc(HEADER_SIZE + 1);
898 26 : nHeaderSize = 0;
899 26 : nBodySize = 0;
900 26 : nHTTPCode = 0;
901 :
902 26 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, this);
903 26 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, VSICurlStreamingHandleReceivedBytesHeader);
904 :
905 26 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, this);
906 26 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlStreamingHandleReceivedBytes);
907 :
908 : char szCurlErrBuf[CURL_ERROR_SIZE+1];
909 26 : szCurlErrBuf[0] = '\0';
910 26 : curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
911 :
912 26 : CURLcode eRet = curl_easy_perform(hCurlHandle);
913 :
914 26 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, NULL);
915 26 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, NULL);
916 26 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, NULL);
917 26 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, NULL);
918 :
919 26 : AcquireMutex();
920 26 : if (!bAskDownloadEnd && eRet == 0 && !bHastComputedFileSize)
921 : {
922 6 : poFS->AcquireMutex();
923 6 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
924 6 : cachedFileProp->fileSize = fileSize = nBodySize;
925 6 : cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
926 : if (ENABLE_DEBUG)
927 : CPLDebug("VSICURL", "File size = " CPL_FRMT_GUIB, fileSize);
928 6 : poFS->ReleaseMutex();
929 : }
930 :
931 26 : bDownloadInProgress = FALSE;
932 26 : bDownloadStopped = TRUE;
933 :
934 : /* Signal to the consumer that the download has ended */
935 26 : CPLCondSignal(hCondProducer);
936 26 : ReleaseMutex();
937 26 : }
938 :
939 26 : static void VSICurlDownloadInThread(void* pArg)
940 : {
941 26 : ((VSICurlStreamingHandle*)pArg)->DownloadInThread();
942 26 : }
943 :
944 : /************************************************************************/
945 : /* StartDownload() */
946 : /************************************************************************/
947 :
948 86 : void VSICurlStreamingHandle::StartDownload()
949 : {
950 86 : if (bDownloadInProgress || bDownloadStopped)
951 60 : return;
952 :
953 : //if (ENABLE_DEBUG)
954 26 : CPLDebug("VSICURL", "Start download for %s", pszURL);
955 :
956 26 : if (hCurlHandle == NULL)
957 26 : hCurlHandle = curl_easy_init();
958 26 : oRingBuffer.Reset();
959 26 : bDownloadInProgress = TRUE;
960 26 : nRingBufferFileOffset = 0;
961 26 : hThread = CPLCreateJoinableThread(VSICurlDownloadInThread, this);
962 : }
963 :
964 : /************************************************************************/
965 : /* StopDownload() */
966 : /************************************************************************/
967 :
968 42 : void VSICurlStreamingHandle::StopDownload()
969 : {
970 42 : if (hThread)
971 : {
972 : //if (ENABLE_DEBUG)
973 26 : CPLDebug("VSICURL", "Stop download for %s", pszURL);
974 :
975 26 : AcquireMutex();
976 : /* Signal to the producer that we ask for download interruption */
977 26 : bAskDownloadEnd = TRUE;
978 26 : CPLCondSignal(hCondConsumer);
979 :
980 : /* Wait for the producer to have finished */
981 56 : while(bDownloadInProgress)
982 4 : CPLCondWait(hCondProducer, hRingBufferMutex);
983 :
984 26 : bAskDownloadEnd = FALSE;
985 :
986 26 : ReleaseMutex();
987 :
988 26 : CPLJoinThread(hThread);
989 26 : hThread = NULL;
990 :
991 26 : curl_easy_cleanup(hCurlHandle);
992 26 : hCurlHandle = NULL;
993 : }
994 :
995 42 : PutRingBufferInCache();
996 :
997 42 : oRingBuffer.Reset();
998 42 : bDownloadStopped = FALSE;
999 42 : }
1000 :
1001 : /************************************************************************/
1002 : /* PutRingBufferInCache() */
1003 : /************************************************************************/
1004 :
1005 135 : void VSICurlStreamingHandle::PutRingBufferInCache()
1006 : {
1007 135 : if (nRingBufferFileOffset >= BKGND_BUFFER_SIZE)
1008 17 : return;
1009 :
1010 118 : AcquireMutex();
1011 :
1012 : /* Cache any remaining bytes available in the ring buffer */
1013 118 : size_t nBufSize = oRingBuffer.GetSize();
1014 118 : if ( nBufSize > 0 )
1015 : {
1016 42 : if (nRingBufferFileOffset + nBufSize > BKGND_BUFFER_SIZE)
1017 0 : nBufSize = (size_t) (BKGND_BUFFER_SIZE - nRingBufferFileOffset);
1018 42 : GByte* pabyTmp = (GByte*) CPLMalloc(nBufSize);
1019 42 : oRingBuffer.Read(pabyTmp, nBufSize);
1020 :
1021 : /* Signal to the producer that we have ingested some bytes */
1022 42 : CPLCondSignal(hCondConsumer);
1023 :
1024 : poFS->AddRegion(pszURL, nRingBufferFileOffset,
1025 42 : nBufSize, pabyTmp);
1026 42 : nRingBufferFileOffset += nBufSize;
1027 42 : CPLFree(pabyTmp);
1028 : }
1029 :
1030 118 : ReleaseMutex();
1031 : }
1032 :
1033 : /************************************************************************/
1034 : /* Read() */
1035 : /************************************************************************/
1036 :
1037 215 : size_t VSICurlStreamingHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
1038 : {
1039 215 : GByte* pabyBuffer = (GByte*)pBuffer;
1040 215 : size_t nBufferRequestSize = nSize * nMemb;
1041 215 : if (nBufferRequestSize == 0)
1042 0 : return 0;
1043 215 : size_t nRemaining = nBufferRequestSize;
1044 :
1045 215 : AcquireMutex();
1046 215 : int bHastComputedFileSizeLocal = bHastComputedFileSize;
1047 215 : vsi_l_offset fileSizeLocal = fileSize;
1048 215 : ReleaseMutex();
1049 :
1050 215 : if (bHastComputedFileSizeLocal && curOffset >= fileSizeLocal)
1051 1 : bEOF = TRUE;
1052 215 : if (bEOF)
1053 1 : return 0;
1054 :
1055 214 : if (curOffset < nRingBufferFileOffset)
1056 93 : PutRingBufferInCache();
1057 :
1058 : if (ENABLE_DEBUG)
1059 : CPLDebug("VSICURL", "Read [" CPL_FRMT_GUIB ", " CPL_FRMT_GUIB "[ in %s",
1060 : curOffset, curOffset + nBufferRequestSize, pszURL);
1061 :
1062 : /* Can we use the cache ? */
1063 214 : poFS->AcquireMutex();
1064 214 : CachedRegion* psRegion = poFS->GetRegion(pszURL);
1065 214 : if( psRegion != NULL && curOffset < psRegion->nSize )
1066 : {
1067 162 : size_t nSz = MIN(nRemaining, (size_t)(psRegion->nSize - curOffset));
1068 : if (ENABLE_DEBUG)
1069 : CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
1070 : (int)curOffset, (int)(curOffset + nSz), pszURL);
1071 162 : memcpy(pabyBuffer, psRegion->pData + curOffset, nSz);
1072 162 : pabyBuffer += nSz;
1073 162 : curOffset += nSz;
1074 162 : nRemaining -= nSz;
1075 : }
1076 :
1077 : /* Is the request partially covered by the cache and going beyond file size ? */
1078 214 : if ( psRegion != NULL && bHastComputedFileSizeLocal &&
1079 : curOffset <= psRegion->nSize &&
1080 : curOffset + nRemaining > fileSizeLocal &&
1081 : fileSize == psRegion->nSize )
1082 : {
1083 88 : size_t nSz = (size_t) (psRegion->nSize - curOffset);
1084 : if (ENABLE_DEBUG && nSz != 0)
1085 : CPLDebug("VSICURL", "Using cache for [%d, %d[ in %s",
1086 : (int)curOffset, (int)(curOffset + nSz), pszURL);
1087 88 : memcpy(pabyBuffer, psRegion->pData + curOffset, nSz);
1088 88 : pabyBuffer += nSz;
1089 88 : curOffset += nSz;
1090 88 : nRemaining -= nSz;
1091 88 : bEOF = TRUE;
1092 : }
1093 214 : poFS->ReleaseMutex();
1094 :
1095 : /* Has a Seek() being done since the last Read() ? */
1096 214 : if (!bEOF && nRemaining > 0 && curOffset != nRingBufferFileOffset)
1097 : {
1098 : /* Backward seek : we need to restart the download from the start */
1099 4 : if (curOffset < nRingBufferFileOffset)
1100 1 : StopDownload();
1101 :
1102 4 : StartDownload();
1103 :
1104 : #define SKIP_BUFFER_SIZE 32768
1105 4 : GByte* pabyTmp = (GByte*)CPLMalloc(SKIP_BUFFER_SIZE);
1106 :
1107 4 : CPLAssert(curOffset >= nRingBufferFileOffset);
1108 4 : vsi_l_offset nBytesToSkip = curOffset - nRingBufferFileOffset;
1109 4109 : while(nBytesToSkip > 0)
1110 : {
1111 4101 : vsi_l_offset nBytesToRead = nBytesToSkip;
1112 :
1113 4101 : AcquireMutex();
1114 4101 : if (nBytesToRead > oRingBuffer.GetSize())
1115 4097 : nBytesToRead = oRingBuffer.GetSize();
1116 4101 : if (nBytesToRead > SKIP_BUFFER_SIZE)
1117 0 : nBytesToRead = SKIP_BUFFER_SIZE;
1118 4101 : oRingBuffer.Read(pabyTmp, (size_t)nBytesToRead);
1119 :
1120 : /* Signal to the producer that we have ingested some bytes */
1121 4101 : CPLCondSignal(hCondConsumer);
1122 4101 : ReleaseMutex();
1123 :
1124 4101 : if (nBytesToRead)
1125 2051 : poFS->AddRegion(pszURL, nRingBufferFileOffset, (size_t)nBytesToRead, pabyTmp);
1126 :
1127 4101 : nBytesToSkip -= nBytesToRead;
1128 4101 : nRingBufferFileOffset += nBytesToRead;
1129 :
1130 4101 : if (nBytesToRead == 0 && nBytesToSkip != 0)
1131 : {
1132 : if (ENABLE_DEBUG)
1133 : CPLDebug("VSICURL", "Waiting for writer to produce some bytes...");
1134 :
1135 2050 : AcquireMutex();
1136 6150 : while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
1137 2050 : CPLCondWait(hCondProducer, hRingBufferMutex);
1138 2050 : int bBufferEmpty = (oRingBuffer.GetSize() == 0);
1139 2050 : ReleaseMutex();
1140 :
1141 2050 : if (bBufferEmpty && !bDownloadInProgress)
1142 0 : break;
1143 : }
1144 : }
1145 :
1146 4 : CPLFree(pabyTmp);
1147 :
1148 4 : if (nBytesToSkip != 0)
1149 : {
1150 0 : bEOF = TRUE;
1151 0 : return 0;
1152 : }
1153 : }
1154 :
1155 214 : if (!bEOF && nRemaining > 0)
1156 : {
1157 82 : StartDownload();
1158 82 : CPLAssert(curOffset == nRingBufferFileOffset);
1159 : }
1160 :
1161 : /* Fill the destination buffer from the ring buffer */
1162 3342 : while(!bEOF && nRemaining > 0)
1163 : {
1164 2927 : AcquireMutex();
1165 2927 : size_t nToRead = oRingBuffer.GetSize();
1166 2927 : if (nToRead > nRemaining)
1167 69 : nToRead = nRemaining;
1168 2927 : oRingBuffer.Read(pabyBuffer, nToRead);
1169 :
1170 : /* Signal to the producer that we have ingested some bytes */
1171 2927 : CPLCondSignal(hCondConsumer);
1172 2927 : ReleaseMutex();
1173 :
1174 2927 : if (nToRead)
1175 1472 : poFS->AddRegion(pszURL, curOffset, nToRead, pabyBuffer);
1176 :
1177 2927 : nRemaining -= nToRead;
1178 2927 : pabyBuffer += nToRead;
1179 2927 : curOffset += nToRead;
1180 2927 : nRingBufferFileOffset += nToRead;
1181 :
1182 2927 : if (nToRead == 0 && nRemaining != 0)
1183 : {
1184 : if (ENABLE_DEBUG)
1185 : CPLDebug("VSICURL", "Waiting for writer to produce some bytes...");
1186 :
1187 1455 : AcquireMutex();
1188 4353 : while(oRingBuffer.GetSize() == 0 && bDownloadInProgress)
1189 1443 : CPLCondWait(hCondProducer, hRingBufferMutex);
1190 1455 : int bBufferEmpty = (oRingBuffer.GetSize() == 0);
1191 1455 : ReleaseMutex();
1192 :
1193 1455 : if (bBufferEmpty && !bDownloadInProgress)
1194 13 : break;
1195 : }
1196 : }
1197 :
1198 : if (ENABLE_DEBUG)
1199 : CPLDebug("VSICURL", "Read(%d) = %d",
1200 : (int)nBufferRequestSize, (int)(nBufferRequestSize - nRemaining));
1201 214 : size_t nRet = (nBufferRequestSize - nRemaining) / nSize;
1202 214 : if (nRet < nMemb)
1203 101 : bEOF = TRUE;
1204 214 : return nRet;
1205 : }
1206 :
1207 : /************************************************************************/
1208 : /* Write() */
1209 : /************************************************************************/
1210 :
1211 0 : size_t VSICurlStreamingHandle::Write( const void *pBuffer, size_t nSize, size_t nMemb )
1212 : {
1213 0 : return 0;
1214 : }
1215 :
1216 : /************************************************************************/
1217 : /* Eof() */
1218 : /************************************************************************/
1219 :
1220 :
1221 244 : int VSICurlStreamingHandle::Eof()
1222 : {
1223 244 : return bEOF;
1224 : }
1225 :
1226 : /************************************************************************/
1227 : /* Flush() */
1228 : /************************************************************************/
1229 :
1230 0 : int VSICurlStreamingHandle::Flush()
1231 : {
1232 0 : return 0;
1233 : }
1234 :
1235 : /************************************************************************/
1236 : /* Close() */
1237 : /************************************************************************/
1238 :
1239 41 : int VSICurlStreamingHandle::Close()
1240 : {
1241 41 : return 0;
1242 : }
1243 :
1244 :
1245 : /************************************************************************/
1246 : /* VSICurlStreamingFSHandler() */
1247 : /************************************************************************/
1248 :
1249 712 : VSICurlStreamingFSHandler::VSICurlStreamingFSHandler()
1250 : {
1251 712 : hMutex = CPLCreateMutex();
1252 712 : CPLReleaseMutex(hMutex);
1253 712 : papsRegions = NULL;
1254 712 : nRegions = 0;
1255 712 : }
1256 :
1257 : /************************************************************************/
1258 : /* ~VSICurlStreamingFSHandler() */
1259 : /************************************************************************/
1260 :
1261 687 : VSICurlStreamingFSHandler::~VSICurlStreamingFSHandler()
1262 : {
1263 : int i;
1264 705 : for(i=0;i<nRegions;i++)
1265 : {
1266 18 : CPLFree(papsRegions[i]->pData);
1267 18 : CPLFree(papsRegions[i]);
1268 : }
1269 687 : CPLFree(papsRegions);
1270 :
1271 687 : std::map<CPLString, CachedFileProp*>::const_iterator iterCacheFileSize;
1272 :
1273 711 : for( iterCacheFileSize = cacheFileSize.begin();
1274 : iterCacheFileSize != cacheFileSize.end();
1275 : iterCacheFileSize++ )
1276 : {
1277 24 : CPLFree(iterCacheFileSize->second);
1278 : }
1279 :
1280 687 : CPLDestroyMutex( hMutex );
1281 687 : hMutex = NULL;
1282 687 : }
1283 :
1284 : /************************************************************************/
1285 : /* GetRegion() */
1286 : /************************************************************************/
1287 :
1288 3229 : void VSICurlStreamingFSHandler::AcquireMutex()
1289 : {
1290 3229 : CPLAcquireMutex(hMutex, 1000.0);
1291 3229 : }
1292 :
1293 : /************************************************************************/
1294 : /* GetRegion() */
1295 : /************************************************************************/
1296 :
1297 3229 : void VSICurlStreamingFSHandler::ReleaseMutex()
1298 : {
1299 3229 : CPLReleaseMutex(hMutex);
1300 3229 : }
1301 :
1302 : /************************************************************************/
1303 : /* GetRegion() */
1304 : /************************************************************************/
1305 :
1306 : /* Should be called undef the FS Lock */
1307 :
1308 3118 : CachedRegion* VSICurlStreamingFSHandler::GetRegion(const char* pszURL)
1309 : {
1310 3118 : unsigned long pszURLHash = CPLHashSetHashStr(pszURL);
1311 :
1312 : int i;
1313 3359 : for(i=0;i<nRegions;i++)
1314 : {
1315 3311 : CachedRegion* psRegion = papsRegions[i];
1316 3311 : if (psRegion->pszURLHash == pszURLHash)
1317 : {
1318 3070 : memmove(papsRegions + 1, papsRegions, i * sizeof(CachedRegion*));
1319 3070 : papsRegions[0] = psRegion;
1320 3070 : return psRegion;
1321 : }
1322 : }
1323 48 : return NULL;
1324 : }
1325 :
1326 : /************************************************************************/
1327 : /* AddRegion() */
1328 : /************************************************************************/
1329 :
1330 3565 : void VSICurlStreamingFSHandler::AddRegion( const char* pszURL,
1331 : vsi_l_offset nFileOffsetStart,
1332 : size_t nSize,
1333 : GByte *pData )
1334 : {
1335 3565 : if (nFileOffsetStart >= BKGND_BUFFER_SIZE)
1336 661 : return;
1337 :
1338 2904 : AcquireMutex();
1339 :
1340 2904 : unsigned long pszURLHash = CPLHashSetHashStr(pszURL);
1341 :
1342 2904 : CachedRegion* psRegion = GetRegion(pszURL);
1343 2904 : if (psRegion == NULL)
1344 : {
1345 24 : if (nRegions == N_MAX_REGIONS)
1346 : {
1347 6 : psRegion = papsRegions[N_MAX_REGIONS-1];
1348 : memmove(papsRegions + 1, papsRegions,
1349 6 : (N_MAX_REGIONS-1) * sizeof(CachedRegion*));
1350 6 : papsRegions[0] = psRegion;
1351 : }
1352 : else
1353 : {
1354 : papsRegions = (CachedRegion**)
1355 18 : CPLRealloc(papsRegions, (nRegions + 1) * sizeof(CachedRegion*));
1356 18 : if (nRegions)
1357 15 : memmove(papsRegions + 1, papsRegions, nRegions * sizeof(CachedRegion*));
1358 18 : nRegions ++;
1359 18 : papsRegions[0] = psRegion =
1360 18 : (CachedRegion*) CPLCalloc(1, sizeof(CachedRegion));
1361 : }
1362 :
1363 24 : psRegion->pszURLHash = pszURLHash;
1364 24 : psRegion->nSize = 0;
1365 24 : if (psRegion->pData == NULL)
1366 18 : psRegion->pData = (GByte*) CPLMalloc(BKGND_BUFFER_SIZE);
1367 : }
1368 :
1369 2904 : if (nFileOffsetStart <= psRegion->nSize &&
1370 : nFileOffsetStart + nSize > psRegion->nSize)
1371 : {
1372 1557 : size_t nSz = MIN(nSize, (size_t) (BKGND_BUFFER_SIZE - nFileOffsetStart));
1373 : if (ENABLE_DEBUG)
1374 : CPLDebug("VSICURL", "Writing [%d, %d[ in cache for %s",
1375 : (int)nFileOffsetStart, (int)(nFileOffsetStart + nSz), pszURL);
1376 1557 : memcpy(psRegion->pData + nFileOffsetStart, pData, nSz);
1377 1557 : psRegion->nSize = (size_t) (nFileOffsetStart + nSz);
1378 : }
1379 :
1380 2904 : ReleaseMutex();
1381 : }
1382 :
1383 : /************************************************************************/
1384 : /* GetCachedFileProp() */
1385 : /************************************************************************/
1386 :
1387 : /* Should be called undef the FS Lock */
1388 :
1389 111 : CachedFileProp* VSICurlStreamingFSHandler::GetCachedFileProp(const char* pszURL)
1390 : {
1391 111 : CachedFileProp* cachedFileProp = cacheFileSize[pszURL];
1392 111 : if (cachedFileProp == NULL)
1393 : {
1394 24 : cachedFileProp = (CachedFileProp*) CPLMalloc(sizeof(CachedFileProp));
1395 24 : cachedFileProp->eExists = EXIST_UNKNOWN;
1396 24 : cachedFileProp->bHastComputedFileSize = FALSE;
1397 24 : cachedFileProp->fileSize = 0;
1398 24 : cachedFileProp->bIsDirectory = FALSE;
1399 24 : cacheFileSize[pszURL] = cachedFileProp;
1400 : }
1401 :
1402 111 : return cachedFileProp;
1403 : }
1404 :
1405 : /************************************************************************/
1406 : /* Open() */
1407 : /************************************************************************/
1408 :
1409 41 : VSIVirtualHandle* VSICurlStreamingFSHandler::Open( const char *pszFilename,
1410 : const char *pszAccess )
1411 : {
1412 41 : if (strchr(pszAccess, 'w') != NULL ||
1413 : strchr(pszAccess, '+') != NULL)
1414 : {
1415 : CPLError(CE_Failure, CPLE_AppDefined,
1416 0 : "Only read-only mode is supported for /vsicurl_streaming");
1417 0 : return NULL;
1418 : }
1419 :
1420 : VSICurlStreamingHandle* poHandle = new VSICurlStreamingHandle(
1421 41 : this, pszFilename + strlen("/vsicurl_streaming/"));
1422 : /* If we didn't get a filelist, check that the file really exists */
1423 41 : if (!poHandle->Exists())
1424 : {
1425 0 : delete poHandle;
1426 0 : poHandle = NULL;
1427 : }
1428 41 : return poHandle;
1429 : }
1430 :
1431 : /************************************************************************/
1432 : /* Stat() */
1433 : /************************************************************************/
1434 :
1435 0 : int VSICurlStreamingFSHandler::Stat( const char *pszFilename,
1436 : VSIStatBufL *pStatBuf,
1437 : int nFlags )
1438 : {
1439 0 : CPLString osFilename(pszFilename);
1440 :
1441 0 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
1442 :
1443 0 : VSICurlStreamingHandle oHandle( this, osFilename + strlen("/vsicurl_streaming/"));
1444 :
1445 0 : if ( oHandle.IsKnownFileSize() ||
1446 : ((nFlags & VSI_STAT_SIZE_FLAG) && !oHandle.IsDirectory() &&
1447 : CSLTestBoolean(CPLGetConfigOption("CPL_VSIL_CURL_SLOW_GET_SIZE", "YES"))) )
1448 0 : pStatBuf->st_size = oHandle.GetFileSize();
1449 :
1450 0 : int nRet = (oHandle.Exists()) ? 0 : -1;
1451 0 : pStatBuf->st_mode = oHandle.IsDirectory() ? S_IFDIR : S_IFREG;
1452 0 : return nRet;
1453 : }
1454 :
1455 : /************************************************************************/
1456 : /* VSIInstallCurlFileHandler() */
1457 : /************************************************************************/
1458 :
1459 : /**
1460 : * \brief Install /vsicurl_streaming/ HTTP/FTP file system handler (requires libcurl)
1461 : *
1462 : * A special file handler is installed that allows on-the-fly reading of files
1463 : * streamed through HTTP/FTP web protocols (typically dynamically generated files),
1464 : * without downloading the entire file.
1465 : *
1466 : * Although this file handler is able seek to random offsets in the file, this will not
1467 : * be efficient. If you need efficient random access and that the server supports range
1468 : * dowloading, you should use the /vsicurl/ file system handler instead.
1469 : *
1470 : * Recognized filenames are of the form /vsicurl_streaming/http://path/to/remote/ressource or
1471 : * /vsicurl_streaming/ftp://path/to/remote/ressource where path/to/remote/ressource is the
1472 : * URL of a remote ressource.
1473 : *
1474 : * The GDAL_HTTP_PROXY and GDAL_HTTP_PROXYUSERPWD configuration options can be
1475 : * used to define a proxy server. The syntax to use is the one of Curl CURLOPT_PROXY
1476 : * and CURLOPT_PROXYUSERPWD options.
1477 : *
1478 : * VSIStatL() will return the size in st_size member and file
1479 : * nature- file or directory - in st_mode member (the later only reliable with FTP
1480 : * resources for now).
1481 : *
1482 : * @since GDAL 1.10
1483 : */
1484 712 : void VSIInstallCurlStreamingFileHandler(void)
1485 : {
1486 712 : VSIFileManager::InstallHandler( "/vsicurl_streaming/", new VSICurlStreamingFSHandler );
1487 712 : }
1488 :
1489 : #ifdef AS_PLUGIN
1490 : CPL_C_START
1491 : void CPL_DLL GDALRegisterMe();
1492 : void GDALRegisterMe()
1493 : {
1494 : VSIInstallCurlStreamingFileHandler();
1495 : }
1496 : CPL_C_END
1497 : #endif
1498 :
1499 : #endif /* !defined(HAVE_CURL) || defined(CPL_MULTIPROC_STUB) */
|
