1 : /******************************************************************************
2 : * $Id: cpl_vsil_curl.cpp 25696 2013-03-02 17:55:51Z rouault $
3 : *
4 : * Project: CPL - Common Portability Library
5 : * Purpose: Implement VSI large file api for HTTP/FTP files
6 : * Author: Even Rouault, even.rouault at mines-paris.org
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2008, Even Rouault
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_string.h"
32 : #include "cpl_multiproc.h"
33 : #include "cpl_hash_set.h"
34 : #include "cpl_time.h"
35 : #include "cpl_vsil_curl_priv.h"
36 :
37 : CPL_CVSID("$Id: cpl_vsil_curl.cpp 25696 2013-03-02 17:55:51Z rouault $");
38 :
39 : #ifndef HAVE_CURL
40 :
41 : void VSIInstallCurlFileHandler(void)
42 : {
43 : /* not supported */
44 : }
45 :
46 : /************************************************************************/
47 : /* VSICurlInstallReadCbk() */
48 : /************************************************************************/
49 :
50 : int VSICurlInstallReadCbk (VSILFILE* fp,
51 : VSICurlReadCbkFunc pfnReadCbk,
52 : void* pfnUserData,
53 : int bStopOnInterrruptUntilUninstall)
54 : {
55 : return FALSE;
56 : }
57 :
58 :
59 : /************************************************************************/
60 : /* VSICurlUninstallReadCbk() */
61 : /************************************************************************/
62 :
63 : int VSICurlUninstallReadCbk(VSILFILE* fp)
64 : {
65 : return FALSE;
66 : }
67 :
68 : #else
69 :
70 : #include <curl/curl.h>
71 :
72 : void CPLHTTPSetOptions(CURL *http_handle, char** papszOptions);
73 : void VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL);
74 :
75 : #include <map>
76 :
77 : #define ENABLE_DEBUG 1
78 :
79 : #define N_MAX_REGIONS 1000
80 :
81 : #define DOWNLOAD_CHUNCK_SIZE 16384
82 :
83 : typedef enum
84 : {
85 : EXIST_UNKNOWN = -1,
86 : EXIST_NO,
87 : EXIST_YES,
88 : } ExistStatus;
89 :
90 : typedef struct
91 : {
92 : ExistStatus eExists;
93 : int bHastComputedFileSize;
94 : vsi_l_offset fileSize;
95 : int bIsDirectory;
96 : time_t mTime;
97 : } CachedFileProp;
98 :
99 : typedef struct
100 : {
101 : int bGotFileList;
102 : char** papszFileList; /* only file name without path */
103 : } CachedDirList;
104 :
105 : typedef struct
106 : {
107 : unsigned long pszURLHash;
108 : vsi_l_offset nFileOffsetStart;
109 : size_t nSize;
110 : char *pData;
111 : } CachedRegion;
112 :
113 :
114 0 : static const char* VSICurlGetCacheFileName()
115 : {
116 0 : return "gdal_vsicurl_cache.bin";
117 : }
118 :
119 : /************************************************************************/
120 : /* VSICurlFindStringSensitiveExceptEscapeSequences() */
121 : /************************************************************************/
122 :
123 75 : static int VSICurlFindStringSensitiveExceptEscapeSequences( char ** papszList,
124 : const char * pszTarget )
125 :
126 : {
127 : int i;
128 :
129 75 : if( papszList == NULL )
130 34 : return -1;
131 :
132 170 : for( i = 0; papszList[i] != NULL; i++ )
133 : {
134 158 : const char* pszIter1 = papszList[i];
135 158 : const char* pszIter2 = pszTarget;
136 : char ch1, ch2;
137 : /* The comparison is case-sensitive, escape for escaped */
138 : /* sequences where letters of the hexadecimal sequence */
139 : /* can be uppercase or lowercase depending on the quoting algorithm */
140 629 : while(TRUE)
141 : {
142 787 : ch1 = *pszIter1;
143 787 : ch2 = *pszIter2;
144 787 : if (ch1 == '\0' || ch2 == '\0')
145 31 : break;
146 756 : if (ch1 == '%' && ch2 == '%' &&
147 0 : pszIter1[1] != '\0' && pszIter1[2] != '\0' &&
148 0 : pszIter2[1] != '\0' && pszIter2[2] != '\0')
149 : {
150 0 : if (!EQUALN(pszIter1+1, pszIter2+1, 2))
151 0 : break;
152 0 : pszIter1 += 2;
153 0 : pszIter2 += 2;
154 : }
155 756 : if (ch1 != ch2)
156 127 : break;
157 629 : pszIter1 ++;
158 629 : pszIter2 ++;
159 : }
160 158 : if (ch1 == ch2 && ch1 == '\0')
161 29 : return i;
162 : }
163 :
164 12 : return -1;
165 : }
166 :
167 : /************************************************************************/
168 : /* VSICurlIsFileInList() */
169 : /************************************************************************/
170 :
171 58 : static int VSICurlIsFileInList( char ** papszList, const char * pszTarget )
172 : {
173 58 : int nRet = VSICurlFindStringSensitiveExceptEscapeSequences(papszList, pszTarget);
174 58 : if (nRet >= 0)
175 29 : return nRet;
176 :
177 : /* If we didn't find anything, try to URL-escape the target filename */
178 29 : char* pszEscaped = CPLEscapeString(pszTarget, -1, CPLES_URL);
179 29 : if (strcmp(pszTarget, pszEscaped) != 0)
180 : {
181 17 : nRet = VSICurlFindStringSensitiveExceptEscapeSequences(papszList, pszEscaped);
182 : }
183 29 : CPLFree(pszEscaped);
184 29 : return nRet;
185 : }
186 :
187 : /************************************************************************/
188 : /* VSICurlFilesystemHandler */
189 : /************************************************************************/
190 :
191 : typedef struct
192 : {
193 : CPLString osURL;
194 : CURL *hCurlHandle;
195 4 : } CachedConnection;
196 :
197 :
198 : class VSICurlFilesystemHandler : public VSIFilesystemHandler
199 : {
200 : void *hMutex;
201 :
202 : CachedRegion **papsRegions;
203 : int nRegions;
204 :
205 : std::map<CPLString, CachedFileProp*> cacheFileSize;
206 : std::map<CPLString, CachedDirList*> cacheDirList;
207 :
208 : int bUseCacheDisk;
209 :
210 : /* Per-thread Curl connection cache */
211 : std::map<GIntBig, CachedConnection*> mapConnections;
212 :
213 : char** GetFileList(const char *pszFilename, int* pbGotFileList);
214 :
215 : char** ParseHTMLFileList(const char* pszFilename,
216 : char* pszData,
217 : int* pbGotFileList);
218 : public:
219 : VSICurlFilesystemHandler();
220 : ~VSICurlFilesystemHandler();
221 :
222 : virtual VSIVirtualHandle *Open( const char *pszFilename,
223 : const char *pszAccess);
224 : virtual int Stat( const char *pszFilename, VSIStatBufL *pStatBuf, int nFlags );
225 : virtual int Unlink( const char *pszFilename );
226 : virtual int Rename( const char *oldpath, const char *newpath );
227 : virtual int Mkdir( const char *pszDirname, long nMode );
228 : virtual int Rmdir( const char *pszDirname );
229 : virtual char **ReadDir( const char *pszDirname );
230 : virtual char **ReadDir( const char *pszDirname, int* pbGotFileList );
231 :
232 :
233 : const CachedRegion* GetRegion(const char* pszURL,
234 : vsi_l_offset nFileOffsetStart);
235 :
236 : void AddRegion(const char* pszURL,
237 : vsi_l_offset nFileOffsetStart,
238 : size_t nSize,
239 : const char *pData);
240 :
241 : CachedFileProp* GetCachedFileProp(const char* pszURL);
242 :
243 : void AddRegionToCacheDisk(CachedRegion* psRegion);
244 : const CachedRegion* GetRegionFromCacheDisk(const char* pszURL,
245 : vsi_l_offset nFileOffsetStart);
246 :
247 : CURL *GetCurlHandleFor(CPLString osURL);
248 : };
249 :
250 : /************************************************************************/
251 : /* VSICurlHandle */
252 : /************************************************************************/
253 :
254 : class VSICurlHandle : public VSIVirtualHandle
255 : {
256 : private:
257 : VSICurlFilesystemHandler* poFS;
258 :
259 : char* pszURL;
260 :
261 : vsi_l_offset curOffset;
262 : vsi_l_offset fileSize;
263 : int bHastComputedFileSize;
264 : ExistStatus eExists;
265 : int bIsDirectory;
266 : time_t mTime;
267 :
268 : vsi_l_offset lastDownloadedOffset;
269 : int nBlocksToDownload;
270 : int bEOF;
271 :
272 : int DownloadRegion(vsi_l_offset startOffset, int nBlocks);
273 :
274 : VSICurlReadCbkFunc pfnReadCbk;
275 : void *pReadCbkUserData;
276 : int bStopOnInterrruptUntilUninstall;
277 : int bInterrupted;
278 :
279 : public:
280 :
281 : VSICurlHandle(VSICurlFilesystemHandler* poFS, const char* pszURL);
282 : ~VSICurlHandle();
283 :
284 : virtual int Seek( vsi_l_offset nOffset, int nWhence );
285 : virtual vsi_l_offset Tell();
286 : virtual size_t Read( void *pBuffer, size_t nSize, size_t nMemb );
287 : virtual int ReadMultiRange( int nRanges, void ** ppData,
288 : const vsi_l_offset* panOffsets, const size_t* panSizes );
289 : virtual size_t Write( const void *pBuffer, size_t nSize, size_t nMemb );
290 : virtual int Eof();
291 : virtual int Flush();
292 : virtual int Close();
293 :
294 818 : int IsKnownFileSize() const { return bHastComputedFileSize; }
295 : vsi_l_offset GetFileSize();
296 : int Exists();
297 818 : int IsDirectory() const { return bIsDirectory; }
298 818 : time_t GetMTime() const { return mTime; }
299 :
300 : int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk,
301 : void* pfnUserData,
302 : int bStopOnInterrruptUntilUninstall);
303 : int UninstallReadCbk();
304 : };
305 :
306 : /************************************************************************/
307 : /* VSICurlHandle() */
308 : /************************************************************************/
309 :
310 894 : VSICurlHandle::VSICurlHandle(VSICurlFilesystemHandler* poFS, const char* pszURL)
311 : {
312 894 : this->poFS = poFS;
313 894 : this->pszURL = CPLStrdup(pszURL);
314 :
315 894 : curOffset = 0;
316 :
317 894 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
318 894 : eExists = cachedFileProp->eExists;
319 894 : fileSize = cachedFileProp->fileSize;
320 894 : bHastComputedFileSize = cachedFileProp->bHastComputedFileSize;
321 894 : bIsDirectory = cachedFileProp->bIsDirectory;
322 894 : mTime = cachedFileProp->mTime;
323 :
324 894 : lastDownloadedOffset = -1;
325 894 : nBlocksToDownload = 1;
326 894 : bEOF = FALSE;
327 :
328 894 : pfnReadCbk = NULL;
329 894 : pReadCbkUserData = NULL;
330 894 : bStopOnInterrruptUntilUninstall = FALSE;
331 894 : bInterrupted = FALSE;
332 894 : }
333 :
334 : /************************************************************************/
335 : /* ~VSICurlHandle() */
336 : /************************************************************************/
337 :
338 894 : VSICurlHandle::~VSICurlHandle()
339 : {
340 894 : CPLFree(pszURL);
341 894 : }
342 :
343 : /************************************************************************/
344 : /* InstallReadCbk() */
345 : /************************************************************************/
346 :
347 1 : int VSICurlHandle::InstallReadCbk(VSICurlReadCbkFunc pfnReadCbkIn,
348 : void* pfnUserDataIn,
349 : int bStopOnInterrruptUntilUninstallIn)
350 : {
351 1 : if (pfnReadCbk != NULL)
352 0 : return FALSE;
353 :
354 1 : pfnReadCbk = pfnReadCbkIn;
355 1 : pReadCbkUserData = pfnUserDataIn;
356 1 : bStopOnInterrruptUntilUninstall = bStopOnInterrruptUntilUninstallIn;
357 1 : bInterrupted = FALSE;
358 1 : return TRUE;
359 : }
360 :
361 : /************************************************************************/
362 : /* UninstallReadCbk() */
363 : /************************************************************************/
364 :
365 1 : int VSICurlHandle::UninstallReadCbk()
366 : {
367 1 : if (pfnReadCbk == NULL)
368 0 : return FALSE;
369 :
370 1 : pfnReadCbk = NULL;
371 1 : pReadCbkUserData = NULL;
372 1 : bStopOnInterrruptUntilUninstall = FALSE;
373 1 : bInterrupted = FALSE;
374 1 : return TRUE;
375 : }
376 :
377 : /************************************************************************/
378 : /* Seek() */
379 : /************************************************************************/
380 :
381 579 : int VSICurlHandle::Seek( vsi_l_offset nOffset, int nWhence )
382 : {
383 579 : if (nWhence == SEEK_SET)
384 : {
385 362 : curOffset = nOffset;
386 : }
387 217 : else if (nWhence == SEEK_CUR)
388 : {
389 152 : curOffset = curOffset + nOffset;
390 : }
391 : else
392 : {
393 65 : curOffset = GetFileSize() + nOffset;
394 : }
395 579 : bEOF = FALSE;
396 579 : return 0;
397 : }
398 :
399 : /************************************************************************/
400 : /* VSICurlSetOptions() */
401 : /************************************************************************/
402 :
403 130 : void VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL)
404 : {
405 130 : curl_easy_setopt(hCurlHandle, CURLOPT_URL, pszURL);
406 :
407 130 : CPLHTTPSetOptions(hCurlHandle, NULL);
408 :
409 : /* 7.16 */
410 : #if LIBCURL_VERSION_NUM >= 0x071000
411 130 : long option = CURLFTPMETHOD_SINGLECWD;
412 130 : curl_easy_setopt(hCurlHandle, CURLOPT_FTP_FILEMETHOD, option);
413 : #endif
414 :
415 : /* 7.12.3 */
416 : #if LIBCURL_VERSION_NUM > 0x070C03
417 : /* ftp://ftp2.cits.rncan.gc.ca/pub/cantopo/250k_tif/ doesn't like EPSV command */
418 130 : curl_easy_setopt(hCurlHandle, CURLOPT_FTP_USE_EPSV, 0);
419 : #endif
420 :
421 130 : curl_easy_setopt(hCurlHandle, CURLOPT_NOBODY, 0);
422 130 : curl_easy_setopt(hCurlHandle, CURLOPT_HTTPGET, 1);
423 130 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADER, 0);
424 :
425 : /* 7.16.4 */
426 : #if LIBCURL_VERSION_NUM <= 0x071004
427 : curl_easy_setopt(hCurlHandle, CURLOPT_FTPLISTONLY, 0);
428 : #elif LIBCURL_VERSION_NUM > 0x071004
429 130 : curl_easy_setopt(hCurlHandle, CURLOPT_DIRLISTONLY, 0);
430 : #endif
431 :
432 130 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, NULL);
433 130 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, NULL);
434 130 : }
435 :
436 :
437 : typedef struct
438 : {
439 : char* pBuffer;
440 : size_t nSize;
441 : int bIsHTTP;
442 : int bIsInHeader;
443 : int bMultiRange;
444 : vsi_l_offset nStartOffset;
445 : vsi_l_offset nEndOffset;
446 : int nHTTPCode;
447 : vsi_l_offset nContentLength;
448 : int bFoundContentRange;
449 : int bError;
450 : int bDownloadHeaderOnly;
451 :
452 : VSILFILE *fp;
453 : VSICurlReadCbkFunc pfnReadCbk;
454 : void *pReadCbkUserData;
455 : int bInterrupted;
456 : } WriteFuncStruct;
457 :
458 : /************************************************************************/
459 : /* VSICURLInitWriteFuncStruct() */
460 : /************************************************************************/
461 :
462 147 : static void VSICURLInitWriteFuncStruct(WriteFuncStruct *psStruct,
463 : VSILFILE *fp,
464 : VSICurlReadCbkFunc pfnReadCbk,
465 : void *pReadCbkUserData)
466 : {
467 147 : psStruct->pBuffer = NULL;
468 147 : psStruct->nSize = 0;
469 147 : psStruct->bIsHTTP = FALSE;
470 147 : psStruct->bIsInHeader = TRUE;
471 147 : psStruct->bMultiRange = FALSE;
472 147 : psStruct->nStartOffset = 0;
473 147 : psStruct->nEndOffset = 0;
474 147 : psStruct->nHTTPCode = 0;
475 147 : psStruct->nContentLength = 0;
476 147 : psStruct->bFoundContentRange = FALSE;
477 147 : psStruct->bError = FALSE;
478 147 : psStruct->bDownloadHeaderOnly = FALSE;
479 :
480 147 : psStruct->fp = fp;
481 147 : psStruct->pfnReadCbk = pfnReadCbk;
482 147 : psStruct->pReadCbkUserData = pReadCbkUserData;
483 147 : psStruct->bInterrupted = FALSE;
484 147 : }
485 :
486 : /************************************************************************/
487 : /* VSICurlHandleWriteFunc() */
488 : /************************************************************************/
489 :
490 1363 : static int VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb, void *req)
491 : {
492 1363 : WriteFuncStruct* psStruct = (WriteFuncStruct*) req;
493 1363 : size_t nSize = count * nmemb;
494 :
495 : char* pNewBuffer = (char*) VSIRealloc(psStruct->pBuffer,
496 1363 : psStruct->nSize + nSize + 1);
497 1363 : if (pNewBuffer)
498 : {
499 1363 : psStruct->pBuffer = pNewBuffer;
500 1363 : memcpy(psStruct->pBuffer + psStruct->nSize, buffer, nSize);
501 1363 : psStruct->pBuffer[psStruct->nSize + nSize] = '\0';
502 1938 : if (psStruct->bIsHTTP && psStruct->bIsInHeader)
503 : {
504 577 : char* pszLine = psStruct->pBuffer + psStruct->nSize;
505 620 : if (EQUALN(pszLine, "HTTP/1.0 ", 9) ||
506 : EQUALN(pszLine, "HTTP/1.1 ", 9))
507 43 : psStruct->nHTTPCode = atoi(pszLine + 9);
508 534 : else if (EQUALN(pszLine, "Content-Length: ", 16))
509 : psStruct->nContentLength = CPLScanUIntBig(pszLine + 16,
510 42 : strlen(pszLine + 16));
511 492 : else if (EQUALN(pszLine, "Content-Range: ", 15))
512 9 : psStruct->bFoundContentRange = TRUE;
513 :
514 : /*if (nSize > 2 && pszLine[nSize - 2] == '\r' &&
515 : pszLine[nSize - 1] == '\n')
516 : {
517 : pszLine[nSize - 2] = 0;
518 : CPLDebug("VSICURL", "%s", pszLine);
519 : pszLine[nSize - 2] = '\r';
520 : }*/
521 :
522 577 : if (pszLine[0] == '\r' || pszLine[0] == '\n')
523 : {
524 43 : if (psStruct->bDownloadHeaderOnly)
525 : {
526 : /* If moved permanently/temporarily, go on. Otherwise stop now*/
527 3 : if (!(psStruct->nHTTPCode == 301 || psStruct->nHTTPCode == 302))
528 2 : return 0;
529 : }
530 : else
531 : {
532 40 : psStruct->bIsInHeader = FALSE;
533 :
534 : /* Detect servers that don't support range downloading */
535 40 : if (psStruct->nHTTPCode == 200 &&
536 : !psStruct->bMultiRange &&
537 : !psStruct->bFoundContentRange &&
538 : (psStruct->nStartOffset != 0 || psStruct->nContentLength > 10 *
539 : (psStruct->nEndOffset - psStruct->nStartOffset + 1)))
540 : {
541 : CPLError(CE_Failure, CPLE_AppDefined,
542 0 : "Range downloading not supported by this server !");
543 0 : psStruct->bError = TRUE;
544 0 : return 0;
545 : }
546 : }
547 : }
548 : }
549 : else
550 : {
551 786 : if (psStruct->pfnReadCbk)
552 : {
553 2 : if ( ! psStruct->pfnReadCbk(psStruct->fp, buffer, nSize,
554 : psStruct->pReadCbkUserData) )
555 : {
556 1 : psStruct->bInterrupted = TRUE;
557 1 : return 0;
558 : }
559 : }
560 : }
561 1360 : psStruct->nSize += nSize;
562 1360 : return nmemb;
563 : }
564 : else
565 : {
566 0 : return 0;
567 : }
568 : }
569 :
570 :
571 : /************************************************************************/
572 : /* GetFileSize() */
573 : /************************************************************************/
574 :
575 118 : vsi_l_offset VSICurlHandle::GetFileSize()
576 : {
577 : WriteFuncStruct sWriteFuncData;
578 : WriteFuncStruct sWriteFuncHeaderData;
579 :
580 118 : if (bHastComputedFileSize)
581 96 : return fileSize;
582 :
583 22 : bHastComputedFileSize = TRUE;
584 :
585 : /* Consider that only the files whose extension ends up with one that is */
586 : /* listed in CPL_VSIL_CURL_ALLOWED_EXTENSIONS exist on the server */
587 : /* This can speeds up dramatically open experience, in case the server */
588 : /* cannot return a file list */
589 : /* For example : */
590 : /* gdalinfo --config CPL_VSIL_CURL_ALLOWED_EXTENSIONS ".tif" /vsicurl/http://igskmncngs506.cr.usgs.gov/gmted/Global_tiles_GMTED/075darcsec/bln/W030/30N030W_20101117_gmted_bln075.tif */
591 : const char* pszAllowedExtensions =
592 22 : CPLGetConfigOption("CPL_VSIL_CURL_ALLOWED_EXTENSIONS", NULL);
593 22 : if (pszAllowedExtensions)
594 : {
595 1 : char** papszExtensions = CSLTokenizeString2( pszAllowedExtensions, ", ", 0 );
596 1 : int nURLLen = strlen(pszURL);
597 1 : int bFound = FALSE;
598 1 : for(int i=0;papszExtensions[i] != NULL;i++)
599 : {
600 1 : int nExtensionLen = strlen(papszExtensions[i]);
601 2 : if (nURLLen > nExtensionLen &&
602 1 : EQUAL(pszURL + nURLLen - nExtensionLen, papszExtensions[i]))
603 : {
604 1 : bFound = TRUE;
605 1 : break;
606 : }
607 : }
608 :
609 1 : if (!bFound)
610 : {
611 0 : eExists = EXIST_NO;
612 0 : fileSize = 0;
613 :
614 0 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
615 0 : cachedFileProp->bHastComputedFileSize = TRUE;
616 0 : cachedFileProp->fileSize = fileSize;
617 0 : cachedFileProp->eExists = eExists;
618 :
619 0 : CSLDestroy(papszExtensions);
620 :
621 0 : return 0;
622 : }
623 :
624 1 : CSLDestroy(papszExtensions);
625 : }
626 :
627 : #if LIBCURL_VERSION_NUM < 0x070B00
628 : /* Curl 7.10.X doesn't manage to unset the CURLOPT_RANGE that would have been */
629 : /* previously set, so we have to reinit the connection handle */
630 : poFS->GetCurlHandleFor("");
631 : #endif
632 22 : CURL* hCurlHandle = poFS->GetCurlHandleFor(pszURL);
633 :
634 22 : VSICurlSetOptions(hCurlHandle, pszURL);
635 :
636 22 : VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, NULL, NULL, NULL);
637 :
638 : /* HACK for mbtiles driver: proper fix would be to auto-detect servers that don't accept HEAD */
639 : /* http://a.tiles.mapbox.com/v3/ doesn't accept HEAD, so let's start a GET */
640 : /* and interrupt is as soon as the header is found */
641 22 : if (strstr(pszURL, ".tiles.mapbox.com/") != NULL)
642 : {
643 2 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &sWriteFuncHeaderData);
644 2 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, VSICurlHandleWriteFunc);
645 :
646 2 : sWriteFuncHeaderData.bIsHTTP = strncmp(pszURL, "http", 4) == 0;
647 2 : sWriteFuncHeaderData.bDownloadHeaderOnly = TRUE;
648 : }
649 : else
650 : {
651 20 : curl_easy_setopt(hCurlHandle, CURLOPT_NOBODY, 1);
652 20 : curl_easy_setopt(hCurlHandle, CURLOPT_HTTPGET, 0);
653 20 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADER, 1);
654 : }
655 :
656 : /* We need that otherwise OSGEO4W's libcurl issue a dummy range request */
657 : /* when doing a HEAD when recycling connections */
658 22 : curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, NULL);
659 :
660 : /* Bug with older curl versions (<=7.16.4) and FTP. See http://curl.haxx.se/mail/lib-2007-08/0312.html */
661 22 : VSICURLInitWriteFuncStruct(&sWriteFuncData, NULL, NULL, NULL);
662 22 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
663 22 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlHandleWriteFunc);
664 :
665 : char szCurlErrBuf[CURL_ERROR_SIZE+1];
666 22 : szCurlErrBuf[0] = '\0';
667 22 : curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
668 :
669 22 : double dfSize = 0;
670 22 : curl_easy_perform(hCurlHandle);
671 :
672 22 : eExists = EXIST_UNKNOWN;
673 :
674 22 : if (strncmp(pszURL, "ftp", 3) == 0)
675 : {
676 1 : if (sWriteFuncData.pBuffer != NULL &&
677 : strncmp(sWriteFuncData.pBuffer, "Content-Length: ", strlen( "Content-Length: ")) == 0)
678 : {
679 1 : const char* pszBuffer = sWriteFuncData.pBuffer + strlen("Content-Length: ");
680 1 : eExists = EXIST_YES;
681 1 : fileSize = CPLScanUIntBig(pszBuffer, sWriteFuncData.nSize - strlen("Content-Length: "));
682 : if (ENABLE_DEBUG)
683 : CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB,
684 1 : pszURL, fileSize);
685 : }
686 : }
687 :
688 22 : if (eExists != EXIST_YES)
689 : {
690 21 : CURLcode code = curl_easy_getinfo(hCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dfSize );
691 21 : if (code == 0)
692 : {
693 21 : eExists = EXIST_YES;
694 21 : if (dfSize < 0)
695 0 : fileSize = 0;
696 : else
697 21 : fileSize = (GUIntBig)dfSize;
698 : }
699 : else
700 : {
701 0 : eExists = EXIST_NO;
702 0 : fileSize = 0;
703 0 : CPLError(CE_Failure, CPLE_AppDefined, "VSICurlHandle::GetFileSize failed");
704 : }
705 :
706 21 : long response_code = 0;
707 21 : curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code);
708 21 : if (response_code != 200)
709 : {
710 10 : eExists = EXIST_NO;
711 10 : fileSize = 0;
712 : }
713 :
714 : /* Try to guess if this is a directory. Generally if this is a directory, */
715 : /* curl will retry with an URL with slash added */
716 21 : char *pszEffectiveURL = NULL;
717 21 : curl_easy_getinfo(hCurlHandle, CURLINFO_EFFECTIVE_URL, &pszEffectiveURL);
718 41 : if (pszEffectiveURL != NULL && strncmp(pszURL, pszEffectiveURL, strlen(pszURL)) == 0 &&
719 20 : pszEffectiveURL[strlen(pszURL)] == '/')
720 : {
721 1 : eExists = EXIST_YES;
722 1 : fileSize = 0;
723 1 : bIsDirectory = TRUE;
724 : }
725 :
726 : if (ENABLE_DEBUG)
727 : CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d",
728 21 : pszURL, fileSize, (int)response_code);
729 : }
730 :
731 22 : CPLFree(sWriteFuncData.pBuffer);
732 22 : CPLFree(sWriteFuncHeaderData.pBuffer);
733 :
734 22 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
735 22 : cachedFileProp->bHastComputedFileSize = TRUE;
736 22 : cachedFileProp->fileSize = fileSize;
737 22 : cachedFileProp->eExists = eExists;
738 22 : cachedFileProp->bIsDirectory = bIsDirectory;
739 :
740 22 : return fileSize;
741 : }
742 :
743 : /************************************************************************/
744 : /* Exists() */
745 : /************************************************************************/
746 :
747 835 : int VSICurlHandle::Exists()
748 : {
749 835 : if (eExists == EXIST_UNKNOWN)
750 20 : GetFileSize();
751 835 : return eExists == EXIST_YES;
752 : }
753 :
754 : /************************************************************************/
755 : /* Tell() */
756 : /************************************************************************/
757 :
758 168 : vsi_l_offset VSICurlHandle::Tell()
759 : {
760 168 : return curOffset;
761 : }
762 :
763 : /************************************************************************/
764 : /* DownloadRegion() */
765 : /************************************************************************/
766 :
767 42 : int VSICurlHandle::DownloadRegion(vsi_l_offset startOffset, int nBlocks)
768 : {
769 : WriteFuncStruct sWriteFuncData;
770 : WriteFuncStruct sWriteFuncHeaderData;
771 :
772 42 : if (bInterrupted && bStopOnInterrruptUntilUninstall)
773 0 : return FALSE;
774 :
775 42 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
776 42 : if (cachedFileProp->eExists == EXIST_NO)
777 0 : return FALSE;
778 :
779 42 : CURL* hCurlHandle = poFS->GetCurlHandleFor(pszURL);
780 42 : VSICurlSetOptions(hCurlHandle, pszURL);
781 :
782 42 : VSICURLInitWriteFuncStruct(&sWriteFuncData, (VSILFILE*)this, pfnReadCbk, pReadCbkUserData);
783 42 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
784 42 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlHandleWriteFunc);
785 :
786 42 : VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, NULL, NULL, NULL);
787 42 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &sWriteFuncHeaderData);
788 42 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, VSICurlHandleWriteFunc);
789 42 : sWriteFuncHeaderData.bIsHTTP = strncmp(pszURL, "http", 4) == 0;
790 42 : sWriteFuncHeaderData.nStartOffset = startOffset;
791 42 : sWriteFuncHeaderData.nEndOffset = startOffset + nBlocks * DOWNLOAD_CHUNCK_SIZE - 1;
792 :
793 : char rangeStr[512];
794 42 : sprintf(rangeStr, CPL_FRMT_GUIB "-" CPL_FRMT_GUIB, startOffset, startOffset + nBlocks * DOWNLOAD_CHUNCK_SIZE - 1);
795 :
796 : if (ENABLE_DEBUG)
797 42 : CPLDebug("VSICURL", "Downloading %s (%s)...", rangeStr, pszURL);
798 :
799 42 : curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, rangeStr);
800 :
801 : char szCurlErrBuf[CURL_ERROR_SIZE+1];
802 42 : szCurlErrBuf[0] = '\0';
803 42 : curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
804 :
805 42 : curl_easy_perform(hCurlHandle);
806 :
807 42 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, NULL);
808 42 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, NULL);
809 42 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, NULL);
810 42 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, NULL);
811 :
812 42 : if (sWriteFuncData.bInterrupted)
813 : {
814 1 : bInterrupted = TRUE;
815 :
816 1 : CPLFree(sWriteFuncData.pBuffer);
817 1 : CPLFree(sWriteFuncHeaderData.pBuffer);
818 :
819 1 : return FALSE;
820 : }
821 :
822 41 : long response_code = 0;
823 41 : curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code);
824 :
825 41 : char *content_type = 0;
826 41 : curl_easy_getinfo(hCurlHandle, CURLINFO_CONTENT_TYPE, &content_type);
827 :
828 : if (ENABLE_DEBUG)
829 41 : CPLDebug("VSICURL", "Got reponse_code=%ld", response_code);
830 :
831 41 : if ((response_code != 200 && response_code != 206 &&
832 : response_code != 225 && response_code != 226 && response_code != 426) || sWriteFuncHeaderData.bError)
833 : {
834 0 : if (response_code >= 400 && szCurlErrBuf[0] != '\0')
835 : {
836 0 : if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0)
837 : CPLError(CE_Failure, CPLE_AppDefined, "%d: %s, %s",
838 : (int)response_code, szCurlErrBuf,
839 0 : "Range downloading not supported by this server !");
840 : else
841 0 : CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", (int)response_code, szCurlErrBuf);
842 : }
843 0 : if (!bHastComputedFileSize && startOffset == 0)
844 : {
845 0 : cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
846 0 : cachedFileProp->fileSize = fileSize = 0;
847 0 : cachedFileProp->eExists = eExists = EXIST_NO;
848 : }
849 0 : CPLFree(sWriteFuncData.pBuffer);
850 0 : CPLFree(sWriteFuncHeaderData.pBuffer);
851 0 : return FALSE;
852 : }
853 :
854 41 : if (!bHastComputedFileSize && sWriteFuncHeaderData.pBuffer)
855 : {
856 : /* Try to retrieve the filesize from the HTTP headers */
857 : /* if in the form : "Content-Range: bytes x-y/filesize" */
858 9 : char* pszContentRange = strstr(sWriteFuncHeaderData.pBuffer, "Content-Range: bytes ");
859 9 : if (pszContentRange)
860 : {
861 4 : char* pszEOL = strchr(pszContentRange, '\n');
862 4 : if (pszEOL)
863 : {
864 4 : *pszEOL = 0;
865 4 : pszEOL = strchr(pszContentRange, '\r');
866 4 : if (pszEOL)
867 4 : *pszEOL = 0;
868 4 : char* pszSlash = strchr(pszContentRange, '/');
869 4 : if (pszSlash)
870 : {
871 4 : pszSlash ++;
872 4 : fileSize = CPLScanUIntBig(pszSlash, strlen(pszSlash));
873 : }
874 : }
875 : }
876 5 : else if (strncmp(pszURL, "ftp", 3) == 0)
877 : {
878 : /* Parse 213 answer for FTP protocol */
879 0 : char* pszSize = strstr(sWriteFuncHeaderData.pBuffer, "213 ");
880 0 : if (pszSize)
881 : {
882 0 : pszSize += 4;
883 0 : char* pszEOL = strchr(pszSize, '\n');
884 0 : if (pszEOL)
885 : {
886 0 : *pszEOL = 0;
887 0 : pszEOL = strchr(pszSize, '\r');
888 0 : if (pszEOL)
889 0 : *pszEOL = 0;
890 :
891 0 : fileSize = CPLScanUIntBig(pszSize, strlen(pszSize));
892 : }
893 : }
894 : }
895 :
896 9 : if (fileSize != 0)
897 : {
898 4 : eExists = EXIST_YES;
899 :
900 : if (ENABLE_DEBUG)
901 : CPLDebug("VSICURL", "GetFileSize(%s)=" CPL_FRMT_GUIB " response_code=%d",
902 4 : pszURL, fileSize, (int)response_code);
903 :
904 4 : bHastComputedFileSize = cachedFileProp->bHastComputedFileSize = TRUE;
905 4 : cachedFileProp->fileSize = fileSize;
906 4 : cachedFileProp->eExists = eExists;
907 : }
908 : }
909 :
910 41 : lastDownloadedOffset = startOffset + nBlocks * DOWNLOAD_CHUNCK_SIZE;
911 :
912 41 : char* pBuffer = sWriteFuncData.pBuffer;
913 41 : int nSize = sWriteFuncData.nSize;
914 :
915 41 : if (nSize > nBlocks * DOWNLOAD_CHUNCK_SIZE)
916 : {
917 : if (ENABLE_DEBUG)
918 : CPLDebug("VSICURL", "Got more data than expected : %d instead of %d",
919 0 : nSize, nBlocks * DOWNLOAD_CHUNCK_SIZE);
920 : }
921 :
922 126 : while(nSize > 0)
923 : {
924 : //if (ENABLE_DEBUG)
925 : // CPLDebug("VSICURL", "Add region %d - %d", startOffset, MIN(DOWNLOAD_CHUNCK_SIZE, nSize));
926 44 : poFS->AddRegion(pszURL, startOffset, MIN(DOWNLOAD_CHUNCK_SIZE, nSize), pBuffer);
927 44 : startOffset += DOWNLOAD_CHUNCK_SIZE;
928 44 : pBuffer += DOWNLOAD_CHUNCK_SIZE;
929 44 : nSize -= DOWNLOAD_CHUNCK_SIZE;
930 : }
931 :
932 41 : CPLFree(sWriteFuncData.pBuffer);
933 41 : CPLFree(sWriteFuncHeaderData.pBuffer);
934 :
935 41 : return TRUE;
936 : }
937 :
938 : /************************************************************************/
939 : /* Read() */
940 : /************************************************************************/
941 :
942 6134 : size_t VSICurlHandle::Read( void *pBuffer, size_t nSize, size_t nMemb )
943 : {
944 6134 : size_t nBufferRequestSize = nSize * nMemb;
945 6134 : if (nBufferRequestSize == 0)
946 0 : return 0;
947 :
948 : //CPLDebug("VSICURL", "offset=%d, size=%d", (int)curOffset, (int)nBufferRequestSize);
949 :
950 6134 : vsi_l_offset iterOffset = curOffset;
951 18392 : while (nBufferRequestSize)
952 : {
953 6136 : const CachedRegion* psRegion = poFS->GetRegion(pszURL, iterOffset);
954 6136 : if (psRegion == NULL)
955 : {
956 : vsi_l_offset nOffsetToDownload =
957 42 : (iterOffset / DOWNLOAD_CHUNCK_SIZE) * DOWNLOAD_CHUNCK_SIZE;
958 :
959 42 : if (nOffsetToDownload == lastDownloadedOffset)
960 : {
961 : /* In case of consecutive reads (of small size), we use a */
962 : /* heuristic that we will read the file sequentially, so */
963 : /* we double the requested size to decrease the number of */
964 : /* client/server roundtrips. */
965 1 : if (nBlocksToDownload < 100)
966 1 : nBlocksToDownload *= 2;
967 : }
968 : else
969 : {
970 : /* Random reads. Cancel the above heuristics */
971 41 : nBlocksToDownload = 1;
972 : }
973 :
974 : /* Ensure that we will request at least the number of blocks */
975 : /* to satisfy the remaining buffer size to read */
976 : vsi_l_offset nEndOffsetToDownload =
977 42 : ((iterOffset + nBufferRequestSize) / DOWNLOAD_CHUNCK_SIZE) * DOWNLOAD_CHUNCK_SIZE;
978 : int nMinBlocksToDownload = 1 + (int)
979 42 : ((nEndOffsetToDownload - nOffsetToDownload) / DOWNLOAD_CHUNCK_SIZE);
980 42 : if (nBlocksToDownload < nMinBlocksToDownload)
981 2 : nBlocksToDownload = nMinBlocksToDownload;
982 :
983 : int i;
984 : /* Avoid reading already cached data */
985 45 : for(i=1;i<nBlocksToDownload;i++)
986 : {
987 3 : if (poFS->GetRegion(pszURL, nOffsetToDownload + i * DOWNLOAD_CHUNCK_SIZE) != NULL)
988 : {
989 0 : nBlocksToDownload = i;
990 0 : break;
991 : }
992 : }
993 :
994 42 : if (DownloadRegion(nOffsetToDownload, nBlocksToDownload) == FALSE)
995 : {
996 1 : if (!bInterrupted)
997 0 : bEOF = TRUE;
998 1 : return 0;
999 : }
1000 41 : psRegion = poFS->GetRegion(pszURL, iterOffset);
1001 : }
1002 6135 : if (psRegion == NULL || psRegion->pData == NULL)
1003 : {
1004 0 : bEOF = TRUE;
1005 0 : return 0;
1006 : }
1007 6135 : int nToCopy = (int) MIN(nBufferRequestSize, psRegion->nSize - (iterOffset - psRegion->nFileOffsetStart));
1008 : memcpy(pBuffer, psRegion->pData + iterOffset - psRegion->nFileOffsetStart,
1009 6135 : nToCopy);
1010 6135 : pBuffer = (char*) pBuffer + nToCopy;
1011 6135 : iterOffset += nToCopy;
1012 6135 : nBufferRequestSize -= nToCopy;
1013 6135 : if (psRegion->nSize != DOWNLOAD_CHUNCK_SIZE && nBufferRequestSize != 0)
1014 : {
1015 11 : break;
1016 : }
1017 : }
1018 :
1019 6133 : size_t ret = (size_t) ((iterOffset - curOffset) / nSize);
1020 6133 : if (ret != nMemb)
1021 11 : bEOF = TRUE;
1022 :
1023 6133 : curOffset = iterOffset;
1024 :
1025 6133 : return ret;
1026 : }
1027 :
1028 :
1029 : /************************************************************************/
1030 : /* ReadMultiRange() */
1031 : /************************************************************************/
1032 :
1033 1 : int VSICurlHandle::ReadMultiRange( int nRanges, void ** ppData,
1034 : const vsi_l_offset* panOffsets,
1035 : const size_t* panSizes )
1036 : {
1037 : WriteFuncStruct sWriteFuncData;
1038 : WriteFuncStruct sWriteFuncHeaderData;
1039 :
1040 1 : if (bInterrupted && bStopOnInterrruptUntilUninstall)
1041 0 : return FALSE;
1042 :
1043 1 : CachedFileProp* cachedFileProp = poFS->GetCachedFileProp(pszURL);
1044 1 : if (cachedFileProp->eExists == EXIST_NO)
1045 0 : return -1;
1046 :
1047 1 : CPLString osRanges, osFirstRange, osLastRange;
1048 : int i;
1049 1 : int nMergedRanges = 0;
1050 1 : vsi_l_offset nTotalReqSize = 0;
1051 129 : for(i=0;i<nRanges;i++)
1052 : {
1053 128 : CPLString osCurRange;
1054 128 : if (i != 0)
1055 127 : osRanges.append(",");
1056 128 : osCurRange = CPLSPrintf(CPL_FRMT_GUIB "-", panOffsets[i]);
1057 256 : while (i + 1 < nRanges && panOffsets[i] + panSizes[i] == panOffsets[i+1])
1058 : {
1059 0 : nTotalReqSize += panSizes[i];
1060 0 : i ++;
1061 : }
1062 128 : nTotalReqSize += panSizes[i];
1063 128 : osCurRange.append(CPLSPrintf(CPL_FRMT_GUIB, panOffsets[i] + panSizes[i]-1));
1064 128 : nMergedRanges ++;
1065 :
1066 128 : osRanges += osCurRange;
1067 :
1068 128 : if (nMergedRanges == 1)
1069 1 : osFirstRange = osCurRange;
1070 128 : osLastRange = osCurRange;
1071 : }
1072 :
1073 1 : const char* pszMaxRanges = CPLGetConfigOption("CPL_VSIL_CURL_MAX_RANGES", "250");
1074 1 : int nMaxRanges = atoi(pszMaxRanges);
1075 1 : if (nMaxRanges <= 0)
1076 0 : nMaxRanges = 250;
1077 1 : if (nMergedRanges > nMaxRanges)
1078 : {
1079 0 : int nHalf = nRanges / 2;
1080 0 : int nRet = ReadMultiRange(nHalf, ppData, panOffsets, panSizes);
1081 0 : if (nRet != 0)
1082 0 : return nRet;
1083 0 : return ReadMultiRange(nRanges - nHalf, ppData + nHalf, panOffsets + nHalf, panSizes + nHalf);
1084 : }
1085 :
1086 1 : CURL* hCurlHandle = poFS->GetCurlHandleFor(pszURL);
1087 1 : VSICurlSetOptions(hCurlHandle, pszURL);
1088 :
1089 1 : VSICURLInitWriteFuncStruct(&sWriteFuncData, (VSILFILE*)this, pfnReadCbk, pReadCbkUserData);
1090 1 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
1091 1 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlHandleWriteFunc);
1092 :
1093 1 : VSICURLInitWriteFuncStruct(&sWriteFuncHeaderData, NULL, NULL, NULL);
1094 1 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, &sWriteFuncHeaderData);
1095 1 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, VSICurlHandleWriteFunc);
1096 1 : sWriteFuncHeaderData.bIsHTTP = strncmp(pszURL, "http", 4) == 0;
1097 1 : sWriteFuncHeaderData.bMultiRange = nMergedRanges > 1;
1098 1 : if (nMergedRanges == 1)
1099 : {
1100 0 : sWriteFuncHeaderData.nStartOffset = panOffsets[0];
1101 0 : sWriteFuncHeaderData.nEndOffset = panOffsets[0] + nTotalReqSize-1;
1102 : }
1103 :
1104 : if (ENABLE_DEBUG)
1105 : {
1106 1 : if (nMergedRanges == 1)
1107 0 : CPLDebug("VSICURL", "Downloading %s (%s)...", osRanges.c_str(), pszURL);
1108 : else
1109 : CPLDebug("VSICURL", "Downloading %s, ..., %s (" CPL_FRMT_GUIB " bytes, %s)...",
1110 1 : osFirstRange.c_str(), osLastRange.c_str(), (GUIntBig)nTotalReqSize, pszURL);
1111 : }
1112 :
1113 1 : curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, osRanges.c_str());
1114 :
1115 : char szCurlErrBuf[CURL_ERROR_SIZE+1];
1116 1 : szCurlErrBuf[0] = '\0';
1117 1 : curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
1118 :
1119 1 : curl_easy_perform(hCurlHandle);
1120 :
1121 1 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, NULL);
1122 1 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, NULL);
1123 1 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERDATA, NULL);
1124 1 : curl_easy_setopt(hCurlHandle, CURLOPT_HEADERFUNCTION, NULL);
1125 :
1126 1 : if (sWriteFuncData.bInterrupted)
1127 : {
1128 0 : bInterrupted = TRUE;
1129 :
1130 0 : CPLFree(sWriteFuncData.pBuffer);
1131 0 : CPLFree(sWriteFuncHeaderData.pBuffer);
1132 :
1133 0 : return -1;
1134 : }
1135 :
1136 1 : long response_code = 0;
1137 1 : curl_easy_getinfo(hCurlHandle, CURLINFO_HTTP_CODE, &response_code);
1138 :
1139 1 : char *content_type = 0;
1140 1 : curl_easy_getinfo(hCurlHandle, CURLINFO_CONTENT_TYPE, &content_type);
1141 :
1142 1 : if ((response_code != 200 && response_code != 206 &&
1143 : response_code != 225 && response_code != 226 && response_code != 426) || sWriteFuncHeaderData.bError)
1144 : {
1145 0 : if (response_code >= 400 && szCurlErrBuf[0] != '\0')
1146 : {
1147 0 : if (strcmp(szCurlErrBuf, "Couldn't use REST") == 0)
1148 : CPLError(CE_Failure, CPLE_AppDefined, "%d: %s, %s",
1149 : (int)response_code, szCurlErrBuf,
1150 0 : "Range downloading not supported by this server !");
1151 : else
1152 0 : CPLError(CE_Failure, CPLE_AppDefined, "%d: %s", (int)response_code, szCurlErrBuf);
1153 : }
1154 : /*
1155 : if (!bHastComputedFileSize && startOffset == 0)
1156 : {
1157 : cachedFileProp->bHastComputedFileSize = bHastComputedFileSize = TRUE;
1158 : cachedFileProp->fileSize = fileSize = 0;
1159 : cachedFileProp->eExists = eExists = EXIST_NO;
1160 : }
1161 : */
1162 0 : CPLFree(sWriteFuncData.pBuffer);
1163 0 : CPLFree(sWriteFuncHeaderData.pBuffer);
1164 0 : return -1;
1165 : }
1166 :
1167 1 : char* pBuffer = sWriteFuncData.pBuffer;
1168 1 : int nSize = sWriteFuncData.nSize;
1169 :
1170 1 : int nRet = -1;
1171 : char* pszBoundary;
1172 1 : CPLString osBoundary;
1173 : char *pszNext;
1174 1 : int iRange = 0;
1175 1 : int iPart = 0;
1176 : char* pszEOL;
1177 :
1178 : /* -------------------------------------------------------------------- */
1179 : /* No multipart if a single range has been requested */
1180 : /* -------------------------------------------------------------------- */
1181 :
1182 1 : if (nMergedRanges == 1)
1183 : {
1184 0 : int nAccSize = 0;
1185 0 : if ((vsi_l_offset)nSize < nTotalReqSize)
1186 0 : goto end;
1187 :
1188 0 : for(i=0;i<nRanges;i++)
1189 : {
1190 0 : memcpy(ppData[i], pBuffer + nAccSize, panSizes[i]);
1191 0 : nAccSize += panSizes[i];
1192 : }
1193 :
1194 0 : nRet = 0;
1195 0 : goto end;
1196 : }
1197 :
1198 : /* -------------------------------------------------------------------- */
1199 : /* Extract boundary name */
1200 : /* -------------------------------------------------------------------- */
1201 :
1202 : pszBoundary = strstr(sWriteFuncHeaderData.pBuffer,
1203 1 : "Content-Type: multipart/byteranges; boundary=");
1204 1 : if( pszBoundary == NULL )
1205 : {
1206 : CPLError( CE_Failure, CPLE_AppDefined, "Could not find '%s'",
1207 0 : "Content-Type: multipart/byteranges; boundary=" );
1208 0 : goto end;
1209 : }
1210 :
1211 1 : pszBoundary += strlen( "Content-Type: multipart/byteranges; boundary=" );
1212 :
1213 1 : pszEOL = strchr(pszBoundary, '\r');
1214 1 : if (pszEOL)
1215 1 : *pszEOL = 0;
1216 1 : pszEOL = strchr(pszBoundary, '\n');
1217 1 : if (pszEOL)
1218 0 : *pszEOL = 0;
1219 :
1220 : /* Remove optional double-quote character around boundary name */
1221 1 : if (pszBoundary[0] == '"')
1222 : {
1223 0 : pszBoundary ++;
1224 0 : char* pszLastDoubleQuote = strrchr(pszBoundary, '"');
1225 0 : if (pszLastDoubleQuote)
1226 0 : *pszLastDoubleQuote = 0;
1227 : }
1228 :
1229 1 : osBoundary = "--";
1230 1 : osBoundary += pszBoundary;
1231 :
1232 : /* -------------------------------------------------------------------- */
1233 : /* Find the start of the first chunk. */
1234 : /* -------------------------------------------------------------------- */
1235 1 : pszNext = strstr(pBuffer,osBoundary.c_str());
1236 1 : if( pszNext == NULL )
1237 : {
1238 0 : CPLError( CE_Failure, CPLE_AppDefined, "No parts found." );
1239 0 : goto end;
1240 : }
1241 :
1242 1 : pszNext += strlen(osBoundary);
1243 2 : while( *pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0' )
1244 0 : pszNext++;
1245 1 : if( *pszNext == '\r' )
1246 1 : pszNext++;
1247 1 : if( *pszNext == '\n' )
1248 1 : pszNext++;
1249 :
1250 : /* -------------------------------------------------------------------- */
1251 : /* Loop over parts... */
1252 : /* -------------------------------------------------------------------- */
1253 129 : while( iPart < nRanges )
1254 : {
1255 : /* -------------------------------------------------------------------- */
1256 : /* Collect headers. */
1257 : /* -------------------------------------------------------------------- */
1258 128 : int bExpectedRange = FALSE;
1259 :
1260 512 : while( *pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0' )
1261 : {
1262 256 : char *pszEOL = strstr(pszNext,"\n");
1263 :
1264 256 : if( pszEOL == NULL )
1265 : {
1266 : CPLError(CE_Failure, CPLE_AppDefined,
1267 0 : "Error while parsing multipart content (at line %d)", __LINE__);
1268 0 : goto end;
1269 : }
1270 :
1271 256 : *pszEOL = '\0';
1272 256 : int bRestoreAntislashR = FALSE;
1273 256 : if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r')
1274 : {
1275 256 : bRestoreAntislashR = TRUE;
1276 256 : pszEOL[-1] = '\0';
1277 : }
1278 :
1279 256 : if (EQUALN(pszNext, "Content-Range: bytes ", strlen("Content-Range: bytes ")))
1280 : {
1281 128 : bExpectedRange = TRUE; /* FIXME */
1282 : }
1283 :
1284 256 : if (bRestoreAntislashR)
1285 256 : pszEOL[-1] = '\r';
1286 256 : *pszEOL = '\n';
1287 :
1288 256 : pszNext = pszEOL + 1;
1289 : }
1290 :
1291 128 : if (!bExpectedRange)
1292 : {
1293 : CPLError(CE_Failure, CPLE_AppDefined,
1294 0 : "Error while parsing multipart content (at line %d)", __LINE__);
1295 0 : goto end;
1296 : }
1297 :
1298 128 : if( *pszNext == '\r' )
1299 128 : pszNext++;
1300 128 : if( *pszNext == '\n' )
1301 128 : pszNext++;
1302 :
1303 : /* -------------------------------------------------------------------- */
1304 : /* Work out the data block size. */
1305 : /* -------------------------------------------------------------------- */
1306 128 : size_t nBytesAvail = nSize - (pszNext - pBuffer);
1307 :
1308 0 : while(TRUE)
1309 : {
1310 128 : if (nBytesAvail < panSizes[iRange])
1311 : {
1312 : CPLError(CE_Failure, CPLE_AppDefined,
1313 0 : "Error while parsing multipart content (at line %d)", __LINE__);
1314 0 : goto end;
1315 : }
1316 :
1317 128 : memcpy(ppData[iRange], pszNext, panSizes[iRange]);
1318 128 : pszNext += panSizes[iRange];
1319 128 : nBytesAvail -= panSizes[iRange];
1320 509 : if( iRange + 1 < nRanges &&
1321 381 : panOffsets[iRange] + panSizes[iRange] == panOffsets[iRange + 1] )
1322 : {
1323 0 : iRange++;
1324 : }
1325 : else
1326 : break;
1327 : }
1328 :
1329 128 : iPart ++;
1330 128 : iRange ++;
1331 :
1332 512 : while( nBytesAvail > 0
1333 : && (*pszNext != '-'
1334 : || strncmp(pszNext,osBoundary,strlen(osBoundary)) != 0) )
1335 : {
1336 256 : pszNext++;
1337 256 : nBytesAvail--;
1338 : }
1339 :
1340 128 : if( nBytesAvail == 0 )
1341 : {
1342 : CPLError(CE_Failure, CPLE_AppDefined,
1343 0 : "Error while parsing multipart content (at line %d)", __LINE__);
1344 0 : goto end;
1345 : }
1346 :
1347 128 : pszNext += strlen(osBoundary);
1348 128 : if( strncmp(pszNext,"--",2) == 0 )
1349 : {
1350 : /* End of multipart */
1351 1 : break;
1352 : }
1353 :
1354 127 : if( *pszNext == '\r' )
1355 127 : pszNext++;
1356 127 : if( *pszNext == '\n' )
1357 127 : pszNext++;
1358 : else
1359 : {
1360 : CPLError(CE_Failure, CPLE_AppDefined,
1361 0 : "Error while parsing multipart content (at line %d)", __LINE__);
1362 0 : goto end;
1363 : }
1364 : }
1365 :
1366 1 : if (iPart == nMergedRanges)
1367 1 : nRet = 0;
1368 : else
1369 : CPLError(CE_Failure, CPLE_AppDefined,
1370 0 : "Got only %d parts, where %d were expected", iPart, nMergedRanges);
1371 :
1372 : end:
1373 1 : CPLFree(sWriteFuncData.pBuffer);
1374 1 : CPLFree(sWriteFuncHeaderData.pBuffer);
1375 :
1376 1 : return nRet;
1377 : }
1378 :
1379 : /************************************************************************/
1380 : /* Write() */
1381 : /************************************************************************/
1382 :
1383 0 : size_t VSICurlHandle::Write( const void *pBuffer, size_t nSize, size_t nMemb )
1384 : {
1385 0 : return 0;
1386 : }
1387 :
1388 : /************************************************************************/
1389 : /* Eof() */
1390 : /************************************************************************/
1391 :
1392 :
1393 7 : int VSICurlHandle::Eof()
1394 : {
1395 7 : return bEOF;
1396 : }
1397 :
1398 : /************************************************************************/
1399 : /* Flush() */
1400 : /************************************************************************/
1401 :
1402 0 : int VSICurlHandle::Flush()
1403 : {
1404 0 : return 0;
1405 : }
1406 :
1407 : /************************************************************************/
1408 : /* Close() */
1409 : /************************************************************************/
1410 :
1411 74 : int VSICurlHandle::Close()
1412 : {
1413 74 : return 0;
1414 : }
1415 :
1416 :
1417 :
1418 :
1419 : /************************************************************************/
1420 : /* VSICurlFilesystemHandler() */
1421 : /************************************************************************/
1422 :
1423 757 : VSICurlFilesystemHandler::VSICurlFilesystemHandler()
1424 : {
1425 757 : hMutex = NULL;
1426 757 : papsRegions = NULL;
1427 757 : nRegions = 0;
1428 757 : bUseCacheDisk = CSLTestBoolean(CPLGetConfigOption("CPL_VSIL_CURL_USE_CACHE", "NO"));
1429 757 : }
1430 :
1431 : /************************************************************************/
1432 : /* ~VSICurlFilesystemHandler() */
1433 : /************************************************************************/
1434 :
1435 704 : VSICurlFilesystemHandler::~VSICurlFilesystemHandler()
1436 : {
1437 : int i;
1438 748 : for(i=0;i<nRegions;i++)
1439 : {
1440 44 : CPLFree(papsRegions[i]->pData);
1441 44 : CPLFree(papsRegions[i]);
1442 : }
1443 704 : CPLFree(papsRegions);
1444 :
1445 704 : std::map<CPLString, CachedFileProp*>::const_iterator iterCacheFileSize;
1446 :
1447 1302 : for( iterCacheFileSize = cacheFileSize.begin(); iterCacheFileSize != cacheFileSize.end(); iterCacheFileSize++ )
1448 : {
1449 598 : CPLFree(iterCacheFileSize->second);
1450 : }
1451 :
1452 704 : std::map<CPLString, CachedDirList*>::const_iterator iterCacheDirList;
1453 :
1454 722 : for( iterCacheDirList = cacheDirList.begin(); iterCacheDirList != cacheDirList.end(); iterCacheDirList++ )
1455 : {
1456 18 : CSLDestroy(iterCacheDirList->second->papszFileList);
1457 18 : CPLFree(iterCacheDirList->second);
1458 : }
1459 :
1460 704 : std::map<GIntBig, CachedConnection*>::const_iterator iterConnections;
1461 706 : for( iterConnections = mapConnections.begin(); iterConnections != mapConnections.end(); iterConnections++ )
1462 : {
1463 2 : curl_easy_cleanup(iterConnections->second->hCurlHandle);
1464 2 : delete iterConnections->second;
1465 : }
1466 :
1467 704 : if( hMutex != NULL )
1468 2 : CPLDestroyMutex( hMutex );
1469 704 : hMutex = NULL;
1470 704 : }
1471 :
1472 : /************************************************************************/
1473 : /* GetCurlHandleFor() */
1474 : /************************************************************************/
1475 :
1476 82 : CURL* VSICurlFilesystemHandler::GetCurlHandleFor(CPLString osURL)
1477 : {
1478 82 : CPLMutexHolder oHolder( &hMutex );
1479 :
1480 82 : std::map<GIntBig, CachedConnection*>::const_iterator iterConnections;
1481 :
1482 82 : iterConnections = mapConnections.find(CPLGetPID());
1483 82 : if (iterConnections == mapConnections.end())
1484 : {
1485 2 : CURL* hCurlHandle = curl_easy_init();
1486 2 : CachedConnection* psCachedConnection = new CachedConnection;
1487 2 : psCachedConnection->osURL = osURL;
1488 2 : psCachedConnection->hCurlHandle = hCurlHandle;
1489 2 : mapConnections[CPLGetPID()] = psCachedConnection;
1490 2 : return hCurlHandle;
1491 : }
1492 : else
1493 : {
1494 80 : CachedConnection* psCachedConnection = iterConnections->second;
1495 80 : if (osURL == psCachedConnection->osURL)
1496 33 : return psCachedConnection->hCurlHandle;
1497 :
1498 47 : const char* pszURL = osURL.c_str();
1499 47 : const char* pszEndOfServ = strchr(pszURL, '.');
1500 47 : if (pszEndOfServ != NULL)
1501 47 : pszEndOfServ = strchr(pszEndOfServ, '/');
1502 47 : if (pszEndOfServ == NULL)
1503 0 : pszURL = pszURL + strlen(pszURL);
1504 : int bReinitConnection = strncmp(psCachedConnection->osURL,
1505 47 : pszURL, pszEndOfServ-pszURL) != 0;
1506 :
1507 47 : if (bReinitConnection)
1508 : {
1509 7 : if (psCachedConnection->hCurlHandle)
1510 7 : curl_easy_cleanup(psCachedConnection->hCurlHandle);
1511 7 : psCachedConnection->hCurlHandle = curl_easy_init();
1512 : }
1513 47 : psCachedConnection->osURL = osURL;
1514 :
1515 47 : return psCachedConnection->hCurlHandle;
1516 0 : }
1517 : }
1518 :
1519 :
1520 : /************************************************************************/
1521 : /* GetRegionFromCacheDisk() */
1522 : /************************************************************************/
1523 :
1524 : const CachedRegion*
1525 0 : VSICurlFilesystemHandler::GetRegionFromCacheDisk(const char* pszURL,
1526 : vsi_l_offset nFileOffsetStart)
1527 : {
1528 0 : nFileOffsetStart = (nFileOffsetStart / DOWNLOAD_CHUNCK_SIZE) * DOWNLOAD_CHUNCK_SIZE;
1529 0 : VSILFILE* fp = VSIFOpenL(VSICurlGetCacheFileName(), "rb");
1530 0 : if (fp)
1531 : {
1532 0 : unsigned long pszURLHash = CPLHashSetHashStr(pszURL);
1533 : unsigned long pszURLHashCached;
1534 : vsi_l_offset nFileOffsetStartCached;
1535 : size_t nSizeCached;
1536 0 : while(TRUE)
1537 : {
1538 0 : if (VSIFReadL(&pszURLHashCached, 1, sizeof(unsigned long), fp) == 0)
1539 : break;
1540 0 : VSIFReadL(&nFileOffsetStartCached, 1, sizeof(vsi_l_offset), fp);
1541 0 : VSIFReadL(&nSizeCached, 1, sizeof(size_t), fp);
1542 0 : if (pszURLHash == pszURLHashCached &&
1543 : nFileOffsetStart == nFileOffsetStartCached)
1544 : {
1545 : if (ENABLE_DEBUG)
1546 0 : CPLDebug("VSICURL", "Got data at offset " CPL_FRMT_GUIB " from disk" , nFileOffsetStart);
1547 0 : if (nSizeCached)
1548 : {
1549 0 : char* pBuffer = (char*) CPLMalloc(nSizeCached);
1550 0 : VSIFReadL(pBuffer, 1, nSizeCached, fp);
1551 0 : AddRegion(pszURL, nFileOffsetStart, nSizeCached, pBuffer);
1552 0 : CPLFree(pBuffer);
1553 : }
1554 : else
1555 : {
1556 0 : AddRegion(pszURL, nFileOffsetStart, 0, NULL);
1557 : }
1558 0 : VSIFCloseL(fp);
1559 0 : return GetRegion(pszURL, nFileOffsetStart);
1560 : }
1561 : else
1562 : {
1563 0 : VSIFSeekL(fp, nSizeCached, SEEK_CUR);
1564 : }
1565 : }
1566 0 : VSIFCloseL(fp);
1567 : }
1568 0 : return NULL;
1569 : }
1570 :
1571 :
1572 : /************************************************************************/
1573 : /* AddRegionToCacheDisk() */
1574 : /************************************************************************/
1575 :
1576 0 : void VSICurlFilesystemHandler::AddRegionToCacheDisk(CachedRegion* psRegion)
1577 : {
1578 0 : VSILFILE* fp = VSIFOpenL(VSICurlGetCacheFileName(), "r+b");
1579 0 : if (fp)
1580 : {
1581 : unsigned long pszURLHashCached;
1582 : vsi_l_offset nFileOffsetStartCached;
1583 : size_t nSizeCached;
1584 0 : while(TRUE)
1585 : {
1586 0 : if (VSIFReadL(&pszURLHashCached, 1, sizeof(unsigned long), fp) == 0)
1587 : break;
1588 0 : VSIFReadL(&nFileOffsetStartCached, 1, sizeof(vsi_l_offset), fp);
1589 0 : VSIFReadL(&nSizeCached, 1, sizeof(size_t), fp);
1590 0 : if (psRegion->pszURLHash == pszURLHashCached &&
1591 : psRegion->nFileOffsetStart == nFileOffsetStartCached)
1592 : {
1593 0 : CPLAssert(psRegion->nSize == nSizeCached);
1594 0 : VSIFCloseL(fp);
1595 0 : return;
1596 : }
1597 : else
1598 : {
1599 0 : VSIFSeekL(fp, nSizeCached, SEEK_CUR);
1600 : }
1601 : }
1602 : }
1603 : else
1604 : {
1605 0 : fp = VSIFOpenL(VSICurlGetCacheFileName(), "wb");
1606 : }
1607 0 : if (fp)
1608 : {
1609 : if (ENABLE_DEBUG)
1610 0 : CPLDebug("VSICURL", "Write data at offset " CPL_FRMT_GUIB " to disk" , psRegion->nFileOffsetStart);
1611 0 : VSIFWriteL(&psRegion->pszURLHash, 1, sizeof(unsigned long), fp);
1612 0 : VSIFWriteL(&psRegion->nFileOffsetStart, 1, sizeof(vsi_l_offset), fp);
1613 0 : VSIFWriteL(&psRegion->nSize, 1, sizeof(size_t), fp);
1614 0 : if (psRegion->nSize)
1615 0 : VSIFWriteL(psRegion->pData, 1, psRegion->nSize, fp);
1616 :
1617 0 : VSIFCloseL(fp);
1618 : }
1619 0 : return;
1620 : }
1621 :
1622 :
1623 : /************************************************************************/
1624 : /* GetRegion() */
1625 : /************************************************************************/
1626 :
1627 6180 : const CachedRegion* VSICurlFilesystemHandler::GetRegion(const char* pszURL,
1628 : vsi_l_offset nFileOffsetStart)
1629 : {
1630 6180 : CPLMutexHolder oHolder( &hMutex );
1631 :
1632 6180 : unsigned long pszURLHash = CPLHashSetHashStr(pszURL);
1633 :
1634 6180 : nFileOffsetStart = (nFileOffsetStart / DOWNLOAD_CHUNCK_SIZE) * DOWNLOAD_CHUNCK_SIZE;
1635 : int i;
1636 7147 : for(i=0;i<nRegions;i++)
1637 : {
1638 7102 : CachedRegion* psRegion = papsRegions[i];
1639 7102 : if (psRegion->pszURLHash == pszURLHash &&
1640 : nFileOffsetStart == psRegion->nFileOffsetStart)
1641 : {
1642 6135 : memmove(papsRegions + 1, papsRegions, i * sizeof(CachedRegion*));
1643 6135 : papsRegions[0] = psRegion;
1644 6135 : return psRegion;
1645 : }
1646 : }
1647 45 : if (bUseCacheDisk)
1648 0 : return GetRegionFromCacheDisk(pszURL, nFileOffsetStart);
1649 45 : return NULL;
1650 : }
1651 :
1652 : /************************************************************************/
1653 : /* AddRegion() */
1654 : /************************************************************************/
1655 :
1656 44 : void VSICurlFilesystemHandler::AddRegion(const char* pszURL,
1657 : vsi_l_offset nFileOffsetStart,
1658 : size_t nSize,
1659 : const char *pData)
1660 : {
1661 44 : CPLMutexHolder oHolder( &hMutex );
1662 :
1663 44 : unsigned long pszURLHash = CPLHashSetHashStr(pszURL);
1664 :
1665 : CachedRegion* psRegion;
1666 44 : if (nRegions == N_MAX_REGIONS)
1667 : {
1668 0 : psRegion = papsRegions[N_MAX_REGIONS-1];
1669 0 : memmove(papsRegions + 1, papsRegions, (N_MAX_REGIONS-1) * sizeof(CachedRegion*));
1670 0 : papsRegions[0] = psRegion;
1671 0 : CPLFree(psRegion->pData);
1672 : }
1673 : else
1674 : {
1675 44 : papsRegions = (CachedRegion**) CPLRealloc(papsRegions, (nRegions + 1) * sizeof(CachedRegion*));
1676 44 : if (nRegions)
1677 42 : memmove(papsRegions + 1, papsRegions, nRegions * sizeof(CachedRegion*));
1678 44 : nRegions ++;
1679 44 : papsRegions[0] = psRegion = (CachedRegion*) CPLMalloc(sizeof(CachedRegion));
1680 : }
1681 :
1682 44 : psRegion->pszURLHash = pszURLHash;
1683 44 : psRegion->nFileOffsetStart = nFileOffsetStart;
1684 44 : psRegion->nSize = nSize;
1685 44 : psRegion->pData = (nSize) ? (char*) CPLMalloc(nSize) : NULL;
1686 44 : if (nSize)
1687 44 : memcpy(psRegion->pData, pData, nSize);
1688 :
1689 44 : if (bUseCacheDisk)
1690 0 : AddRegionToCacheDisk(psRegion);
1691 44 : }
1692 :
1693 : /************************************************************************/
1694 : /* GetCachedFileProp() */
1695 : /************************************************************************/
1696 :
1697 2362 : CachedFileProp* VSICurlFilesystemHandler::GetCachedFileProp(const char* pszURL)
1698 : {
1699 2362 : CPLMutexHolder oHolder( &hMutex );
1700 :
1701 2362 : CachedFileProp* cachedFileProp = cacheFileSize[pszURL];
1702 2362 : if (cachedFileProp == NULL)
1703 : {
1704 598 : cachedFileProp = (CachedFileProp*) CPLMalloc(sizeof(CachedFileProp));
1705 598 : cachedFileProp->eExists = EXIST_UNKNOWN;
1706 598 : cachedFileProp->bHastComputedFileSize = FALSE;
1707 598 : cachedFileProp->fileSize = 0;
1708 598 : cachedFileProp->bIsDirectory = FALSE;
1709 598 : cacheFileSize[pszURL] = cachedFileProp;
1710 : }
1711 :
1712 2362 : return cachedFileProp;
1713 : }
1714 :
1715 : /************************************************************************/
1716 : /* Open() */
1717 : /************************************************************************/
1718 :
1719 88 : VSIVirtualHandle* VSICurlFilesystemHandler::Open( const char *pszFilename,
1720 : const char *pszAccess)
1721 : {
1722 88 : if (strchr(pszAccess, 'w') != NULL ||
1723 : strchr(pszAccess, '+') != NULL)
1724 : {
1725 : CPLError(CE_Failure, CPLE_AppDefined,
1726 0 : "Only read-only mode is supported for /vsicurl");
1727 0 : return NULL;
1728 : }
1729 :
1730 : const char* pszOptionVal =
1731 88 : CPLGetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", "NO" );
1732 : int bSkipReadDir = EQUAL(pszOptionVal, "EMPTY_DIR") ||
1733 88 : CSLTestBoolean(pszOptionVal);
1734 :
1735 88 : CPLString osFilename(pszFilename);
1736 88 : int bGotFileList = TRUE;
1737 88 : if (strchr(CPLGetFilename(osFilename), '.') != NULL &&
1738 : strncmp(CPLGetExtension(osFilename), "zip", 3) != 0 && !bSkipReadDir)
1739 : {
1740 58 : char** papszFileList = ReadDir(CPLGetDirname(osFilename), &bGotFileList);
1741 58 : int bFound = (VSICurlIsFileInList(papszFileList, CPLGetFilename(osFilename)) != -1);
1742 58 : CSLDestroy(papszFileList);
1743 58 : if (bGotFileList && !bFound)
1744 : {
1745 12 : return NULL;
1746 : }
1747 : }
1748 :
1749 76 : VSICurlHandle* poHandle = new VSICurlHandle( this, osFilename + strlen("/vsicurl/"));
1750 76 : if (!bGotFileList)
1751 : {
1752 : /* If we didn't get a filelist, check that the file really exists */
1753 17 : if (!poHandle->Exists())
1754 : {
1755 2 : delete poHandle;
1756 2 : poHandle = NULL;
1757 : }
1758 : }
1759 :
1760 76 : if( CSLTestBoolean( CPLGetConfigOption( "VSI_CACHE", "FALSE" ) ) )
1761 0 : return VSICreateCachedFile( poHandle );
1762 : else
1763 76 : return poHandle;
1764 : }
1765 :
1766 : /************************************************************************/
1767 : /* VSICurlParserFindEOL() */
1768 : /* */
1769 : /* Small helper function for VSICurlPaseHTMLFileList() to find */
1770 : /* the end of a line in the directory listing. Either a <br> */
1771 : /* or newline. */
1772 : /************************************************************************/
1773 :
1774 722 : static char *VSICurlParserFindEOL( char *pszData )
1775 :
1776 : {
1777 40340 : while( *pszData != '\0' && *pszData != '\n' && !EQUALN(pszData,"<br>",4) )
1778 38896 : pszData++;
1779 :
1780 722 : if( *pszData == '\0' )
1781 16 : return NULL;
1782 : else
1783 706 : return pszData;
1784 : }
1785 :
1786 :
1787 : /************************************************************************/
1788 : /* VSICurlParseHTMLDateTimeFileSize() */
1789 : /************************************************************************/
1790 :
1791 : static const char* const apszMonths[] = { "January", "February", "March",
1792 : "April", "May", "June", "July",
1793 : "August", "September", "October",
1794 : "November", "December" };
1795 :
1796 589 : static int VSICurlParseHTMLDateTimeFileSize(const char* pszStr,
1797 : struct tm& brokendowntime,
1798 : GUIntBig& nFileSize,
1799 : GIntBig& mTime)
1800 : {
1801 : int iMonth;
1802 7573 : for(iMonth=0;iMonth<12;iMonth++)
1803 : {
1804 : char szMonth[32];
1805 6994 : szMonth[0] = '-';
1806 6994 : memcpy(szMonth + 1, apszMonths[iMonth], 3);
1807 6994 : szMonth[4] = '-';
1808 6994 : szMonth[5] = '\0';
1809 6994 : const char* pszMonthFound = strstr(pszStr, szMonth);
1810 6994 : if (pszMonthFound)
1811 : {
1812 : /* Format of Apache, like in http://download.osgeo.org/gdal/data/gtiff/ */
1813 : /* "17-May-2010 12:26" */
1814 30 : if (pszMonthFound - pszStr > 2 && strlen(pszMonthFound) > 15 &&
1815 20 : pszMonthFound[-2 + 11] == ' ' && pszMonthFound[-2 + 14] == ':')
1816 : {
1817 10 : pszMonthFound -= 2;
1818 10 : int nDay = atoi(pszMonthFound);
1819 10 : int nYear = atoi(pszMonthFound + 7);
1820 10 : int nHour = atoi(pszMonthFound + 12);
1821 10 : int nMin = atoi(pszMonthFound + 15);
1822 10 : if (nDay >= 1 && nDay <= 31 && nYear >= 1900 &&
1823 : nHour >= 0 && nHour <= 24 && nMin >= 0 && nMin < 60)
1824 : {
1825 10 : brokendowntime.tm_year = nYear - 1900;
1826 10 : brokendowntime.tm_mon = iMonth;
1827 10 : brokendowntime.tm_mday = nDay;
1828 10 : brokendowntime.tm_hour = nHour;
1829 10 : brokendowntime.tm_min = nMin;
1830 10 : mTime = CPLYMDHMSToUnixTime(&brokendowntime);
1831 :
1832 10 : return TRUE;
1833 : }
1834 : }
1835 0 : return FALSE;
1836 : }
1837 :
1838 : /* Microsoft IIS */
1839 6984 : szMonth[0] = ' ';
1840 6984 : strcpy(szMonth + 1, apszMonths[iMonth]);
1841 6984 : strcat(szMonth, " ");
1842 6984 : pszMonthFound = strstr(pszStr, szMonth);
1843 6984 : if (pszMonthFound)
1844 : {
1845 0 : int nLenMonth = strlen(apszMonths[iMonth]);
1846 0 : if (pszMonthFound - pszStr > 2 &&
1847 0 : pszMonthFound[-1] != ',' &&
1848 0 : pszMonthFound[-2] != ' ' &&
1849 : (int)strlen(pszMonthFound-2) > 2 + 1 + nLenMonth + 1 + 4 + 1 + 5 + 1 + 4)
1850 : {
1851 : /* Format of http://ortho.linz.govt.nz/tifs/1994_95/ */
1852 : /* " Friday, 21 April 2006 12:05 p.m. 48062343 m35a_fy_94_95.tif" */
1853 0 : pszMonthFound -= 2;
1854 0 : int nDay = atoi(pszMonthFound);
1855 0 : int nCurOffset = 2 + 1 + nLenMonth + 1;
1856 0 : int nYear = atoi(pszMonthFound + nCurOffset);
1857 0 : nCurOffset += 4 + 1;
1858 0 : int nHour = atoi(pszMonthFound + nCurOffset);
1859 0 : if (nHour < 10)
1860 0 : nCurOffset += 1 + 1;
1861 : else
1862 0 : nCurOffset += 2 + 1;
1863 0 : int nMin = atoi(pszMonthFound + nCurOffset);
1864 0 : nCurOffset += 2 + 1;
1865 0 : if (strncmp(pszMonthFound + nCurOffset, "p.m.", 4) == 0)
1866 0 : nHour += 12;
1867 0 : else if (strncmp(pszMonthFound + nCurOffset, "a.m.", 4) != 0)
1868 0 : nHour = -1;
1869 0 : nCurOffset += 4;
1870 :
1871 0 : const char* pszFilesize = pszMonthFound + nCurOffset;
1872 0 : while(*pszFilesize == ' ')
1873 0 : pszFilesize ++;
1874 0 : if (*pszFilesize >= '1' && *pszFilesize <= '9')
1875 0 : nFileSize = CPLScanUIntBig(pszFilesize, strlen(pszFilesize));
1876 :
1877 0 : if (nDay >= 1 && nDay <= 31 && nYear >= 1900 &&
1878 : nHour >= 0 && nHour <= 24 && nMin >= 0 && nMin < 60)
1879 : {
1880 0 : brokendowntime.tm_year = nYear - 1900;
1881 0 : brokendowntime.tm_mon = iMonth;
1882 0 : brokendowntime.tm_mday = nDay;
1883 0 : brokendowntime.tm_hour = nHour;
1884 0 : brokendowntime.tm_min = nMin;
1885 0 : mTime = CPLYMDHMSToUnixTime(&brokendowntime);
1886 :
1887 0 : return TRUE;
1888 : }
1889 0 : nFileSize = 0;
1890 : }
1891 0 : else if (pszMonthFound - pszStr > 1 &&
1892 0 : pszMonthFound[-1] == ',' &&
1893 : (int)strlen(pszMonthFound) > 1 + nLenMonth + 1 + 2 + 1 + 1 + 4 + 1 + 5 + 1 + 2)
1894 : {
1895 : /* Format of http://publicfiles.dep.state.fl.us/dear/BWR_GIS/2007NWFLULC/ */
1896 : /* " Sunday, June 20, 2010 6:46 PM 233170905 NWF2007LULCForSDE.zip" */
1897 0 : pszMonthFound += 1;
1898 0 : int nCurOffset = nLenMonth + 1;
1899 0 : int nDay = atoi(pszMonthFound + nCurOffset);
1900 0 : nCurOffset += 2 + 1 + 1;
1901 0 : int nYear = atoi(pszMonthFound + nCurOffset);
1902 0 : nCurOffset += 4 + 1;
1903 0 : int nHour = atoi(pszMonthFound + nCurOffset);
1904 0 : nCurOffset += 2 + 1;
1905 0 : int nMin = atoi(pszMonthFound + nCurOffset);
1906 0 : nCurOffset += 2 + 1;
1907 0 : if (strncmp(pszMonthFound + nCurOffset, "PM", 2) == 0)
1908 0 : nHour += 12;
1909 0 : else if (strncmp(pszMonthFound + nCurOffset, "AM", 2) != 0)
1910 0 : nHour = -1;
1911 0 : nCurOffset += 2;
1912 :
1913 0 : const char* pszFilesize = pszMonthFound + nCurOffset;
1914 0 : while(*pszFilesize == ' ')
1915 0 : pszFilesize ++;
1916 0 : if (*pszFilesize >= '1' && *pszFilesize <= '9')
1917 0 : nFileSize = CPLScanUIntBig(pszFilesize, strlen(pszFilesize));
1918 :
1919 0 : if (nDay >= 1 && nDay <= 31 && nYear >= 1900 &&
1920 : nHour >= 0 && nHour <= 24 && nMin >= 0 && nMin < 60)
1921 : {
1922 0 : brokendowntime.tm_year = nYear - 1900;
1923 0 : brokendowntime.tm_mon = iMonth;
1924 0 : brokendowntime.tm_mday = nDay;
1925 0 : brokendowntime.tm_hour = nHour;
1926 0 : brokendowntime.tm_min = nMin;
1927 0 : mTime = CPLYMDHMSToUnixTime(&brokendowntime);
1928 :
1929 0 : return TRUE;
1930 : }
1931 0 : nFileSize = 0;
1932 : }
1933 0 : return FALSE;
1934 : }
1935 : }
1936 :
1937 579 : return FALSE;
1938 : }
1939 :
1940 : /************************************************************************/
1941 : /* ParseHTMLFileList() */
1942 : /* */
1943 : /* Parse a file list document and return all the components. */
1944 : /************************************************************************/
1945 :
1946 16 : char** VSICurlFilesystemHandler::ParseHTMLFileList(const char* pszFilename,
1947 : char* pszData,
1948 : int* pbGotFileList)
1949 : {
1950 16 : CPLStringList oFileList;
1951 16 : char* pszLine = pszData;
1952 : char* c;
1953 16 : int nCount = 0;
1954 16 : int bIsHTMLDirList = FALSE;
1955 16 : CPLString osExpectedString;
1956 16 : CPLString osExpectedString2;
1957 16 : CPLString osExpectedString3;
1958 16 : CPLString osExpectedString4;
1959 16 : CPLString osExpectedString_unescaped;
1960 :
1961 16 : *pbGotFileList = FALSE;
1962 :
1963 : const char* pszDir;
1964 16 : if (EQUALN(pszFilename, "/vsicurl/http://", strlen("/vsicurl/http://")))
1965 16 : pszDir = strchr(pszFilename + strlen("/vsicurl/http://"), '/');
1966 0 : else if (EQUALN(pszFilename, "/vsicurl/https://", strlen("/vsicurl/https://")))
1967 0 : pszDir = strchr(pszFilename + strlen("/vsicurl/https://"), '/');
1968 : else
1969 0 : pszDir = strchr(pszFilename + strlen("/vsicurl/ftp://"), '/');
1970 16 : if (pszDir == NULL)
1971 0 : pszDir = "";
1972 : /* Apache */
1973 16 : osExpectedString = "<title>Index of ";
1974 16 : osExpectedString += pszDir;
1975 16 : osExpectedString += "</title>";
1976 : /* shttpd */
1977 16 : osExpectedString2 = "<title>Index of ";
1978 16 : osExpectedString2 += pszDir;
1979 16 : osExpectedString2 += "/</title>";
1980 : /* FTP */
1981 16 : osExpectedString3 = "FTP Listing of ";
1982 16 : osExpectedString3 += pszDir;
1983 16 : osExpectedString3 += "/";
1984 : /* Apache 1.3.33 */
1985 16 : osExpectedString4 = "<TITLE>Index of ";
1986 16 : osExpectedString4 += pszDir;
1987 16 : osExpectedString4 += "</TITLE>";
1988 :
1989 : /* The listing of http://dds.cr.usgs.gov/srtm/SRTM_image_sample/picture%20examples/ */
1990 : /* has "<title>Index of /srtm/SRTM_image_sample/picture examples</title>" so we must */
1991 : /* try unescaped %20 also */
1992 : /* Similar with http://datalib.usask.ca/gis/Data/Central_America_goodbutdoweown%3f/ */
1993 16 : if (strchr(pszDir, '%'))
1994 : {
1995 0 : char* pszUnescapedDir = CPLUnescapeString(pszDir, NULL, CPLES_URL);
1996 0 : osExpectedString_unescaped = "<title>Index of ";
1997 0 : osExpectedString_unescaped += pszUnescapedDir;
1998 0 : osExpectedString_unescaped += "</title>";
1999 0 : CPLFree(pszUnescapedDir);
2000 : }
2001 :
2002 16 : int nCountTable = 0;
2003 :
2004 738 : while( (c = VSICurlParserFindEOL( pszLine )) != NULL )
2005 : {
2006 706 : *c = 0;
2007 :
2008 : /* To avoid false positive on pages such as http://www.ngs.noaa.gov/PC_PROD/USGG2009BETA */
2009 : /* This is a heuristics, but normal HTML listing of files have not more than one table */
2010 706 : if (strstr(pszLine, "<table"))
2011 : {
2012 3 : nCountTable ++;
2013 3 : if (nCountTable == 2)
2014 : {
2015 0 : *pbGotFileList = FALSE;
2016 0 : return NULL;
2017 : }
2018 : }
2019 :
2020 706 : if (!bIsHTMLDirList &&
2021 : (strstr(pszLine, osExpectedString.c_str()) ||
2022 : strstr(pszLine, osExpectedString2.c_str()) ||
2023 : strstr(pszLine, osExpectedString3.c_str()) ||
2024 : strstr(pszLine, osExpectedString4.c_str()) ||
2025 : (osExpectedString_unescaped.size() != 0 && strstr(pszLine, osExpectedString_unescaped.c_str()))))
2026 : {
2027 3 : bIsHTMLDirList = TRUE;
2028 3 : *pbGotFileList = TRUE;
2029 : }
2030 : /* Subversion HTTP listing */
2031 : /* or Microsoft-IIS/6.0 listing (e.g. http://ortho.linz.govt.nz/tifs/2005_06/) */
2032 716 : else if (!bIsHTMLDirList && strstr(pszLine, "<title>"))
2033 : {
2034 : /* Detect something like : <html><head><title>gdal - Revision 20739: /trunk/autotest/gcore/data</title></head> */
2035 : /* The annoying thing is that what is after ': ' is a subpart of what is after http://server/ */
2036 13 : char* pszSubDir = strstr(pszLine, ": ");
2037 13 : if (pszSubDir == NULL)
2038 : /* or <title>ortho.linz.govt.nz - /tifs/2005_06/</title> */
2039 0 : pszSubDir = strstr(pszLine, "- ");
2040 13 : if (pszSubDir)
2041 : {
2042 13 : pszSubDir += 2;
2043 13 : char* pszTmp = strstr(pszSubDir, "</title>");
2044 13 : if (pszTmp)
2045 : {
2046 13 : if (pszTmp[-1] == '/')
2047 0 : pszTmp[-1] = 0;
2048 : else
2049 13 : *pszTmp = 0;
2050 13 : if (strstr(pszDir, pszSubDir))
2051 : {
2052 13 : bIsHTMLDirList = TRUE;
2053 13 : *pbGotFileList = TRUE;
2054 : }
2055 : }
2056 : }
2057 : }
2058 690 : else if (bIsHTMLDirList &&
2059 : (strstr(pszLine, "<a href=\"") != NULL || strstr(pszLine, "<A HREF=\"") != NULL) &&
2060 : strstr(pszLine, "<a href=\"http://") == NULL && /* exclude absolute links, like to subversion home */
2061 : strstr(pszLine, "Parent Directory") == NULL /* exclude parent directory */)
2062 : {
2063 592 : char *beginFilename = strstr(pszLine, "<a href=\"");
2064 592 : if (beginFilename == NULL)
2065 0 : beginFilename = strstr(pszLine, "<A HREF=\"");
2066 592 : beginFilename += strlen("<a href=\"");
2067 592 : char *endQuote = strchr(beginFilename, '"');
2068 592 : if (endQuote && strncmp(beginFilename, "?C=", 3) != 0 && strncmp(beginFilename, "?N=", 3) != 0)
2069 : {
2070 : struct tm brokendowntime;
2071 589 : memset(&brokendowntime, 0, sizeof(brokendowntime));
2072 589 : GUIntBig nFileSize = 0;
2073 589 : GIntBig mTime = 0;
2074 :
2075 : VSICurlParseHTMLDateTimeFileSize(pszLine,
2076 : brokendowntime,
2077 : nFileSize,
2078 589 : mTime);
2079 :
2080 589 : *endQuote = '\0';
2081 :
2082 : /* Remove trailing slash, that are returned for directories by */
2083 : /* Apache */
2084 589 : int bIsDirectory = FALSE;
2085 589 : if (endQuote[-1] == '/')
2086 : {
2087 24 : bIsDirectory = TRUE;
2088 24 : endQuote[-1] = 0;
2089 : }
2090 :
2091 : /* shttpd links include slashes from the root directory. Skip them */
2092 1178 : while(strchr(beginFilename, '/'))
2093 0 : beginFilename = strchr(beginFilename, '/') + 1;
2094 :
2095 589 : if (strcmp(beginFilename, ".") != 0 &&
2096 : strcmp(beginFilename, "..") != 0)
2097 : {
2098 : CPLString osCachedFilename =
2099 576 : CPLSPrintf("%s/%s", pszFilename + strlen("/vsicurl/"), beginFilename);
2100 576 : CachedFileProp* cachedFileProp = GetCachedFileProp(osCachedFilename);
2101 576 : cachedFileProp->eExists = EXIST_YES;
2102 576 : cachedFileProp->bIsDirectory = bIsDirectory;
2103 576 : cachedFileProp->mTime = mTime;
2104 576 : cachedFileProp->bHastComputedFileSize = nFileSize > 0;
2105 576 : cachedFileProp->fileSize = nFileSize;
2106 :
2107 576 : oFileList.AddString( beginFilename );
2108 : if (ENABLE_DEBUG)
2109 : CPLDebug("VSICURL", "File[%d] = %s, is_dir = %d, size = " CPL_FRMT_GUIB ", time = %04d/%02d/%02d %02d:%02d:%02d",
2110 : nCount, beginFilename, bIsDirectory, nFileSize,
2111 : brokendowntime.tm_year + 1900, brokendowntime.tm_mon + 1, brokendowntime.tm_mday,
2112 576 : brokendowntime.tm_hour, brokendowntime.tm_min, brokendowntime.tm_sec);
2113 576 : nCount ++;
2114 : }
2115 : }
2116 : }
2117 706 : pszLine = c + 1;
2118 : }
2119 :
2120 16 : return oFileList.StealList();
2121 : }
2122 :
2123 :
2124 : /************************************************************************/
2125 : /* VSICurlGetToken() */
2126 : /************************************************************************/
2127 :
2128 16 : static char* VSICurlGetToken(char* pszCurPtr, char** ppszNextToken)
2129 : {
2130 16 : if (pszCurPtr == NULL)
2131 0 : return NULL;
2132 :
2133 32 : while((*pszCurPtr) == ' ')
2134 0 : pszCurPtr ++;
2135 16 : if (*pszCurPtr == '\0')
2136 0 : return NULL;
2137 :
2138 16 : char* pszToken = pszCurPtr;
2139 104 : while((*pszCurPtr) != ' ' && (*pszCurPtr) != '\0')
2140 72 : pszCurPtr ++;
2141 16 : if (*pszCurPtr == '\0')
2142 0 : *ppszNextToken = NULL;
2143 : else
2144 : {
2145 16 : *pszCurPtr = '\0';
2146 16 : pszCurPtr ++;
2147 56 : while((*pszCurPtr) == ' ')
2148 24 : pszCurPtr ++;
2149 16 : *ppszNextToken = pszCurPtr;
2150 : }
2151 :
2152 16 : return pszToken;
2153 : }
2154 :
2155 : /************************************************************************/
2156 : /* VSICurlParseFullFTPLine() */
2157 : /************************************************************************/
2158 :
2159 : /* Parse lines like the following ones :
2160 : -rw-r--r-- 1 10003 100 430 Jul 04 2008 COPYING
2161 : lrwxrwxrwx 1 ftp ftp 28 Jun 14 14:13 MPlayer -> mirrors/mplayerhq.hu/MPlayer
2162 : -rw-r--r-- 1 ftp ftp 725614592 May 13 20:13 Fedora-15-x86_64-Live-KDE.iso
2163 : drwxr-xr-x 280 1003 1003 6656 Aug 26 04:17 gnu
2164 : */
2165 :
2166 2 : static int VSICurlParseFullFTPLine(char* pszLine,
2167 : char*& pszFilename,
2168 : int& bSizeValid,
2169 : GUIntBig& nSize,
2170 : int& bIsDirectory,
2171 : GIntBig& nUnixTime)
2172 : {
2173 2 : char* pszNextToken = pszLine;
2174 2 : char* pszPermissions = VSICurlGetToken(pszNextToken, &pszNextToken);
2175 2 : if (pszPermissions == NULL || strlen(pszPermissions) != 10)
2176 0 : return FALSE;
2177 2 : bIsDirectory = (pszPermissions[0] == 'd');
2178 :
2179 : int i;
2180 8 : for(i = 0; i < 3; i++)
2181 : {
2182 6 : if (VSICurlGetToken(pszNextToken, &pszNextToken) == NULL)
2183 0 : return FALSE;
2184 : }
2185 :
2186 2 : char* pszSize = VSICurlGetToken(pszNextToken, &pszNextToken);
2187 2 : if (pszSize == NULL)
2188 0 : return FALSE;
2189 :
2190 2 : if (pszPermissions[0] == '-')
2191 : {
2192 : /* Regular file */
2193 2 : bSizeValid = TRUE;
2194 2 : nSize = CPLScanUIntBig(pszSize, strlen(pszSize));
2195 : }
2196 :
2197 : struct tm brokendowntime;
2198 2 : memset(&brokendowntime, 0, sizeof(brokendowntime));
2199 2 : int bBrokenDownTimeValid = TRUE;
2200 :
2201 2 : char* pszMonth = VSICurlGetToken(pszNextToken, &pszNextToken);
2202 2 : if (pszMonth == NULL || strlen(pszMonth) != 3)
2203 0 : return FALSE;
2204 :
2205 20 : for(i = 0; i < 12; i++)
2206 : {
2207 20 : if (EQUALN(pszMonth, apszMonths[i], 3))
2208 2 : break;
2209 : }
2210 2 : if (i < 12)
2211 2 : brokendowntime.tm_mon = i;
2212 : else
2213 0 : bBrokenDownTimeValid = FALSE;
2214 :
2215 2 : char* pszDay = VSICurlGetToken(pszNextToken, &pszNextToken);
2216 2 : if (pszDay == NULL || (strlen(pszDay) != 1 && strlen(pszDay) != 2))
2217 0 : return FALSE;
2218 2 : int nDay = atoi(pszDay);
2219 4 : if (nDay >= 1 && nDay <= 31)
2220 2 : brokendowntime.tm_mday = nDay;
2221 : else
2222 0 : bBrokenDownTimeValid = FALSE;
2223 :
2224 2 : char* pszHourOrYear = VSICurlGetToken(pszNextToken, &pszNextToken);
2225 2 : if (pszHourOrYear == NULL || (strlen(pszHourOrYear) != 4 && strlen(pszHourOrYear) != 5))
2226 0 : return FALSE;
2227 2 : if (strlen(pszHourOrYear) == 4)
2228 : {
2229 2 : brokendowntime.tm_year = atoi(pszHourOrYear) - 1900;
2230 : }
2231 : else
2232 : {
2233 : time_t sTime;
2234 0 : time(&sTime);
2235 : struct tm currentBrokendowntime;
2236 0 : CPLUnixTimeToYMDHMS((GIntBig)sTime, ¤tBrokendowntime);
2237 0 : brokendowntime.tm_year = currentBrokendowntime.tm_year;
2238 0 : brokendowntime.tm_hour = atoi(pszHourOrYear);
2239 0 : brokendowntime.tm_min = atoi(pszHourOrYear + 3);
2240 : }
2241 :
2242 2 : if (bBrokenDownTimeValid)
2243 2 : nUnixTime = CPLYMDHMSToUnixTime(&brokendowntime);
2244 : else
2245 0 : nUnixTime = 0;
2246 :
2247 2 : if (pszNextToken == NULL)
2248 0 : return FALSE;
2249 :
2250 2 : pszFilename = pszNextToken;
2251 :
2252 2 : char* pszCurPtr = pszFilename;
2253 33 : while( *pszCurPtr != '\0')
2254 : {
2255 : /* In case of a link, stop before the pointed part of the link */
2256 29 : if (pszPermissions[0] == 'l' && strncmp(pszCurPtr, " -> ", 4) == 0)
2257 : {
2258 0 : break;
2259 : }
2260 29 : pszCurPtr ++;
2261 : }
2262 2 : *pszCurPtr = '\0';
2263 :
2264 2 : return TRUE;
2265 : }
2266 :
2267 : /************************************************************************/
2268 : /* GetFileList() */
2269 : /************************************************************************/
2270 :
2271 18 : char** VSICurlFilesystemHandler::GetFileList(const char *pszDirname, int* pbGotFileList)
2272 : {
2273 : if (ENABLE_DEBUG)
2274 18 : CPLDebug("VSICURL", "GetFileList(%s)" , pszDirname);
2275 :
2276 18 : *pbGotFileList = FALSE;
2277 :
2278 : /* HACK (optimization in fact) for MBTiles driver */
2279 18 : if (strstr(pszDirname, ".tiles.mapbox.com") != NULL)
2280 1 : return NULL;
2281 :
2282 17 : if (strncmp(pszDirname, "/vsicurl/ftp", strlen("/vsicurl/ftp")) == 0)
2283 : {
2284 : WriteFuncStruct sWriteFuncData;
2285 1 : sWriteFuncData.pBuffer = NULL;
2286 :
2287 1 : CPLString osDirname(pszDirname + strlen("/vsicurl/"));
2288 1 : osDirname += '/';
2289 :
2290 1 : char** papszFileList = NULL;
2291 :
2292 1 : for(int iTry=0;iTry<2;iTry++)
2293 : {
2294 1 : CURL* hCurlHandle = GetCurlHandleFor(osDirname);
2295 1 : VSICurlSetOptions(hCurlHandle, osDirname.c_str());
2296 :
2297 : /* On the first pass, we want to try fetching all the possible */
2298 : /* informations (filename, file/directory, size). If that */
2299 : /* does not work, then try again with CURLOPT_DIRLISTONLY set */
2300 1 : if (iTry == 1)
2301 : {
2302 : /* 7.16.4 */
2303 : #if LIBCURL_VERSION_NUM <= 0x071004
2304 : curl_easy_setopt(hCurlHandle, CURLOPT_FTPLISTONLY, 1);
2305 : #elif LIBCURL_VERSION_NUM > 0x071004
2306 0 : curl_easy_setopt(hCurlHandle, CURLOPT_DIRLISTONLY, 1);
2307 : #endif
2308 : }
2309 :
2310 1 : VSICURLInitWriteFuncStruct(&sWriteFuncData, NULL, NULL, NULL);
2311 1 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
2312 1 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlHandleWriteFunc);
2313 :
2314 : char szCurlErrBuf[CURL_ERROR_SIZE+1];
2315 1 : szCurlErrBuf[0] = '\0';
2316 1 : curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
2317 :
2318 1 : curl_easy_perform(hCurlHandle);
2319 :
2320 1 : if (sWriteFuncData.pBuffer == NULL)
2321 0 : return NULL;
2322 :
2323 1 : char* pszLine = sWriteFuncData.pBuffer;
2324 : char* c;
2325 1 : int nCount = 0;
2326 :
2327 1 : if (EQUALN(pszLine, "<!DOCTYPE HTML", strlen("<!DOCTYPE HTML")) ||
2328 : EQUALN(pszLine, "<HTML>", 6))
2329 : {
2330 : papszFileList = ParseHTMLFileList(pszDirname,
2331 : sWriteFuncData.pBuffer,
2332 0 : pbGotFileList);
2333 0 : break;
2334 : }
2335 1 : else if (iTry == 0)
2336 : {
2337 1 : CPLStringList oFileList;
2338 1 : *pbGotFileList = TRUE;
2339 :
2340 4 : while( (c = strchr(pszLine, '\n')) != NULL)
2341 : {
2342 2 : *c = 0;
2343 2 : if (c - pszLine > 0 && c[-1] == '\r')
2344 0 : c[-1] = 0;
2345 :
2346 2 : char* pszFilename = NULL;
2347 2 : int bSizeValid = FALSE;
2348 2 : GUIntBig nFileSize = 0;
2349 2 : int bIsDirectory = FALSE;
2350 2 : GIntBig mUnixTime = 0;
2351 2 : if (!VSICurlParseFullFTPLine(pszLine, pszFilename,
2352 : bSizeValid, nFileSize,
2353 : bIsDirectory, mUnixTime))
2354 0 : break;
2355 :
2356 2 : if (strcmp(pszFilename, ".") != 0 &&
2357 : strcmp(pszFilename, "..") != 0)
2358 : {
2359 : CPLString osCachedFilename =
2360 2 : CPLSPrintf("%s/%s", pszDirname + strlen("/vsicurl/"), pszFilename);
2361 2 : CachedFileProp* cachedFileProp = GetCachedFileProp(osCachedFilename);
2362 2 : cachedFileProp->eExists = EXIST_YES;
2363 2 : cachedFileProp->bHastComputedFileSize = bSizeValid;
2364 2 : cachedFileProp->fileSize = nFileSize;
2365 2 : cachedFileProp->bIsDirectory = bIsDirectory;
2366 2 : cachedFileProp->mTime = mUnixTime;
2367 :
2368 2 : oFileList.AddString(pszFilename);
2369 : if (ENABLE_DEBUG)
2370 : {
2371 : struct tm brokendowntime;
2372 2 : CPLUnixTimeToYMDHMS(mUnixTime, &brokendowntime);
2373 : CPLDebug("VSICURL", "File[%d] = %s, is_dir = %d, size = " CPL_FRMT_GUIB ", time = %04d/%02d/%02d %02d:%02d:%02d",
2374 : nCount, pszFilename, bIsDirectory, nFileSize,
2375 : brokendowntime.tm_year + 1900, brokendowntime.tm_mon + 1, brokendowntime.tm_mday,
2376 2 : brokendowntime.tm_hour, brokendowntime.tm_min, brokendowntime.tm_sec);
2377 : }
2378 :
2379 2 : nCount ++;
2380 : }
2381 :
2382 2 : pszLine = c + 1;
2383 : }
2384 :
2385 1 : if (c == NULL)
2386 : {
2387 1 : papszFileList = oFileList.StealList();
2388 : break;
2389 0 : }
2390 : }
2391 : else
2392 : {
2393 0 : CPLStringList oFileList;
2394 0 : *pbGotFileList = TRUE;
2395 :
2396 0 : while( (c = strchr(pszLine, '\n')) != NULL)
2397 : {
2398 0 : *c = 0;
2399 0 : if (c - pszLine > 0 && c[-1] == '\r')
2400 0 : c[-1] = 0;
2401 :
2402 0 : if (strcmp(pszLine, ".") != 0 &&
2403 : strcmp(pszLine, "..") != 0)
2404 : {
2405 0 : oFileList.AddString(pszLine);
2406 : if (ENABLE_DEBUG)
2407 0 : CPLDebug("VSICURL", "File[%d] = %s", nCount, pszLine);
2408 0 : nCount ++;
2409 : }
2410 :
2411 0 : pszLine = c + 1;
2412 : }
2413 :
2414 0 : papszFileList = oFileList.StealList();
2415 : }
2416 :
2417 0 : CPLFree(sWriteFuncData.pBuffer);
2418 0 : sWriteFuncData.pBuffer = NULL;
2419 : }
2420 :
2421 1 : CPLFree(sWriteFuncData.pBuffer);
2422 :
2423 1 : return papszFileList;
2424 : }
2425 :
2426 : /* Try to recognize HTML pages that list the content of a directory */
2427 : /* Currently this supports what Apache and shttpd can return */
2428 16 : else if (strncmp(pszDirname, "/vsicurl/http://", strlen("/vsicurl/http://")) == 0 ||
2429 : strncmp(pszDirname, "/vsicurl/https://", strlen("/vsicurl/https://")) == 0)
2430 : {
2431 : WriteFuncStruct sWriteFuncData;
2432 :
2433 16 : CPLString osDirname(pszDirname + strlen("/vsicurl/"));
2434 16 : osDirname += '/';
2435 :
2436 : #if LIBCURL_VERSION_NUM < 0x070B00
2437 : /* Curl 7.10.X doesn't manage to unset the CURLOPT_RANGE that would have been */
2438 : /* previously set, so we have to reinit the connection handle */
2439 : GetCurlHandleFor("");
2440 : #endif
2441 :
2442 16 : CURL* hCurlHandle = GetCurlHandleFor(osDirname);
2443 16 : VSICurlSetOptions(hCurlHandle, osDirname.c_str());
2444 :
2445 16 : curl_easy_setopt(hCurlHandle, CURLOPT_RANGE, NULL);
2446 :
2447 16 : VSICURLInitWriteFuncStruct(&sWriteFuncData, NULL, NULL, NULL);
2448 16 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEDATA, &sWriteFuncData);
2449 16 : curl_easy_setopt(hCurlHandle, CURLOPT_WRITEFUNCTION, VSICurlHandleWriteFunc);
2450 :
2451 : char szCurlErrBuf[CURL_ERROR_SIZE+1];
2452 16 : szCurlErrBuf[0] = '\0';
2453 16 : curl_easy_setopt(hCurlHandle, CURLOPT_ERRORBUFFER, szCurlErrBuf );
2454 :
2455 16 : curl_easy_perform(hCurlHandle);
2456 :
2457 16 : if (sWriteFuncData.pBuffer == NULL)
2458 0 : return NULL;
2459 :
2460 : char** papszFileList = ParseHTMLFileList(pszDirname,
2461 : sWriteFuncData.pBuffer,
2462 16 : pbGotFileList);
2463 :
2464 16 : CPLFree(sWriteFuncData.pBuffer);
2465 16 : return papszFileList;
2466 : }
2467 :
2468 0 : return NULL;
2469 : }
2470 :
2471 : /************************************************************************/
2472 : /* Stat() */
2473 : /************************************************************************/
2474 :
2475 818 : int VSICurlFilesystemHandler::Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
2476 : int nFlags )
2477 : {
2478 818 : CPLString osFilename(pszFilename);
2479 :
2480 818 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
2481 :
2482 : const char* pszOptionVal =
2483 818 : CPLGetConfigOption( "GDAL_DISABLE_READDIR_ON_OPEN", "NO" );
2484 : int bSkipReadDir = EQUAL(pszOptionVal, "EMPTY_DIR") ||
2485 818 : CSLTestBoolean(pszOptionVal);
2486 :
2487 : /* Does it look like a FTP directory ? */
2488 820 : if (strncmp(osFilename, "/vsicurl/ftp", strlen("/vsicurl/ftp")) == 0 &&
2489 2 : pszFilename[strlen(osFilename) - 1] == '/' && !bSkipReadDir)
2490 : {
2491 0 : char** papszFileList = ReadDir(osFilename);
2492 0 : if (papszFileList)
2493 : {
2494 0 : pStatBuf->st_mode = S_IFDIR;
2495 0 : pStatBuf->st_size = 0;
2496 :
2497 0 : CSLDestroy(papszFileList);
2498 :
2499 0 : return 0;
2500 : }
2501 0 : return -1;
2502 : }
2503 818 : else if (strchr(CPLGetFilename(osFilename), '.') != NULL &&
2504 : !EQUALN(CPLGetExtension(osFilename), "zip", 3) &&
2505 : strstr(osFilename, ".zip.") != NULL &&
2506 : strstr(osFilename, ".ZIP.") != NULL &&
2507 : !bSkipReadDir)
2508 : {
2509 : int bGotFileList;
2510 0 : char** papszFileList = ReadDir(CPLGetDirname(osFilename), &bGotFileList);
2511 0 : int bFound = (VSICurlIsFileInList(papszFileList, CPLGetFilename(osFilename)) != -1);
2512 0 : CSLDestroy(papszFileList);
2513 0 : if (bGotFileList && !bFound)
2514 : {
2515 0 : return -1;
2516 : }
2517 : }
2518 :
2519 818 : VSICurlHandle oHandle( this, osFilename + strlen("/vsicurl/"));
2520 :
2521 818 : if ( oHandle.IsKnownFileSize() ||
2522 : ((nFlags & VSI_STAT_SIZE_FLAG) && !oHandle.IsDirectory() &&
2523 : CSLTestBoolean(CPLGetConfigOption("CPL_VSIL_CURL_SLOW_GET_SIZE", "YES"))) )
2524 33 : pStatBuf->st_size = oHandle.GetFileSize();
2525 :
2526 818 : int nRet = (oHandle.Exists()) ? 0 : -1;
2527 818 : pStatBuf->st_mtime = oHandle.GetMTime();
2528 818 : pStatBuf->st_mode = oHandle.IsDirectory() ? S_IFDIR : S_IFREG;
2529 818 : return nRet;
2530 : }
2531 :
2532 : /************************************************************************/
2533 : /* Unlink() */
2534 : /************************************************************************/
2535 :
2536 0 : int VSICurlFilesystemHandler::Unlink( const char *pszFilename )
2537 : {
2538 0 : return -1;
2539 : }
2540 :
2541 : /************************************************************************/
2542 : /* Rename() */
2543 : /************************************************************************/
2544 :
2545 0 : int VSICurlFilesystemHandler::Rename( const char *oldpath, const char *newpath )
2546 : {
2547 0 : return -1;
2548 : }
2549 :
2550 : /************************************************************************/
2551 : /* Mkdir() */
2552 : /************************************************************************/
2553 :
2554 0 : int VSICurlFilesystemHandler::Mkdir( const char *pszDirname, long nMode )
2555 : {
2556 0 : return -1;
2557 : }
2558 : /************************************************************************/
2559 : /* Rmdir() */
2560 : /************************************************************************/
2561 :
2562 0 : int VSICurlFilesystemHandler::Rmdir( const char *pszDirname )
2563 : {
2564 0 : return -1;
2565 : }
2566 :
2567 : /************************************************************************/
2568 : /* ReadDir() */
2569 : /************************************************************************/
2570 :
2571 825 : char** VSICurlFilesystemHandler::ReadDir( const char *pszDirname, int* pbGotFileList )
2572 : {
2573 825 : CPLString osDirname(pszDirname);
2574 1650 : while (osDirname[strlen(osDirname) - 1] == '/')
2575 0 : osDirname.erase(strlen(osDirname) - 1);
2576 :
2577 825 : const char* pszUpDir = strstr(osDirname, "/..");
2578 825 : if (pszUpDir != NULL)
2579 : {
2580 0 : int pos = pszUpDir - osDirname.c_str() - 1;
2581 0 : while(pos >= 0 && osDirname[pos] != '/')
2582 0 : pos --;
2583 0 : if (pos >= 1)
2584 : {
2585 0 : osDirname = osDirname.substr(0, pos) + CPLString(pszUpDir + 3);
2586 : }
2587 : }
2588 :
2589 825 : CPLMutexHolder oHolder( &hMutex );
2590 :
2591 : /* If we know the file exists and is not a directory, then don't try to list its content */
2592 825 : CachedFileProp* cachedFileProp = GetCachedFileProp(osDirname.c_str() + strlen("/vsicurl/"));
2593 825 : if (cachedFileProp->eExists == EXIST_YES && !cachedFileProp->bIsDirectory)
2594 : {
2595 738 : if (pbGotFileList)
2596 0 : *pbGotFileList = TRUE;
2597 738 : return NULL;
2598 : }
2599 :
2600 87 : CachedDirList* psCachedDirList = cacheDirList[osDirname];
2601 87 : if (psCachedDirList == NULL)
2602 : {
2603 18 : psCachedDirList = (CachedDirList*) CPLMalloc(sizeof(CachedDirList));
2604 18 : psCachedDirList->papszFileList = GetFileList(osDirname, &psCachedDirList->bGotFileList);
2605 18 : cacheDirList[osDirname] = psCachedDirList;
2606 : }
2607 :
2608 87 : if (pbGotFileList)
2609 58 : *pbGotFileList = psCachedDirList->bGotFileList;
2610 :
2611 87 : return CSLDuplicate(psCachedDirList->papszFileList);
2612 : }
2613 :
2614 : /************************************************************************/
2615 : /* ReadDir() */
2616 : /************************************************************************/
2617 :
2618 767 : char** VSICurlFilesystemHandler::ReadDir( const char *pszDirname )
2619 : {
2620 767 : return ReadDir(pszDirname, NULL);
2621 : }
2622 :
2623 : /************************************************************************/
2624 : /* VSIInstallCurlFileHandler() */
2625 : /************************************************************************/
2626 :
2627 : /**
2628 : * \brief Install /vsicurl/ HTTP/FTP file system handler (requires libcurl)
2629 : *
2630 : * A special file handler is installed that allows reading on-the-fly of files
2631 : * available through HTTP/FTP web protocols, without downloading the entire file.
2632 : *
2633 : * Recognized filenames are of the form /vsicurl/http://path/to/remote/ressource or
2634 : * /vsicurl/ftp://path/to/remote/ressource where path/to/remote/ressource is the
2635 : * URL of a remote ressource.
2636 : *
2637 : * Partial downloads (requires the HTTP server to support random reading) are done
2638 : * with a 16 KB granularity by default. If the driver detects sequential reading
2639 : * it will progressively increase the chunk size up to 2 MB to improve download
2640 : * performance.
2641 : *
2642 : * The GDAL_HTTP_PROXY, GDAL_HTTP_PROXYUSERPWD and GDAL_PROXY_AUTH configuration options can be
2643 : * used to define a proxy server. The syntax to use is the one of Curl CURLOPT_PROXY,
2644 : * CURLOPT_PROXYUSERPWD and CURLOPT_PROXYAUTH options.
2645 : *
2646 : * Starting with GDAL 1.10, the file can be cached in RAM by setting the configuration option
2647 : * VSI_CACHE to TRUE. The cache size defaults to 25 MB, but can be modified by setting
2648 : * the configuration option VSI_CACHE_SIZE (in bytes).
2649 : *
2650 : * VSIStatL() will return the size in st_size member and file
2651 : * nature- file or directory - in st_mode member (the later only reliable with FTP
2652 : * resources for now).
2653 : *
2654 : * VSIReadDir() should be able to parse the HTML directory listing returned by the
2655 : * most popular web servers, such as Apache or Microsoft IIS.
2656 : *
2657 : * This special file handler can be combined with other virtual filesystems handlers,
2658 : * such as /vsizip. For example, /vsizip//vsicurl/path/to/remote/file.zip/path/inside/zip
2659 : *
2660 : * @since GDAL 1.8.0
2661 : */
2662 757 : void VSIInstallCurlFileHandler(void)
2663 : {
2664 757 : VSIFileManager::InstallHandler( "/vsicurl/", new VSICurlFilesystemHandler );
2665 757 : }
2666 :
2667 : /************************************************************************/
2668 : /* VSICurlInstallReadCbk() */
2669 : /************************************************************************/
2670 :
2671 1 : int VSICurlInstallReadCbk (VSILFILE* fp,
2672 : VSICurlReadCbkFunc pfnReadCbk,
2673 : void* pfnUserData,
2674 : int bStopOnInterrruptUntilUninstall)
2675 : {
2676 : return ((VSICurlHandle*)fp)->InstallReadCbk(pfnReadCbk, pfnUserData,
2677 1 : bStopOnInterrruptUntilUninstall);
2678 : }
2679 :
2680 :
2681 : /************************************************************************/
2682 : /* VSICurlUninstallReadCbk() */
2683 : /************************************************************************/
2684 :
2685 1 : int VSICurlUninstallReadCbk(VSILFILE* fp)
2686 : {
2687 1 : return ((VSICurlHandle*)fp)->UninstallReadCbk();
2688 : }
2689 :
2690 : #endif /* HAVE_CURL */
|
