1 : /******************************************************************************
2 : * $Id: gdalhttp.cpp 22007 2011-03-21 20:44:14Z rouault $
3 : *
4 : * Project: WMS Client Driver
5 : * Purpose: Implementation of Dataset and RasterBand classes for WMS
6 : * and other similar services.
7 : * Author: Adam Nowacki, nowak@xpam.de
8 : *
9 : ******************************************************************************
10 : * Copyright (c) 2007, Adam Nowacki
11 : *
12 : * Permission is hereby granted, free of charge, to any person obtaining a
13 : * copy of this software and associated documentation files (the "Software"),
14 : * to deal in the Software without restriction, including without limitation
15 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 : * and/or sell copies of the Software, and to permit persons to whom the
17 : * Software is furnished to do so, subject to the following conditions:
18 : *
19 : * The above copyright notice and this permission notice shall be included
20 : * in all copies or substantial portions of the Software.
21 : *
22 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 : * DEALINGS IN THE SOFTWARE.
29 : ****************************************************************************/
30 :
31 : #include "stdinc.h"
32 :
33 : /* CURLINFO_RESPONSE_CODE was known as CURLINFO_HTTP_CODE in libcurl 7.10.7 and earlier */
34 : #if LIBCURL_VERSION_NUM < 0x070a07
35 : #define CURLINFO_RESPONSE_CODE CURLINFO_HTTP_CODE
36 : #endif
37 :
38 64 : static size_t CPLHTTPWriteFunc(void *buffer, size_t count, size_t nmemb, void *req) {
39 64 : CPLHTTPRequest *psRequest = reinterpret_cast<CPLHTTPRequest *>(req);
40 64 : size_t size = count * nmemb;
41 :
42 64 : if (size == 0) return 0;
43 :
44 64 : const size_t required_size = psRequest->nDataLen + size + 1;
45 64 : if (required_size > psRequest->nDataAlloc) {
46 13 : size_t new_size = required_size * 2;
47 13 : if (new_size < 512) new_size = 512;
48 13 : psRequest->nDataAlloc = new_size;
49 13 : GByte * pabyNewData = reinterpret_cast<GByte *>(VSIRealloc(psRequest->pabyData, new_size));
50 13 : if (pabyNewData == NULL) {
51 0 : VSIFree(psRequest->pabyData);
52 0 : psRequest->pabyData = NULL;
53 0 : psRequest->pszError = CPLStrdup(CPLString().Printf("Out of memory allocating %u bytes for HTTP data buffer.", static_cast<int>(new_size)));
54 0 : psRequest->nDataAlloc = 0;
55 0 : psRequest->nDataLen = 0;
56 0 : return 0;
57 : }
58 13 : psRequest->pabyData = pabyNewData;
59 : }
60 64 : memcpy(psRequest->pabyData + psRequest->nDataLen, buffer, size);
61 64 : psRequest->nDataLen += size;
62 64 : psRequest->pabyData[psRequest->nDataLen] = 0;
63 64 : return nmemb;
64 : }
65 :
66 3 : void CPLHTTPInitializeRequest(CPLHTTPRequest *psRequest, const char *pszURL, const char *const *papszOptions) {
67 3 : psRequest->pszURL = CPLStrdup(pszURL);
68 3 : psRequest->papszOptions = CSLDuplicate(const_cast<char **>(papszOptions));
69 3 : psRequest->nStatus = 0;
70 3 : psRequest->pszContentType = 0;
71 3 : psRequest->pszError = 0;
72 3 : psRequest->pabyData = 0;
73 3 : psRequest->nDataLen = 0;
74 3 : psRequest->nDataAlloc = 0;
75 3 : psRequest->m_curl_handle = 0;
76 3 : psRequest->m_headers = 0;
77 3 : psRequest->m_curl_error = 0;
78 :
79 3 : psRequest->m_curl_handle = curl_easy_init();
80 3 : if (psRequest->m_curl_handle == NULL) {
81 0 : CPLError(CE_Fatal, CPLE_AppDefined, "CPLHTTPInitializeRequest(): Unable to create CURL handle.");
82 : }
83 :
84 : /* Set User-Agent */
85 3 : const char *pszUserAgent = CSLFetchNameValue(const_cast<char **>(psRequest->papszOptions), "USERAGENT");
86 3 : if (pszUserAgent == NULL)
87 3 : pszUserAgent = "GDAL WMS driver (http://www.gdal.org/frmt_wms.html)";
88 3 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_USERAGENT, pszUserAgent);
89 :
90 : /* Set Referer */
91 3 : const char *pszReferer = CSLFetchNameValue(const_cast<char **>(psRequest->papszOptions), "REFERER");
92 3 : if (pszReferer != NULL)
93 0 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_REFERER, pszReferer);
94 :
95 : /* Set URL */
96 3 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_URL, psRequest->pszURL);
97 :
98 : /* Set timeout.*/
99 3 : const char *timeout = CSLFetchNameValue(const_cast<char **>(psRequest->papszOptions), "TIMEOUT");
100 3 : if (timeout != NULL) {
101 3 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_TIMEOUT, atoi(timeout));
102 : }
103 :
104 : /* Set Headers (copied&pasted from cpl_http.cpp, but unused by callers of CPLHTTPInitializeRequest) .*/
105 3 : const char *headers = CSLFetchNameValue(const_cast<char **>(psRequest->papszOptions), "HEADERS");
106 3 : if (headers != NULL) {
107 0 : psRequest->m_headers = curl_slist_append(psRequest->m_headers, headers);
108 0 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_HTTPHEADER, psRequest->m_headers);
109 : }
110 :
111 3 : if (CSLFetchBoolean(const_cast<char **>(psRequest->papszOptions), "UNSAFESSL", 0)) {
112 0 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_SSL_VERIFYPEER, 0L);
113 0 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_SSL_VERIFYHOST, 0L);
114 : }
115 :
116 : /* Enable following redirections. Requires libcurl 7.10.1 at least */
117 3 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_FOLLOWLOCATION, 1);
118 3 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_MAXREDIRS, 10);
119 :
120 : /* NOSIGNAL should be set to true for timeout to work in multithread
121 : environments on Unix, requires libcurl 7.10 or more recent.
122 : (this force avoiding the use of sgnal handlers) */
123 : #ifdef CURLOPT_NOSIGNAL
124 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_NOSIGNAL, 1);
125 : #endif
126 :
127 3 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_WRITEDATA, psRequest);
128 3 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_WRITEFUNCTION, CPLHTTPWriteFunc);
129 :
130 3 : psRequest->m_curl_error = reinterpret_cast<char *>(CPLMalloc(CURL_ERROR_SIZE + 1));
131 3 : psRequest->m_curl_error[0] = '\0';
132 3 : curl_easy_setopt(psRequest->m_curl_handle, CURLOPT_ERRORBUFFER, psRequest->m_curl_error);
133 :
134 : /* Set Proxy parameters */
135 3 : const char* pszProxy = CSLFetchNameValue( const_cast<char **>(psRequest->papszOptions), "PROXY" );
136 3 : if (pszProxy == NULL)
137 3 : pszProxy = CPLGetConfigOption("GDAL_HTTP_PROXY", NULL);
138 3 : if (pszProxy)
139 0 : curl_easy_setopt(psRequest->m_curl_handle,CURLOPT_PROXY,pszProxy);
140 :
141 3 : const char* pszProxyUserPwd = CSLFetchNameValue( const_cast<char **>(psRequest->papszOptions), "PROXYUSERPWD" );
142 3 : if (pszProxyUserPwd == NULL)
143 3 : pszProxyUserPwd = CPLGetConfigOption("GDAL_HTTP_PROXYUSERPWD", NULL);
144 3 : if (pszProxyUserPwd)
145 0 : curl_easy_setopt(psRequest->m_curl_handle,CURLOPT_PROXYUSERPWD,pszProxyUserPwd);
146 3 : }
147 :
148 3 : void CPLHTTPCleanupRequest(CPLHTTPRequest *psRequest) {
149 3 : if (psRequest->m_curl_handle) {
150 3 : curl_easy_cleanup(psRequest->m_curl_handle);
151 3 : psRequest->m_curl_handle = 0;
152 : }
153 3 : if (psRequest->m_headers) {
154 0 : curl_slist_free_all(psRequest->m_headers);
155 0 : psRequest->m_headers = 0;
156 : }
157 3 : if (psRequest->m_curl_error) {
158 3 : CPLFree(psRequest->m_curl_error);
159 3 : psRequest->m_curl_error = 0;
160 : }
161 :
162 3 : if (psRequest->pszContentType) {
163 3 : CPLFree(psRequest->pszContentType);
164 3 : psRequest->pszContentType = 0;
165 : }
166 3 : if (psRequest->pszError) {
167 0 : CPLFree(psRequest->pszError);
168 0 : psRequest->pszError = 0;
169 : }
170 3 : if (psRequest->pabyData) {
171 3 : CPLFree(psRequest->pabyData);
172 3 : psRequest->pabyData = 0;
173 3 : psRequest->nDataLen = 0;
174 3 : psRequest->nDataAlloc = 0;
175 : }
176 3 : if (psRequest->papszOptions) {
177 3 : CSLDestroy(psRequest->papszOptions);
178 3 : psRequest->papszOptions = 0;
179 : }
180 3 : if (psRequest->pszURL) {
181 3 : CPLFree(psRequest->pszURL);
182 3 : psRequest->pszURL = 0;
183 : }
184 3 : }
185 :
186 2 : CPLErr CPLHTTPFetchMulti(CPLHTTPRequest *pasRequest, int nRequestCount, const char *const *papszOptions) {
187 2 : CPLErr ret = CE_None;
188 2 : CURLM *curl_multi = 0;
189 : int still_running;
190 : int max_conn;
191 : int i, conn_i;
192 :
193 2 : const char *max_conn_opt = CSLFetchNameValue(const_cast<char **>(papszOptions), "MAXCONN");
194 4 : if (max_conn_opt && (max_conn_opt[0] != '\0')) {
195 2 : max_conn = MAX(1, MIN(atoi(max_conn_opt), 1000));
196 : } else {
197 0 : max_conn = 5;
198 : }
199 :
200 2 : curl_multi = curl_multi_init();
201 2 : if (curl_multi == NULL) {
202 0 : CPLError(CE_Fatal, CPLE_AppDefined, "CPLHTTPFetchMulti(): Unable to create CURL multi-handle.");
203 : }
204 :
205 : // add at most max_conn requests
206 5 : for (conn_i = 0; conn_i < MIN(nRequestCount, max_conn); ++conn_i) {
207 3 : CPLHTTPRequest *const psRequest = &pasRequest[conn_i];
208 3 : CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1, nRequestCount, pasRequest[conn_i].pszURL);
209 3 : curl_multi_add_handle(curl_multi, psRequest->m_curl_handle);
210 : }
211 :
212 6 : while (curl_multi_perform(curl_multi, &still_running) == CURLM_CALL_MULTI_PERFORM);
213 87 : while (still_running || (conn_i != nRequestCount)) {
214 : struct timeval timeout;
215 : fd_set fdread, fdwrite, fdexcep;
216 : int maxfd;
217 : CURLMsg *msg;
218 : int msgs_in_queue;
219 :
220 84 : do {
221 84 : msg = curl_multi_info_read(curl_multi, &msgs_in_queue);
222 84 : if (msg != NULL) {
223 1 : if (msg->msg == CURLMSG_DONE) { // transfer completed, check if we have more waiting and add them
224 1 : if (conn_i < nRequestCount) {
225 0 : CPLHTTPRequest *const psRequest = &pasRequest[conn_i];
226 0 : CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1, nRequestCount, pasRequest[conn_i].pszURL);
227 0 : curl_multi_add_handle(curl_multi, psRequest->m_curl_handle);
228 0 : ++conn_i;
229 : }
230 : }
231 : }
232 : } while (msg != NULL);
233 83 : FD_ZERO(&fdread);
234 83 : FD_ZERO(&fdwrite);
235 83 : FD_ZERO(&fdexcep);
236 83 : curl_multi_fdset(curl_multi, &fdread, &fdwrite, &fdexcep, &maxfd);
237 83 : timeout.tv_sec = 0;
238 83 : timeout.tv_usec = 100000;
239 83 : select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
240 98 : while (curl_multi_perform(curl_multi, &still_running) == CURLM_CALL_MULTI_PERFORM);
241 : }
242 :
243 2 : if (conn_i != nRequestCount) { // something gone really really wrong
244 0 : CPLError(CE_Fatal, CPLE_AppDefined, "CPLHTTPFetchMulti(): conn_i != nRequestCount, this should never happen ...");
245 : }
246 5 : for (i = 0; i < nRequestCount; ++i) {
247 3 : CPLHTTPRequest *const psRequest = &pasRequest[i];
248 :
249 3 : long response_code = 0;
250 3 : curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_RESPONSE_CODE, &response_code);
251 3 : psRequest->nStatus = response_code;
252 :
253 3 : char *content_type = 0;
254 3 : curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_CONTENT_TYPE, &content_type);
255 3 : if (content_type) psRequest->pszContentType = CPLStrdup(content_type);
256 :
257 3 : if ((psRequest->pszError == NULL) && (psRequest->m_curl_error != NULL) && (psRequest->m_curl_error[0] != '\0')) {
258 0 : psRequest->pszError = CPLStrdup(psRequest->m_curl_error);
259 : }
260 :
261 : /* In the case of a file:// URL, curl will return a status == 0, so if there's no */
262 : /* error returned, patch the status code to be 200, as it would be for http:// */
263 3 : if (strncmp(psRequest->pszURL, "file://", 7) == 0 && psRequest->nStatus == 0 &&
264 : psRequest->pszError == NULL)
265 : {
266 0 : psRequest->nStatus = 200;
267 : }
268 :
269 : CPLDebug("HTTP", "Request [%d] %s : status = %d, content type = %s, error = %s",
270 : i, psRequest->pszURL, psRequest->nStatus,
271 : (psRequest->pszContentType) ? psRequest->pszContentType : "(null)",
272 3 : (psRequest->pszError) ? psRequest->pszError : "(null)");
273 :
274 3 : curl_multi_remove_handle(curl_multi, pasRequest[i].m_curl_handle);
275 : }
276 2 : curl_multi_cleanup(curl_multi);
277 :
278 2 : return ret;
279 : }
|
