Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: libcurl based HTTP client
4 : * Purpose: libcurl based HTTP client
5 : * Author: Frank Warmerdam, warmerdam@pobox.com
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2006, Frank Warmerdam
9 : * Copyright (c) 2008-2013, Even Rouault <even dot rouault at spatialys.com>
10 : *
11 : * SPDX-License-Identifier: MIT
12 : ****************************************************************************/
13 :
14 : #include "cpl_port.h"
15 : #include "cpl_http.h"
16 :
17 : #include <cstddef>
18 : #include <cstring>
19 :
20 : #include <algorithm>
21 : #include <array>
22 : #include <map>
23 : #include <mutex>
24 : #include <string>
25 : #include <vector>
26 :
27 : #include "cpl_http.h"
28 : #include "cpl_error.h"
29 : #include "cpl_multiproc.h"
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_vsil_curl_class.h"
32 :
33 : // gcc or clang complains about C-style cast in #define like
34 : // CURL_ZERO_TERMINATED
35 : #if defined(__GNUC__)
36 : #pragma GCC diagnostic push
37 : #pragma GCC diagnostic ignored "-Wold-style-cast"
38 : #endif
39 :
40 : #ifdef HAVE_CURL
41 :
42 : #include "cpl_curl_priv.h"
43 :
44 : #ifdef HAVE_OPENSSL_CRYPTO
45 : #include <openssl/err.h>
46 : #include <openssl/ssl.h>
47 : #include <openssl/x509v3.h>
48 :
49 : #if defined(_WIN32)
50 : #include <wincrypt.h>
51 : #endif
52 :
53 : #endif
54 :
55 : #ifdef HAVE_SIGACTION
56 : #include <signal.h>
57 : #endif
58 :
59 : #define unchecked_curl_easy_setopt(handle, opt, param) \
60 : CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param))
61 :
62 : #endif // HAVE_CURL
63 :
64 : // list of named persistent http sessions
65 :
66 : #ifdef HAVE_CURL
67 : static std::map<CPLString, CURL *> *poSessionMap = nullptr;
68 : static std::map<CPLString, CURLM *> *poSessionMultiMap = nullptr;
69 : static CPLMutex *hSessionMapMutex = nullptr;
70 : static bool bHasCheckVersion = false;
71 : static bool bSupportGZip = false;
72 : static bool bSupportHTTP2 = false;
73 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
74 : static std::vector<X509 *> *poWindowsCertificateList = nullptr;
75 :
76 : #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
77 : #define EVP_PKEY_get0_RSA(x) (x->pkey.rsa)
78 : #define EVP_PKEY_get0_DSA(x) (x->pkey.dsa)
79 : #define X509_get_extension_flags(x) (x->ex_flags)
80 : #define X509_get_key_usage(x) (x->ex_kusage)
81 : #define X509_get_extended_key_usage(x) (x->ex_xkusage)
82 : #endif
83 :
84 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
85 :
86 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
87 :
88 : // Ported from https://curl.haxx.se/libcurl/c/opensslthreadlock.html
89 : static CPLMutex **pahSSLMutex = nullptr;
90 :
91 : static void CPLOpenSSLLockingFunction(int mode, int n, const char * /*file*/,
92 : int /*line*/)
93 : {
94 : if (mode & CRYPTO_LOCK)
95 : {
96 : CPLAcquireMutex(pahSSLMutex[n], 3600.0);
97 : }
98 : else
99 : {
100 : CPLReleaseMutex(pahSSLMutex[n]);
101 : }
102 : }
103 :
104 : static unsigned long CPLOpenSSLIdCallback(void)
105 : {
106 : return static_cast<unsigned long>(CPLGetPID());
107 : }
108 :
109 : static void CPLOpenSSLInit()
110 : {
111 : if (strstr(curl_version(), "OpenSSL") &&
112 : CPLTestBool(CPLGetConfigOption("CPL_OPENSSL_INIT_ENABLED", "YES")) &&
113 : CRYPTO_get_id_callback() == nullptr)
114 : {
115 : pahSSLMutex = static_cast<CPLMutex **>(
116 : CPLMalloc(CRYPTO_num_locks() * sizeof(CPLMutex *)));
117 : for (int i = 0; i < CRYPTO_num_locks(); i++)
118 : {
119 : pahSSLMutex[i] = CPLCreateMutex();
120 : CPLReleaseMutex(pahSSLMutex[i]);
121 : }
122 : CRYPTO_set_id_callback(CPLOpenSSLIdCallback);
123 : CRYPTO_set_locking_callback(CPLOpenSSLLockingFunction);
124 : }
125 : }
126 :
127 : static void CPLOpenSSLCleanup()
128 : {
129 : if (pahSSLMutex)
130 : {
131 : for (int i = 0; i < CRYPTO_num_locks(); i++)
132 : {
133 : CPLDestroyMutex(pahSSLMutex[i]);
134 : }
135 : CPLFree(pahSSLMutex);
136 : pahSSLMutex = nullptr;
137 : CRYPTO_set_id_callback(nullptr);
138 : CRYPTO_set_locking_callback(nullptr);
139 : }
140 : }
141 :
142 : #endif
143 :
144 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
145 :
146 : /************************************************************************/
147 : /* CPLWindowsCertificateListCleanup() */
148 : /************************************************************************/
149 :
150 : static void CPLWindowsCertificateListCleanup()
151 : {
152 : if (poWindowsCertificateList)
153 : {
154 : for (auto &&pX509 : *poWindowsCertificateList)
155 : {
156 : X509_free(pX509);
157 : }
158 : delete poWindowsCertificateList;
159 : poWindowsCertificateList = nullptr;
160 : }
161 : }
162 :
163 : /************************************************************************/
164 : /* LoadCAPICertificates() */
165 : /************************************************************************/
166 :
167 : static CPLErr LoadCAPICertificates(const char *pszName,
168 : std::vector<X509 *> *poCertificateList)
169 : {
170 : CPLAssert(pszName);
171 : CPLAssert(poCertificateList);
172 :
173 : HCERTSTORE pCertStore = CertOpenSystemStore(
174 : reinterpret_cast<HCRYPTPROV_LEGACY>(nullptr), pszName);
175 : if (pCertStore == nullptr)
176 : {
177 : CPLError(CE_Failure, CPLE_AppDefined,
178 : "CPLLoadCAPICertificates(): Unable open system "
179 : "certificate store %s.",
180 : pszName);
181 : return CE_Failure;
182 : }
183 :
184 : PCCERT_CONTEXT pCertificate =
185 : CertEnumCertificatesInStore(pCertStore, nullptr);
186 : while (pCertificate != nullptr)
187 : {
188 : X509 *pX509 = d2i_X509(
189 : nullptr,
190 : const_cast<unsigned char const **>(&pCertificate->pbCertEncoded),
191 : pCertificate->cbCertEncoded);
192 : if (pX509 == nullptr)
193 : {
194 : CPLError(CE_Warning, CPLE_AppDefined,
195 : "CPLLoadCAPICertificates(): CertEnumCertificatesInStore() "
196 : "returned a null certificate, skipping.");
197 : }
198 : else
199 : {
200 : #ifdef DEBUG_VERBOSE
201 : char szSubject[256] = {0};
202 : CPLString osSubject;
203 : X509_NAME *pName = X509_get_subject_name(pX509);
204 : if (pName)
205 : {
206 : X509_NAME_oneline(pName, szSubject, sizeof(szSubject));
207 : osSubject = szSubject;
208 : }
209 : if (!osSubject.empty())
210 : CPLDebug("HTTP", "SSL Certificate: %s", osSubject.c_str());
211 : #endif
212 : poCertificateList->push_back(pX509);
213 : }
214 : pCertificate = CertEnumCertificatesInStore(pCertStore, pCertificate);
215 : }
216 : CertCloseStore(pCertStore, 0);
217 : return CE_None;
218 : }
219 :
220 : /************************************************************************/
221 : /* CPL_ssl_ctx_callback() */
222 : /************************************************************************/
223 :
224 : // Load certificates from Windows Crypto API store.
225 : static CURLcode CPL_ssl_ctx_callback(CURL *, void *pSSL, void *)
226 : {
227 : SSL_CTX *pSSL_CTX = static_cast<SSL_CTX *>(pSSL);
228 : if (pSSL_CTX == nullptr)
229 : {
230 : CPLError(CE_Failure, CPLE_AppDefined,
231 : "CPL_ssl_ctx_callback(): OpenSSL context pointer is NULL.");
232 : return CURLE_ABORTED_BY_CALLBACK;
233 : }
234 :
235 : static std::mutex goMutex;
236 : {
237 : std::lock_guard<std::mutex> oLock(goMutex);
238 : if (poWindowsCertificateList == nullptr)
239 : {
240 : poWindowsCertificateList = new std::vector<X509 *>();
241 : if (!poWindowsCertificateList)
242 : {
243 : CPLError(CE_Failure, CPLE_AppDefined,
244 : "CPL_ssl_ctx_callback(): Unable to allocate "
245 : "structure to hold certificates.");
246 : return CURLE_FAILED_INIT;
247 : }
248 :
249 : const std::array<const char *, 3> aszStores{
250 : {"CA", "AuthRoot", "ROOT"}};
251 : for (auto &&pszStore : aszStores)
252 : {
253 : if (LoadCAPICertificates(pszStore, poWindowsCertificateList) ==
254 : CE_Failure)
255 : {
256 : CPLError(
257 : CE_Failure, CPLE_AppDefined,
258 : "CPL_ssl_ctx_callback(): Unable to load certificates "
259 : "from '%s' store.",
260 : pszStore);
261 : return CURLE_FAILED_INIT;
262 : }
263 : }
264 :
265 : CPLDebug("HTTP", "Loading %d certificates from Windows store.",
266 : static_cast<int>(poWindowsCertificateList->size()));
267 : }
268 : }
269 :
270 : X509_STORE *pX509Store = SSL_CTX_get_cert_store(pSSL_CTX);
271 : for (X509 *x509 : *poWindowsCertificateList)
272 : X509_STORE_add_cert(pX509Store, x509);
273 :
274 : return CURLE_OK;
275 : }
276 :
277 : #endif // defined(_WIN32) && defined (HAVE_OPENSSL_CRYPTO)
278 :
279 : /************************************************************************/
280 : /* CheckCurlFeatures() */
281 : /************************************************************************/
282 :
283 2549 : static void CheckCurlFeatures()
284 : {
285 5098 : CPLMutexHolder oHolder(&hSessionMapMutex);
286 2549 : if (!bHasCheckVersion)
287 : {
288 6 : const char *pszVersion = curl_version();
289 6 : CPLDebug("HTTP", "%s", pszVersion);
290 6 : bSupportGZip = strstr(pszVersion, "zlib/") != nullptr;
291 6 : bSupportHTTP2 = strstr(curl_version(), "nghttp2/") != nullptr;
292 6 : bHasCheckVersion = true;
293 :
294 6 : curl_version_info_data *data = curl_version_info(CURLVERSION_NOW);
295 6 : if (data->version_num < LIBCURL_VERSION_NUM)
296 : {
297 0 : CPLError(CE_Warning, CPLE_AppDefined,
298 : "GDAL was built against curl %d.%d.%d, but is "
299 : "running against %s. Runtime failure is likely !",
300 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
301 : LIBCURL_VERSION_PATCH, data->version);
302 : }
303 6 : else if (data->version_num > LIBCURL_VERSION_NUM)
304 : {
305 0 : CPLDebug("HTTP",
306 : "GDAL was built against curl %d.%d.%d, but is "
307 : "running against %s.",
308 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
309 : LIBCURL_VERSION_PATCH, data->version);
310 : }
311 :
312 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
313 : CPLOpenSSLInit();
314 : #endif
315 : }
316 2549 : }
317 :
318 : /************************************************************************/
319 : /* CPLWriteFct() */
320 : /* */
321 : /* Append incoming text to our collection buffer, reallocating */
322 : /* it larger as needed. */
323 : /************************************************************************/
324 :
325 : class CPLHTTPResultWithLimit
326 : {
327 : public:
328 : CPLHTTPResult *psResult = nullptr;
329 : int nMaxFileSize = 0;
330 : };
331 :
332 12282 : static size_t CPLWriteFct(void *buffer, size_t size, size_t nmemb,
333 : void *reqInfo)
334 :
335 : {
336 12282 : CPLHTTPResultWithLimit *psResultWithLimit =
337 : static_cast<CPLHTTPResultWithLimit *>(reqInfo);
338 12282 : CPLHTTPResult *psResult = psResultWithLimit->psResult;
339 :
340 12282 : int nBytesToWrite = static_cast<int>(nmemb) * static_cast<int>(size);
341 12282 : int nNewSize = psResult->nDataLen + nBytesToWrite + 1;
342 12282 : if (nNewSize > psResult->nDataAlloc)
343 : {
344 1604 : psResult->nDataAlloc = static_cast<int>(nNewSize * 1.25 + 100);
345 : GByte *pabyNewData = static_cast<GByte *>(
346 1604 : VSIRealloc(psResult->pabyData, psResult->nDataAlloc));
347 1604 : if (pabyNewData == nullptr)
348 : {
349 0 : VSIFree(psResult->pabyData);
350 0 : psResult->pabyData = nullptr;
351 0 : psResult->pszErrBuf = CPLStrdup(CPLString().Printf(
352 : "Out of memory allocating %d bytes for HTTP data buffer.",
353 0 : psResult->nDataAlloc));
354 0 : psResult->nDataAlloc = psResult->nDataLen = 0;
355 :
356 0 : return 0;
357 : }
358 1604 : psResult->pabyData = pabyNewData;
359 : }
360 :
361 12282 : memcpy(psResult->pabyData + psResult->nDataLen, buffer, nBytesToWrite);
362 :
363 12282 : psResult->nDataLen += nBytesToWrite;
364 12282 : psResult->pabyData[psResult->nDataLen] = 0;
365 :
366 12282 : if (psResultWithLimit->nMaxFileSize > 0 &&
367 8 : psResult->nDataLen > psResultWithLimit->nMaxFileSize)
368 : {
369 1 : CPLError(CE_Failure, CPLE_AppDefined, "Maximum file size reached");
370 1 : return 0;
371 : }
372 :
373 12281 : return nmemb;
374 : }
375 :
376 : /************************************************************************/
377 : /* CPLHdrWriteFct() */
378 : /************************************************************************/
379 7381 : static size_t CPLHdrWriteFct(void *buffer, size_t size, size_t nmemb,
380 : void *reqInfo)
381 : {
382 7381 : CPLHTTPResult *psResult = static_cast<CPLHTTPResult *>(reqInfo);
383 : // Copy the buffer to a char* and initialize with zeros (zero
384 : // terminate as well).
385 7381 : size_t nBytes = size * nmemb;
386 7381 : char *pszHdr = static_cast<char *>(CPLCalloc(1, nBytes + 1));
387 7381 : memcpy(pszHdr, buffer, nBytes);
388 7381 : size_t nIdx = nBytes - 1;
389 : // Remove end of line characters
390 20766 : while (nIdx > 0 && (pszHdr[nIdx] == '\r' || pszHdr[nIdx] == '\n'))
391 : {
392 13385 : pszHdr[nIdx] = 0;
393 13385 : nIdx--;
394 : }
395 7381 : char *pszKey = nullptr;
396 7381 : const char *pszValue = CPLParseNameValue(pszHdr, &pszKey);
397 7381 : if (pszKey && pszValue)
398 : {
399 5107 : psResult->papszHeaders =
400 5107 : CSLAddNameValue(psResult->papszHeaders, pszKey, pszValue);
401 : }
402 7381 : CPLFree(pszHdr);
403 7381 : CPLFree(pszKey);
404 7381 : return nmemb;
405 : }
406 :
407 : /************************************************************************/
408 : /* CPLHTTPReadFunction() */
409 : /************************************************************************/
410 0 : static size_t CPLHTTPReadFunction(char *buffer, size_t size, size_t nitems,
411 : void *arg)
412 : {
413 0 : return VSIFReadL(buffer, size, nitems, static_cast<VSILFILE *>(arg));
414 : }
415 :
416 : /************************************************************************/
417 : /* CPLHTTPSeekFunction() */
418 : /************************************************************************/
419 0 : static int CPLHTTPSeekFunction(void *arg, curl_off_t offset, int origin)
420 : {
421 0 : if (VSIFSeekL(static_cast<VSILFILE *>(arg), offset, origin) == 0)
422 0 : return CURL_SEEKFUNC_OK;
423 : else
424 0 : return CURL_SEEKFUNC_FAIL;
425 : }
426 :
427 : /************************************************************************/
428 : /* CPLHTTPFreeFunction() */
429 : /************************************************************************/
430 0 : static void CPLHTTPFreeFunction(void *arg)
431 : {
432 0 : VSIFCloseL(static_cast<VSILFILE *>(arg));
433 0 : }
434 :
435 : typedef struct
436 : {
437 : GDALProgressFunc pfnProgress;
438 : void *pProgressArg;
439 : } CurlProcessData, *CurlProcessDataL;
440 :
441 356 : static int NewProcessFunction(void *p, curl_off_t dltotal, curl_off_t dlnow,
442 : curl_off_t ultotal, curl_off_t ulnow)
443 : {
444 356 : CurlProcessDataL pData = static_cast<CurlProcessDataL>(p);
445 356 : if (nullptr != pData && pData->pfnProgress)
446 : {
447 356 : if (dltotal > 0)
448 : {
449 0 : const double dfDone = double(dlnow) / dltotal;
450 0 : return pData->pfnProgress(dfDone, "Downloading ...",
451 : pData->pProgressArg) == TRUE
452 0 : ? 0
453 0 : : 1;
454 : }
455 356 : else if (ultotal > 0)
456 : {
457 0 : const double dfDone = double(ulnow) / ultotal;
458 0 : return pData->pfnProgress(dfDone, "Uploading ...",
459 : pData->pProgressArg) == TRUE
460 0 : ? 0
461 0 : : 1;
462 : }
463 : }
464 356 : return 0;
465 : }
466 :
467 : #endif /* def HAVE_CURL */
468 :
469 : /************************************************************************/
470 : /* CPLHTTPSetDefaultUserAgent() */
471 : /************************************************************************/
472 :
473 : static std::string gosDefaultUserAgent;
474 :
475 : /**
476 : * \brief Set the default user agent.
477 : *
478 : * GDAL core will by default call this method with "GDAL/x.y.z" where x.y.z
479 : * is the GDAL version number (during driver initialization). Applications may
480 : * override it.
481 : *
482 : * @param pszUserAgent String (or nullptr to cancel the default user agent)
483 : *
484 : * @since GDAL 3.7
485 : */
486 1383 : void CPLHTTPSetDefaultUserAgent(const char *pszUserAgent)
487 : {
488 1383 : gosDefaultUserAgent = pszUserAgent ? pszUserAgent : "";
489 1383 : }
490 :
491 : /************************************************************************/
492 : /* CPLHTTPGetOptionsFromEnv() */
493 : /************************************************************************/
494 :
495 : typedef struct
496 : {
497 : const char *pszEnvVar;
498 : const char *pszOptionName;
499 : } TupleEnvVarOptionName;
500 :
501 : constexpr TupleEnvVarOptionName asAssocEnvVarOptionName[] = {
502 : {"GDAL_HTTP_VERSION", "HTTP_VERSION"},
503 : {"GDAL_HTTP_CONNECTTIMEOUT", "CONNECTTIMEOUT"},
504 : {"GDAL_HTTP_TIMEOUT", "TIMEOUT"},
505 : {"GDAL_HTTP_LOW_SPEED_TIME", "LOW_SPEED_TIME"},
506 : {"GDAL_HTTP_LOW_SPEED_LIMIT", "LOW_SPEED_LIMIT"},
507 : {"GDAL_HTTP_USERPWD", "USERPWD"},
508 : {"GDAL_HTTP_PROXY", "PROXY"},
509 : {"GDAL_HTTPS_PROXY", "HTTPS_PROXY"},
510 : {"GDAL_HTTP_PROXYUSERPWD", "PROXYUSERPWD"},
511 : {"GDAL_PROXY_AUTH", "PROXYAUTH"},
512 : {"GDAL_HTTP_NETRC", "NETRC"},
513 : {"GDAL_HTTP_NETRC_FILE", "NETRC_FILE"},
514 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
515 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
516 : {"GDAL_HTTP_RETRY_CODES", "RETRY_CODES"},
517 : {"GDAL_CURL_CA_BUNDLE", "CAINFO"},
518 : {"CURL_CA_BUNDLE", "CAINFO"},
519 : {"SSL_CERT_FILE", "CAINFO"},
520 : {"GDAL_HTTP_CAPATH", "CAPATH"},
521 : {"GDAL_HTTP_SSL_VERIFYSTATUS", "SSL_VERIFYSTATUS"},
522 : {"GDAL_HTTP_USE_CAPI_STORE", "USE_CAPI_STORE"},
523 : {"GDAL_HTTP_HEADERS", "HEADERS"},
524 : {"GDAL_HTTP_HEADER_FILE", "HEADER_FILE"},
525 : {"GDAL_HTTP_AUTH", "HTTPAUTH"},
526 : {"GDAL_GSSAPI_DELEGATION", "GSSAPI_DELEGATION"},
527 : {"GDAL_HTTP_BEARER", "HTTP_BEARER"},
528 : {"GDAL_HTTP_COOKIE", "COOKIE"},
529 : {"GDAL_HTTP_COOKIEFILE", "COOKIEFILE"},
530 : {"GDAL_HTTP_COOKIEJAR", "COOKIEJAR"},
531 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
532 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
533 : {"GDAL_HTTP_TCP_KEEPALIVE", "TCP_KEEPALIVE"},
534 : {"GDAL_HTTP_TCP_KEEPIDLE", "TCP_KEEPIDLE"},
535 : {"GDAL_HTTP_TCP_KEEPINTVL", "TCP_KEEPINTVL"},
536 : };
537 :
538 1484 : char **CPLHTTPGetOptionsFromEnv(const char *pszFilename)
539 : {
540 2968 : CPLStringList aosOptions;
541 2968 : std::string osNonStreamingFilename;
542 1484 : if (pszFilename && STARTS_WITH(pszFilename, "/vsi"))
543 : {
544 : VSIFilesystemHandler *poFSHandler =
545 1482 : VSIFileManager::GetHandler(pszFilename);
546 : osNonStreamingFilename =
547 1482 : poFSHandler->GetNonStreamingFilename(pszFilename);
548 1482 : if (osNonStreamingFilename == pszFilename)
549 : {
550 1418 : osNonStreamingFilename.clear();
551 : }
552 : else
553 : {
554 : // CPLDebug("HTTP", "Non-streaming filename for %s: %s", pszFilename, osNonStreamingFilename.c_str());
555 : }
556 : }
557 51940 : for (const auto &sTuple : asAssocEnvVarOptionName)
558 : {
559 50456 : const char *pszVal = nullptr;
560 50456 : if (pszFilename)
561 : {
562 50456 : pszVal = VSIGetPathSpecificOption(pszFilename, sTuple.pszEnvVar,
563 : nullptr);
564 50456 : if (!pszVal && !osNonStreamingFilename.empty())
565 : {
566 2164 : pszVal = VSIGetPathSpecificOption(
567 2164 : osNonStreamingFilename.c_str(), sTuple.pszEnvVar, nullptr);
568 : }
569 : }
570 50456 : if (!pszVal)
571 : {
572 50298 : pszVal = CPLGetConfigOption(sTuple.pszEnvVar, nullptr);
573 : }
574 50456 : if (pszVal)
575 : {
576 158 : aosOptions.AddNameValue(sTuple.pszOptionName, pszVal);
577 : }
578 : }
579 2968 : return aosOptions.StealList();
580 : }
581 :
582 : /************************************************************************/
583 : /* CPLHTTPGetNewRetryDelay() */
584 : /************************************************************************/
585 :
586 : /** Return the new retry delay.
587 : *
588 : * This takes into account the HTTP response code, the previous delay, the
589 : * HTTP payload error message, the Curl error message and a potential list of
590 : * retriable HTTP codes.
591 : *
592 : * @param response_code HTTP response code (e.g. 400)
593 : * @param dfOldDelay Previous delay (nominally in second)
594 : * @param pszErrBuf HTTP response body of the failed request (may be NULL)
595 : * @param pszCurlError Curl error as returned by CURLOPT_ERRORBUFFER (may be NULL)
596 : * @param pszRetriableCodes nullptr to limit to the default hard-coded scenarios,
597 : * "ALL" to ask to retry for all non-200 codes, or a comma-separated list of
598 : * HTTP codes (e.g. "400,500") that are accepted for retry.
599 : * @return the new delay, or 0 if no retry should be attempted.
600 : */
601 215 : static double CPLHTTPGetNewRetryDelay(int response_code, double dfOldDelay,
602 : const char *pszErrBuf,
603 : const char *pszCurlError,
604 : const char *pszRetriableCodes)
605 : {
606 215 : bool bRetry = false;
607 215 : if (pszRetriableCodes && pszRetriableCodes[0])
608 : {
609 5 : bRetry = EQUAL(pszRetriableCodes, "ALL") ||
610 2 : strstr(pszRetriableCodes, CPLSPrintf("%d", response_code));
611 : }
612 212 : else if (response_code == 429 || response_code == 500 ||
613 208 : (response_code >= 502 && response_code <= 504) ||
614 : // S3 sends some client timeout errors as 400 Client Error
615 6 : (response_code == 400 && pszErrBuf &&
616 193 : strstr(pszErrBuf, "RequestTimeout")) ||
617 193 : (pszCurlError &&
618 193 : (strstr(pszCurlError, "Connection timed out") ||
619 193 : strstr(pszCurlError, "Operation timed out") ||
620 193 : strstr(pszCurlError, "Connection reset by peer") ||
621 193 : strstr(pszCurlError, "Connection was reset"))))
622 : {
623 23 : bRetry = true;
624 : }
625 215 : if (bRetry)
626 : {
627 : // 'Operation timed out': seen during some long running operation 'hang'
628 : // no error but no response from server and we are in the cURL loop
629 : // infinitely.
630 :
631 : // 'Connection was reset': was found with Azure: server resets
632 : // connection during TLS handshake (10054 error code). It seems like
633 : // the server process crashed or something forced TCP reset;
634 : // the request succeeds on retry.
635 :
636 : // Use an exponential backoff factor of 2 plus some random jitter
637 : // We don't care about cryptographic quality randomness, hence:
638 : // coverity[dont_call]
639 25 : return dfOldDelay * (2 + rand() * 0.5 / RAND_MAX);
640 : }
641 : else
642 : {
643 190 : return 0;
644 : }
645 : }
646 :
647 : /*! @cond Doxygen_Suppress */
648 :
649 : /************************************************************************/
650 : /* CPLHTTPRetryParameters() */
651 : /************************************************************************/
652 :
653 : /** Constructs a CPLHTTPRetryParameters instance from configuration
654 : * options or path-specific options.
655 : *
656 : * @param aosHTTPOptions HTTP options returned by CPLHTTPGetOptionsFromEnv()
657 : */
658 1279 : CPLHTTPRetryParameters::CPLHTTPRetryParameters(
659 1279 : const CPLStringList &aosHTTPOptions)
660 1279 : : nMaxRetry(atoi(aosHTTPOptions.FetchNameValueDef(
661 : "MAX_RETRY", CPLSPrintf("%d", CPL_HTTP_MAX_RETRY)))),
662 1279 : dfInitialDelay(CPLAtof(aosHTTPOptions.FetchNameValueDef(
663 : "RETRY_DELAY", CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)))),
664 2558 : osRetryCodes(aosHTTPOptions.FetchNameValueDef("RETRY_CODES", ""))
665 : {
666 1279 : }
667 :
668 : /************************************************************************/
669 : /* CPLHTTPRetryContext() */
670 : /************************************************************************/
671 :
672 : /** Constructor */
673 1079 : CPLHTTPRetryContext::CPLHTTPRetryContext(const CPLHTTPRetryParameters &oParams)
674 1079 : : m_oParameters(oParams), m_dfNextDelay(oParams.dfInitialDelay)
675 : {
676 1079 : }
677 :
678 : /************************************************************************/
679 : /* CPLHTTPRetryContext::CanRetry() */
680 : /************************************************************************/
681 :
682 : /** Returns whether we can attempt a new retry, based on the retry counter,
683 : * and increment that counter.
684 : */
685 0 : bool CPLHTTPRetryContext::CanRetry()
686 : {
687 0 : if (m_nRetryCount >= m_oParameters.nMaxRetry)
688 0 : return false;
689 0 : m_nRetryCount++;
690 0 : return true;
691 : }
692 :
693 : /** Returns whether we can attempt a new retry, based on the retry counter,
694 : * the response code, payload and curl error buffers.
695 : *
696 : * If successful, the retry counter is incremented, and GetCurrentDelay()
697 : * returns the delay to apply with CPLSleep().
698 : */
699 245 : bool CPLHTTPRetryContext::CanRetry(int response_code, const char *pszErrBuf,
700 : const char *pszCurlError)
701 : {
702 245 : if (m_nRetryCount >= m_oParameters.nMaxRetry)
703 227 : return false;
704 18 : m_dfCurDelay = m_dfNextDelay;
705 18 : m_dfNextDelay = CPLHTTPGetNewRetryDelay(response_code, m_dfNextDelay,
706 : pszErrBuf, pszCurlError,
707 : m_oParameters.osRetryCodes.c_str());
708 18 : if (m_dfNextDelay == 0.0)
709 2 : return false;
710 16 : m_nRetryCount++;
711 16 : return true;
712 : }
713 :
714 : /************************************************************************/
715 : /* CPLHTTPRetryContext::GetCurrentDelay() */
716 : /************************************************************************/
717 :
718 : /** Returns the delay to apply. Only valid after a successful call to CanRetry() */
719 32 : double CPLHTTPRetryContext::GetCurrentDelay() const
720 : {
721 32 : if (m_nRetryCount == 0)
722 0 : CPLDebug("CPL",
723 : "GetCurrentDelay() should only be called after CanRetry()");
724 32 : return m_dfCurDelay;
725 : }
726 :
727 : /*! @endcond Doxygen_Suppress */
728 :
729 : #ifdef HAVE_CURL
730 :
731 : /************************************************************************/
732 : /* CPLHTTPEmitFetchDebug() */
733 : /************************************************************************/
734 :
735 1148 : static void CPLHTTPEmitFetchDebug(const char *pszURL,
736 : const char *pszExtraDebug = "")
737 : {
738 1148 : const char *pszArobase = strchr(pszURL, '@');
739 1148 : const char *pszSlash = strchr(pszURL, '/');
740 1148 : const char *pszColon = (pszSlash) ? strchr(pszSlash, ':') : nullptr;
741 1148 : if (pszArobase != nullptr && pszColon != nullptr &&
742 0 : pszArobase - pszColon > 0)
743 : {
744 : /* http://user:password@www.example.com */
745 0 : char *pszSanitizedURL = CPLStrdup(pszURL);
746 0 : pszSanitizedURL[pszColon - pszURL] = 0;
747 0 : CPLDebug("HTTP", "Fetch(%s:#password#%s%s)", pszSanitizedURL,
748 : pszArobase, pszExtraDebug);
749 0 : CPLFree(pszSanitizedURL);
750 : }
751 : else
752 : {
753 1148 : CPLDebug("HTTP", "Fetch(%s%s)", pszURL, pszExtraDebug);
754 : }
755 1148 : }
756 :
757 : #endif
758 :
759 : #ifdef HAVE_CURL
760 :
761 : /************************************************************************/
762 : /* class CPLHTTPPostFields */
763 : /************************************************************************/
764 :
765 : class CPLHTTPPostFields
766 : {
767 : public:
768 1148 : CPLHTTPPostFields() = default;
769 : CPLHTTPPostFields &operator=(const CPLHTTPPostFields &) = delete;
770 : CPLHTTPPostFields(const CPLHTTPPostFields &) = delete;
771 :
772 1148 : CPLErr Fill(CURL *http_handle, CSLConstList papszOptions)
773 : {
774 : // Fill POST form if present
775 : const char *pszFormFilePath =
776 1148 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
777 : const char *pszParametersCount =
778 1148 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
779 :
780 1148 : if (pszFormFilePath != nullptr || pszParametersCount != nullptr)
781 : {
782 2 : mime = curl_mime_init(http_handle);
783 2 : curl_mimepart *mimepart = curl_mime_addpart(mime);
784 2 : if (pszFormFilePath != nullptr)
785 : {
786 : const char *pszFormFileName =
787 1 : CSLFetchNameValue(papszOptions, "FORM_FILE_NAME");
788 1 : const char *pszFilename = CPLGetFilename(pszFormFilePath);
789 1 : if (pszFormFileName == nullptr)
790 : {
791 1 : pszFormFileName = pszFilename;
792 : }
793 :
794 : VSIStatBufL sStat;
795 1 : if (VSIStatL(pszFormFilePath, &sStat) == 0)
796 : {
797 0 : VSILFILE *mime_fp = VSIFOpenL(pszFormFilePath, "rb");
798 0 : if (mime_fp != nullptr)
799 : {
800 0 : curl_mime_name(mimepart, pszFormFileName);
801 0 : CPL_IGNORE_RET_VAL(
802 0 : curl_mime_filename(mimepart, pszFilename));
803 0 : curl_mime_data_cb(
804 : mimepart, sStat.st_size, CPLHTTPReadFunction,
805 : CPLHTTPSeekFunction, CPLHTTPFreeFunction, mime_fp);
806 : }
807 : else
808 : {
809 : osErrMsg = CPLSPrintf("Failed to open file %s",
810 0 : pszFormFilePath);
811 1 : return CE_Failure;
812 : }
813 :
814 0 : CPLDebug("HTTP", "Send file: %s, COPYNAME: %s",
815 : pszFormFilePath, pszFormFileName);
816 : }
817 : else
818 : {
819 : osErrMsg =
820 1 : CPLSPrintf("File '%s' not found", pszFormFilePath);
821 1 : return CE_Failure;
822 : }
823 : }
824 :
825 1 : int nParametersCount = 0;
826 1 : if (pszParametersCount != nullptr)
827 : {
828 1 : nParametersCount = atoi(pszParametersCount);
829 : }
830 :
831 2 : for (int i = 0; i < nParametersCount; ++i)
832 : {
833 2 : const char *pszKey = CSLFetchNameValue(
834 : papszOptions, CPLSPrintf("FORM_KEY_%d", i));
835 2 : const char *pszValue = CSLFetchNameValue(
836 : papszOptions, CPLSPrintf("FORM_VALUE_%d", i));
837 :
838 2 : if (nullptr == pszKey)
839 : {
840 : osErrMsg = CPLSPrintf("Key #%d is not exists. Maybe wrong "
841 : "count of form items",
842 1 : i);
843 1 : return CE_Failure;
844 : }
845 :
846 1 : if (nullptr == pszValue)
847 : {
848 : osErrMsg = CPLSPrintf("Value #%d is not exists. Maybe "
849 : "wrong count of form items",
850 0 : i);
851 0 : return CE_Failure;
852 : }
853 :
854 1 : mimepart = curl_mime_addpart(mime);
855 1 : curl_mime_name(mimepart, pszKey);
856 1 : CPL_IGNORE_RET_VAL(
857 1 : curl_mime_data(mimepart, pszValue, CURL_ZERO_TERMINATED));
858 :
859 1 : CPLDebug("HTTP", "COPYNAME: %s, COPYCONTENTS: %s", pszKey,
860 : pszValue);
861 : }
862 :
863 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MIMEPOST, mime);
864 : }
865 1146 : return CE_None;
866 : }
867 :
868 1148 : ~CPLHTTPPostFields()
869 1148 : {
870 1148 : if (mime != nullptr)
871 : {
872 2 : curl_mime_free(mime);
873 : }
874 1148 : }
875 :
876 2 : std::string GetErrorMessage() const
877 : {
878 2 : return osErrMsg;
879 : }
880 :
881 : private:
882 : curl_mime *mime = nullptr;
883 : std::string osErrMsg{};
884 : };
885 :
886 : /************************************************************************/
887 : /* CPLHTTPFetchCleanup() */
888 : /************************************************************************/
889 :
890 1148 : static void CPLHTTPFetchCleanup(CURL *http_handle, struct curl_slist *headers,
891 : const char *pszPersistent,
892 : CSLConstList papszOptions)
893 : {
894 1148 : if (CSLFetchNameValue(papszOptions, "POSTFIELDS"))
895 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 0);
896 1148 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, nullptr);
897 :
898 1148 : if (!pszPersistent)
899 651 : curl_easy_cleanup(http_handle);
900 :
901 1148 : curl_slist_free_all(headers);
902 1148 : }
903 : #endif // HAVE_CURL
904 :
905 : struct CPLHTTPFetchContext
906 : {
907 : std::vector<std::pair<CPLHTTPFetchCallbackFunc, void *>> stack{};
908 : };
909 :
910 : /************************************************************************/
911 : /* GetHTTPFetchContext() */
912 : /************************************************************************/
913 :
914 1340 : static CPLHTTPFetchContext *GetHTTPFetchContext(bool bAlloc)
915 : {
916 1340 : int bError = FALSE;
917 : CPLHTTPFetchContext *psCtx = static_cast<CPLHTTPFetchContext *>(
918 1340 : CPLGetTLSEx(CTLS_HTTPFETCHCALLBACK, &bError));
919 1340 : if (bError)
920 0 : return nullptr;
921 :
922 1340 : if (psCtx == nullptr && bAlloc)
923 : {
924 1 : const auto FreeFunc = [](void *pData)
925 1 : { delete static_cast<CPLHTTPFetchContext *>(pData); };
926 1 : psCtx = new CPLHTTPFetchContext();
927 1 : CPLSetTLSWithFreeFuncEx(CTLS_HTTPFETCHCALLBACK, psCtx, FreeFunc,
928 : &bError);
929 1 : if (bError)
930 : {
931 0 : delete psCtx;
932 0 : psCtx = nullptr;
933 : }
934 : }
935 1340 : return psCtx;
936 : }
937 :
938 : /************************************************************************/
939 : /* CPLHTTPSetFetchCallback() */
940 : /************************************************************************/
941 :
942 : static CPLHTTPFetchCallbackFunc gpsHTTPFetchCallbackFunc = nullptr;
943 : static void *gpHTTPFetchCallbackUserData = nullptr;
944 :
945 : /** Installs an alternate callback to the default implementation of
946 : * CPLHTTPFetchEx().
947 : *
948 : * This callback will be used by all threads, unless contextual callbacks are
949 : * installed with CPLHTTPPushFetchCallback().
950 : *
951 : * It is the responsibility of the caller to make sure this function is not
952 : * called concurrently, or during CPLHTTPFetchEx() execution.
953 : *
954 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called
955 : * (or NULL to restore default handler)
956 : * @param pUserData Last argument to provide to the pFunc callback.
957 : *
958 : * @since GDAL 3.2
959 : */
960 2 : void CPLHTTPSetFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
961 : {
962 2 : gpsHTTPFetchCallbackFunc = pFunc;
963 2 : gpHTTPFetchCallbackUserData = pUserData;
964 2 : }
965 :
966 : /************************************************************************/
967 : /* CPLHTTPPushFetchCallback() */
968 : /************************************************************************/
969 :
970 : /** Installs an alternate callback to the default implementation of
971 : * CPLHTTPFetchEx().
972 : *
973 : * This callback will only be used in the thread where this function has been
974 : * called. It must be un-installed by CPLHTTPPopFetchCallback(), which must also
975 : * be called from the same thread.
976 : *
977 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called.
978 : * @param pUserData Last argument to provide to the pFunc callback.
979 : * @return TRUE in case of success.
980 : *
981 : * @since GDAL 3.2
982 : */
983 1 : int CPLHTTPPushFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
984 : {
985 1 : auto psCtx = GetHTTPFetchContext(true);
986 1 : if (psCtx == nullptr)
987 0 : return false;
988 : psCtx->stack.emplace_back(
989 1 : std::pair<CPLHTTPFetchCallbackFunc, void *>(pFunc, pUserData));
990 1 : return true;
991 : }
992 :
993 : /************************************************************************/
994 : /* CPLHTTPPopFetchCallback() */
995 : /************************************************************************/
996 :
997 : /** Uninstalls a callback set by CPLHTTPPushFetchCallback().
998 : *
999 : * @see CPLHTTPPushFetchCallback()
1000 : * @return TRUE in case of success.
1001 : * @since GDAL 3.2
1002 : */
1003 2 : int CPLHTTPPopFetchCallback(void)
1004 : {
1005 2 : auto psCtx = GetHTTPFetchContext(false);
1006 2 : if (psCtx == nullptr || psCtx->stack.empty())
1007 : {
1008 1 : CPLError(
1009 : CE_Failure, CPLE_AppDefined,
1010 : "CPLHTTPPushFetchCallback / CPLHTTPPopFetchCallback not balanced");
1011 1 : return false;
1012 : }
1013 : else
1014 : {
1015 1 : psCtx->stack.pop_back();
1016 1 : return true;
1017 : }
1018 : }
1019 :
1020 : /************************************************************************/
1021 : /* CPLHTTPFetch() */
1022 : /************************************************************************/
1023 :
1024 : // NOTE: when adding an option below, add it in asAssocEnvVarOptionName[]
1025 :
1026 : // clang-format off
1027 : /**
1028 : * \brief Fetch a document from an url and return in a string.
1029 : *
1030 : * Different options may be passed through the papszOptions function parameter,
1031 : * or for most of them through a configuration option:
1032 : * <ul>
1033 : * <li>CONNECTTIMEOUT=val, where
1034 : * val is in seconds (possibly with decimals). This is the maximum delay for the
1035 : * connection to be established before being aborted (GDAL >= 2.2).
1036 : * Corresponding configuration option: GDAL_HTTP_CONNECTTIMEOUT.
1037 : * </li>
1038 : * <li>TIMEOUT=val, where val is in seconds. This is the maximum delay for the
1039 : * whole request to complete before being aborted.
1040 : * Corresponding configuration option: GDAL_HTTP_TIMEOUT.
1041 : * </li>
1042 : * <li>LOW_SPEED_TIME=val,
1043 : * where val is in seconds. This is the maximum time where the transfer speed
1044 : * should be below the LOW_SPEED_LIMIT (if not specified 1b/s), before the
1045 : * transfer to be considered too slow and aborted. (GDAL >= 2.1).
1046 : * Corresponding configuration option: GDAL_HTTP_LOW_SPEED_TIME.
1047 : * </li>
1048 : * <li>LOW_SPEED_LIMIT=val, where val is in bytes/second. See LOW_SPEED_TIME.
1049 : * Has only effect if LOW_SPEED_TIME is specified too. (GDAL >= 2.1).
1050 : * Corresponding configuration option: GDAL_HTTP_LOW_SPEED_LIMIT.
1051 : * </li>
1052 : * <li>HEADERS=val, where val is an extra header to use when getting a web page.
1053 : * For example "Accept: application/x-ogcwkt"
1054 : * Corresponding configuration option: GDAL_HTTP_HEADERS.
1055 : * Starting with GDAL 3.6, the GDAL_HTTP_HEADERS configuration option can also
1056 : * be used to specify a comma separated list of key: value pairs. This is an
1057 : * alternative to the GDAL_HTTP_HEADER_FILE mechanism. If a comma or a
1058 : * double-quote character is needed in the value, then the key: value pair must
1059 : * be enclosed in double-quote characters. In that situation, backslash and
1060 : * double quote character must be backslash-escaped. e.g GDAL_HTTP_HEADERS=Foo:
1061 : * Bar,"Baz: escaped backslash \\, escaped double-quote \", end of
1062 : * value",Another: Header
1063 : * </li>
1064 : * <li>HEADER_FILE=filename: filename of a text file with "key: value" headers.
1065 : * The content of the file is not cached, and thus it is read again before
1066 : * issuing each HTTP request. (GDAL >= 2.2)
1067 : * Corresponding configuration option: GDAL_HTTP_HEADER_FILE.
1068 : * </li>
1069 : * <li>HTTPAUTH=[BASIC/NTLM/NEGOTIATE/ANY/ANYSAFE/BEARER] to specify an
1070 : * authentication scheme to use.
1071 : * Corresponding configuration option: GDAL_HTTP_AUTH.
1072 : * </li>
1073 : * <li>USERPWD=userid:password to specify a user and password for
1074 : * authentication.
1075 : * Corresponding configuration option: GDAL_HTTP_USERPWD.
1076 : * </li>
1077 : * <li>GSSAPI_DELEGATION=[NONE/POLICY/ALWAYS] set allowed
1078 : * GSS-API delegation. Relevant only with HTTPAUTH=NEGOTIATE (GDAL >= 3.3).
1079 : * Corresponding configuration option: GDAL_GSSAPI_DELEGATION (note: no "HTTP_" in the name)
1080 : * </li>
1081 : * <li>HTTP_BEARER=val set OAuth 2.0 Bearer Access Token.
1082 : * Relevant only with HTTPAUTH=BEARER (GDAL >= 3.9).
1083 : * Corresponding configuration option: GDAL_HTTP_BEARER
1084 : * </li>
1085 : * <li>POSTFIELDS=val, where val is a nul-terminated string to be passed to the
1086 : * server with a POST request.
1087 : * No Corresponding configuration option.
1088 : * </li>
1089 : * <li>PROXY=val, to make requests go through a
1090 : * proxy server, where val is of the form proxy.server.com:port_number. This
1091 : * option affects both HTTP and HTTPS URLs.
1092 : * Corresponding configuration option: GDAL_HTTP_PROXY.
1093 : * </li>
1094 : * <li>HTTPS_PROXY=val (GDAL
1095 : * >= 2.4), the same meaning as PROXY, but this option is taken into account
1096 : * only for HTTPS URLs.
1097 : * Corresponding configuration option: GDAL_HTTPS_PROXY.
1098 : * </li>
1099 : * <li>PROXYUSERPWD=val, where val is of the form username:password.
1100 : * Corresponding configuration option: GDAL_HTTP_PROXYUSERPWD
1101 : * </li>
1102 : * <li>PROXYAUTH=[BASIC/NTLM/DIGEST/NEGOTIATE/ANY/ANYSAFE] to
1103 : * specify an proxy authentication scheme to use..
1104 : * Corresponding configuration option: GDAL_PROXYAUTH (note: no "HTTP_" in the name)
1105 : * </li>
1106 : * <li>NETRC=[YES/NO] to
1107 : * enable or disable use of $HOME/.netrc (or NETRC_FILE), default YES.
1108 : * Corresponding configuration option: GDAL_HTTP_NETRC.
1109 : * </li>
1110 : * <li>NETRC_FILE=file name to read .netrc info from (GDAL >= 3.7).
1111 : * Corresponding configuration option: GDAL_HTTP_NETRC_FILE.
1112 : * </li>
1113 : * <li>CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc...
1114 : * No corresponding configuration option.
1115 : * </li>
1116 : * <li>FORM_FILE_NAME=val, where val is upload file name. If this
1117 : * option and FORM_FILE_PATH present, request type will set to POST.
1118 : * No corresponding configuration option.
1119 : * </li>
1120 : * <li>FORM_FILE_PATH=val, where val is upload file path.
1121 : * No corresponding configuration option.
1122 : * </li>
1123 : * <li>FORM_KEY_0=val...FORM_KEY_N, where val is name of form item.
1124 : * No corresponding configuration option.
1125 : * </li>
1126 : * <li>FORM_VALUE_0=val...FORM_VALUE_N, where val is value of the form
1127 : * item.
1128 : * No corresponding configuration option.
1129 : * </li>
1130 : * <li>FORM_ITEM_COUNT=val, where val is count of form items.
1131 : * No corresponding configuration option.
1132 : * </li>
1133 : * <li>COOKIE=val, where val is formatted as COOKIE1=VALUE1; COOKIE2=VALUE2;...
1134 : * Corresponding configuration option: GDAL_HTTP_COOKIE.</li>
1135 : * <li>COOKIEFILE=val, where val is file name to read cookies from
1136 : * (GDAL >= 2.4).
1137 : * Corresponding configuration option: GDAL_HTTP_COOKIEFILE.</li>
1138 : * <li>COOKIEJAR=val, where val is file name to store cookies to (GDAL >= 2.4).
1139 : * Corresponding configuration option: GDAL_HTTP_COOKIEJAR.</li>
1140 : * <li>MAX_RETRY=val, where val is the maximum number of
1141 : * retry attempts, when a retry is allowed (cf RETRY_CODES option).
1142 : * Default is 0, meaning no retry.
1143 : * Corresponding configuration option: GDAL_HTTP_MAX_RETRY.
1144 : * </li>
1145 : * <li>RETRY_DELAY=val, where val is the number of seconds
1146 : * between retry attempts. Default is 30.
1147 : * Corresponding configuration option: GDAL_HTTP_RETRY_DELAY.
1148 : * <li>RETRY_CODES=val, where val is "ALL" or a comma-separated list of HTTP
1149 : * codes that are considered for retry. By default, 429, 500, 502, 503 or 504
1150 : * HTTP errors are considered, as well as other situations with a particular
1151 : * HTTP or Curl error message. (GDAL >= 3.10).
1152 : * Corresponding configuration option: GDAL_HTTP_RETRY_CODES.
1153 : * </li>
1154 : * <li>MAX_FILE_SIZE=val, where val is a number of bytes (GDAL >= 2.2)
1155 : * No corresponding configuration option.
1156 : * </li>
1157 : * <li>CAINFO=/path/to/bundle.crt. This is path to Certificate Authority (CA)
1158 : * bundle file. By default, it will be looked for in a system location. If
1159 : * the CAINFO option is not defined, GDAL will also look in the
1160 : * CURL_CA_BUNDLE and SSL_CERT_FILE environment variables respectively
1161 : * and use the first one found as the CAINFO value (GDAL >= 2.1.3). The
1162 : * GDAL_CURL_CA_BUNDLE environment variable may also be used to set the
1163 : * CAINFO value in GDAL >= 3.2.</li>
1164 : * <li>HTTP_VERSION=1.0/1.1/2/2TLS (GDAL >= 2.3)/2PRIOR_KNOWLEDGE (GDAL >= 3.10).
1165 : * Specify HTTP version to use.
1166 : * Will default to 1.1 generally (except on some controlled environments,
1167 : * like Google Compute Engine VMs, where 2TLS will be the default).
1168 : * Support for HTTP/2 requires curl 7.33 or later, built against nghttp2.
1169 : * "2TLS" means that HTTP/2 will be attempted for HTTPS connections only.
1170 : * Whereas "2" means that HTTP/2 will be attempted for HTTP or HTTPS.
1171 : * "2PRIOR_KNOWLEDGE" means that the server will be assumed to support
1172 : * HTTP/2.
1173 : * Corresponding configuration option: GDAL_HTTP_VERSION.
1174 : * </li>
1175 : * <li>SSL_VERIFYSTATUS=YES/NO (GDAL >= 2.3, and curl >= 7.41): determines
1176 : * whether the status of the server cert using the "Certificate Status Request"
1177 : * TLS extension (aka. OCSP stapling) should be checked. If this option is
1178 : * enabled but the server does not support the TLS extension, the verification
1179 : * will fail. Default to NO.
1180 : * Corresponding configuration option: GDAL_HTTP_SSL_VERIFYSTATUS.
1181 : * </li>
1182 : * <li>USE_CAPI_STORE=YES/NO (GDAL >= 2.3,
1183 : * Windows only): whether CA certificates from the Windows certificate store.
1184 : * Defaults to NO.
1185 : * Corresponding configuration option: GDAL_HTTP_USE_CAPI_STORE.
1186 : * </li>
1187 : * <li>TCP_KEEPALIVE=YES/NO (GDAL >= 3.6): whether to
1188 : * enable TCP keep-alive. Defaults to NO.
1189 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPALIVE.
1190 : * </li>
1191 : * <li>TCP_KEEPIDLE=integer, in
1192 : * seconds (GDAL >= 3.6): keep-alive idle time. Defaults to 60. Only taken into
1193 : * account if TCP_KEEPALIVE=YES.
1194 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPIDLE.
1195 : * </li>
1196 : * <li>TCP_KEEPINTVL=integer, in seconds
1197 : * (GDAL >= 3.6): interval time between keep-alive probes. Defaults to 60. Only
1198 : * taken into account if TCP_KEEPALIVE=YES.
1199 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPINTVL.
1200 : * </li>
1201 : * <li>USERAGENT=string: value of User-Agent header. Starting with GDAL 3.7,
1202 : * GDAL core sets it by default (during driver initialization) to GDAL/x.y.z
1203 : * where x.y.z is the GDAL version number. Applications may override it with the
1204 : * CPLHTTPSetDefaultUserAgent() function.
1205 : * Corresponding configuration option: GDAL_HTTP_USERAGENT.
1206 : * </li>
1207 : * <li>SSLCERT=filename (GDAL >= 3.7): Filename of the the SSL client certificate.
1208 : * Cf https://curl.se/libcurl/c/CURLOPT_SSLCERT.html.
1209 : * Corresponding configuration option: GDAL_HTTP_SSLCERT.
1210 : * </li>
1211 : * <li>SSLCERTTYPE=string (GDAL >= 3.7): Format of the SSL certificate: "PEM"
1212 : * or "DER". Cf https://curl.se/libcurl/c/CURLOPT_SSLCERTTYPE.html.
1213 : * Corresponding configuration option: GDAL_HTTP_SSLCERTTYPE.
1214 : * </li>
1215 : * <li>SSLKEY=filename (GDAL >= 3.7): Private key file for TLS and SSL client
1216 : * certificate. Cf https://curl.se/libcurl/c/CURLOPT_SSLKEY.html.
1217 : * Corresponding configuration option: GDAL_HTTP_SSLKEY.
1218 : * </li>
1219 : * <li>KEYPASSWD=string (GDAL >= 3.7): Passphrase to private key.
1220 : * Cf https://curl.se/libcurl/c/CURLOPT_KEYPASSWD.html.
1221 : * Corresponding configuration option: GDAL_HTTP_KEYPASSWD.
1222 : * </ul>
1223 : *
1224 : * If an option is specified through papszOptions and as a configuration option,
1225 : * the former takes precedence over the later.
1226 : *
1227 : * Starting with GDAL 3.7, the above configuration options can also be specified
1228 : * as path-specific options with VSISetPathSpecificOption().
1229 : *
1230 : * @param pszURL valid URL recognized by underlying download library (libcurl)
1231 : * @param papszOptions option list as a NULL-terminated array of strings. May be
1232 : * NULL.
1233 : *
1234 : * @return a CPLHTTPResult* structure that must be freed by
1235 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled
1236 : */
1237 : // clang-format on
1238 2082 : CPLHTTPResult *CPLHTTPFetch(const char *pszURL, CSLConstList papszOptions)
1239 : {
1240 2082 : return CPLHTTPFetchEx(pszURL, papszOptions, nullptr, nullptr, nullptr,
1241 2082 : nullptr);
1242 : }
1243 :
1244 : /**
1245 : * Fetch a document from an url and return in a string.
1246 : * @param pszURL Url to fetch document from web.
1247 : * @param papszOptions Option list as a NULL-terminated array of strings.
1248 : * Available keys see in CPLHTTPFetch.
1249 : * @param pfnProgress Callback for reporting algorithm progress matching the
1250 : * GDALProgressFunc() semantics. May be NULL.
1251 : * @param pProgressArg Callback argument passed to pfnProgress.
1252 : * @param pfnWrite Write function pointer matching the CPLHTTPWriteFunc()
1253 : * semantics. May be NULL.
1254 : * @param pWriteArg Argument which will pass to a write function.
1255 : * @return A CPLHTTPResult* structure that must be freed by
1256 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled.
1257 : */
1258 2112 : CPLHTTPResult *CPLHTTPFetchEx(const char *pszURL, CSLConstList papszOptions,
1259 : GDALProgressFunc pfnProgress, void *pProgressArg,
1260 : CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg)
1261 :
1262 : {
1263 2912 : if (STARTS_WITH(pszURL, "/vsimem/") &&
1264 : // Disabled by default for potential security issues.
1265 800 : CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")))
1266 : {
1267 775 : CPLString osURL(pszURL);
1268 : const char *pszCustomRequest =
1269 775 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
1270 775 : if (pszCustomRequest != nullptr)
1271 : {
1272 5 : osURL += "&CUSTOMREQUEST=";
1273 5 : osURL += pszCustomRequest;
1274 : }
1275 775 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
1276 775 : if (pszUserPwd != nullptr)
1277 : {
1278 1 : osURL += "&USERPWD=";
1279 1 : osURL += pszUserPwd;
1280 : }
1281 775 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
1282 775 : if (pszPost != nullptr) // Hack: We append post content to filename.
1283 : {
1284 281 : osURL += "&POSTFIELDS=";
1285 281 : osURL += pszPost;
1286 : }
1287 775 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1288 907 : if (pszHeaders != nullptr &&
1289 132 : CPLTestBool(
1290 : CPLGetConfigOption("CPL_CURL_VSIMEM_PRINT_HEADERS", "FALSE")))
1291 : {
1292 5 : osURL += "&HEADERS=";
1293 5 : osURL += pszHeaders;
1294 : }
1295 775 : vsi_l_offset nLength = 0;
1296 : CPLHTTPResult *psResult =
1297 775 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1298 775 : GByte *pabyData = VSIGetMemFileBuffer(osURL, &nLength, FALSE);
1299 775 : if (pabyData == nullptr)
1300 : {
1301 119 : CPLDebug("HTTP", "Cannot find %s", osURL.c_str());
1302 119 : psResult->nStatus = 1;
1303 119 : psResult->pszErrBuf =
1304 119 : CPLStrdup(CPLSPrintf("HTTP error code : %d", 404));
1305 119 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1306 : }
1307 656 : else if (nLength != 0)
1308 : {
1309 613 : psResult->nDataLen = static_cast<int>(nLength);
1310 613 : psResult->pabyData = static_cast<GByte *>(
1311 613 : CPLMalloc(static_cast<size_t>(nLength) + 1));
1312 613 : memcpy(psResult->pabyData, pabyData, static_cast<size_t>(nLength));
1313 613 : psResult->pabyData[static_cast<size_t>(nLength)] = 0;
1314 : }
1315 :
1316 775 : if (psResult->pabyData != nullptr &&
1317 613 : STARTS_WITH(reinterpret_cast<char *>(psResult->pabyData),
1318 : "Content-Type: "))
1319 : {
1320 7 : const char *pszContentType =
1321 7 : reinterpret_cast<char *>(psResult->pabyData) +
1322 : strlen("Content-type: ");
1323 7 : const char *pszEOL = strchr(pszContentType, '\r');
1324 7 : if (pszEOL)
1325 7 : pszEOL = strchr(pszContentType, '\n');
1326 7 : if (pszEOL)
1327 : {
1328 7 : size_t nContentLength = pszEOL - pszContentType;
1329 7 : psResult->pszContentType =
1330 7 : static_cast<char *>(CPLMalloc(nContentLength + 1));
1331 7 : memcpy(psResult->pszContentType, pszContentType,
1332 : nContentLength);
1333 7 : psResult->pszContentType[nContentLength] = 0;
1334 : }
1335 : }
1336 :
1337 775 : return psResult;
1338 : }
1339 :
1340 : // Try to user alternate network layer if set.
1341 1337 : auto pCtx = GetHTTPFetchContext(false);
1342 1337 : if (pCtx)
1343 : {
1344 3 : for (size_t i = pCtx->stack.size(); i > 0;)
1345 : {
1346 1 : --i;
1347 1 : const auto &cbk = pCtx->stack[i];
1348 1 : auto cbkFunc = cbk.first;
1349 1 : auto pUserData = cbk.second;
1350 1 : auto res = cbkFunc(pszURL, papszOptions, pfnProgress, pProgressArg,
1351 : pfnWrite, pWriteArg, pUserData);
1352 1 : if (res)
1353 : {
1354 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1355 : {
1356 0 : CPLHTTPDestroyResult(res);
1357 0 : res = nullptr;
1358 : }
1359 1 : return res;
1360 : }
1361 : }
1362 : }
1363 :
1364 1336 : if (gpsHTTPFetchCallbackFunc)
1365 : {
1366 1 : auto res = gpsHTTPFetchCallbackFunc(pszURL, papszOptions, pfnProgress,
1367 : pProgressArg, pfnWrite, pWriteArg,
1368 : gpHTTPFetchCallbackUserData);
1369 1 : if (res)
1370 : {
1371 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1372 : {
1373 0 : CPLHTTPDestroyResult(res);
1374 0 : res = nullptr;
1375 : }
1376 1 : return res;
1377 : }
1378 : }
1379 :
1380 : #ifndef HAVE_CURL
1381 : CPLError(CE_Failure, CPLE_NotSupported,
1382 : "GDAL/OGR not compiled with libcurl support, "
1383 : "remote requests not supported.");
1384 : return nullptr;
1385 : #else
1386 :
1387 : /* -------------------------------------------------------------------- */
1388 : /* Are we using a persistent named session? If so, search for */
1389 : /* or create it. */
1390 : /* */
1391 : /* Currently this code does not attempt to protect against */
1392 : /* multiple threads asking for the same named session. If that */
1393 : /* occurs it will be in use in multiple threads at once, which */
1394 : /* will lead to potential crashes in libcurl. */
1395 : /* -------------------------------------------------------------------- */
1396 1335 : CURL *http_handle = nullptr;
1397 :
1398 1335 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1399 : const char *pszClosePersistent =
1400 1335 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1401 1335 : if (pszPersistent)
1402 : {
1403 994 : CPLString osSessionName = pszPersistent;
1404 497 : CPLMutexHolder oHolder(&hSessionMapMutex);
1405 :
1406 497 : if (poSessionMap == nullptr)
1407 134 : poSessionMap = new std::map<CPLString, CURL *>;
1408 497 : if (poSessionMap->count(osSessionName) == 0)
1409 : {
1410 157 : (*poSessionMap)[osSessionName] = curl_easy_init();
1411 157 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1412 : osSessionName.c_str());
1413 : }
1414 :
1415 497 : http_handle = (*poSessionMap)[osSessionName];
1416 : }
1417 : /* -------------------------------------------------------------------- */
1418 : /* Are we requested to close a persistent named session? */
1419 : /* -------------------------------------------------------------------- */
1420 838 : else if (pszClosePersistent)
1421 : {
1422 374 : CPLString osSessionName = pszClosePersistent;
1423 187 : CPLMutexHolder oHolder(&hSessionMapMutex);
1424 :
1425 187 : if (poSessionMap)
1426 : {
1427 : std::map<CPLString, CURL *>::iterator oIter =
1428 157 : poSessionMap->find(osSessionName);
1429 157 : if (oIter != poSessionMap->end())
1430 : {
1431 157 : curl_easy_cleanup(oIter->second);
1432 157 : poSessionMap->erase(oIter);
1433 157 : if (poSessionMap->empty())
1434 : {
1435 134 : delete poSessionMap;
1436 134 : poSessionMap = nullptr;
1437 : }
1438 157 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1439 : osSessionName.c_str());
1440 : }
1441 : else
1442 : {
1443 0 : CPLDebug("HTTP",
1444 : "Could not find persistent session named '%s'.",
1445 : osSessionName.c_str());
1446 : }
1447 : }
1448 :
1449 187 : return nullptr;
1450 : }
1451 : else
1452 651 : http_handle = curl_easy_init();
1453 :
1454 : /* -------------------------------------------------------------------- */
1455 : /* Setup the request. */
1456 : /* -------------------------------------------------------------------- */
1457 1148 : char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
1458 :
1459 1148 : CPLHTTPEmitFetchDebug(pszURL);
1460 :
1461 : CPLHTTPResult *psResult =
1462 1148 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1463 :
1464 : struct curl_slist *headers = reinterpret_cast<struct curl_slist *>(
1465 1148 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1466 1148 : if (headers != nullptr)
1467 574 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers);
1468 :
1469 : // Are we making a head request.
1470 1148 : const char *pszNoBody = nullptr;
1471 1148 : if ((pszNoBody = CSLFetchNameValue(papszOptions, "NO_BODY")) != nullptr)
1472 : {
1473 0 : if (CPLTestBool(pszNoBody))
1474 : {
1475 0 : CPLDebug("HTTP", "HEAD Request: %s", pszURL);
1476 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOBODY, 1L);
1477 : }
1478 : }
1479 :
1480 : // Capture response headers.
1481 1148 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA, psResult);
1482 1148 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1483 : CPLHdrWriteFct);
1484 :
1485 1148 : CPLHTTPResultWithLimit sResultWithLimit;
1486 1148 : if (nullptr == pfnWrite)
1487 : {
1488 1147 : pfnWrite = CPLWriteFct;
1489 :
1490 1147 : sResultWithLimit.psResult = psResult;
1491 1147 : sResultWithLimit.nMaxFileSize = 0;
1492 : const char *pszMaxFileSize =
1493 1147 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1494 1147 : if (pszMaxFileSize != nullptr)
1495 : {
1496 10 : sResultWithLimit.nMaxFileSize = atoi(pszMaxFileSize);
1497 : // Only useful if size is returned by server before actual download.
1498 10 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1499 : sResultWithLimit.nMaxFileSize);
1500 : }
1501 1147 : pWriteArg = &sResultWithLimit;
1502 : }
1503 :
1504 1148 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, pWriteArg);
1505 1148 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, pfnWrite);
1506 :
1507 1148 : CurlProcessData stProcessData = {pfnProgress, pProgressArg};
1508 1148 : if (nullptr != pfnProgress)
1509 : {
1510 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFOFUNCTION,
1511 : NewProcessFunction);
1512 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFODATA,
1513 : &stProcessData);
1514 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOPROGRESS, 0L);
1515 : }
1516 :
1517 1148 : szCurlErrBuf[0] = '\0';
1518 :
1519 1148 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER, szCurlErrBuf);
1520 :
1521 1148 : bool bGZipRequested = false;
1522 1148 : if (bSupportGZip && CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1523 : {
1524 1148 : bGZipRequested = true;
1525 1148 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1526 : }
1527 :
1528 2296 : CPLHTTPPostFields oPostFields;
1529 1148 : if (oPostFields.Fill(http_handle, papszOptions) != CE_None)
1530 : {
1531 2 : psResult->nStatus = 34; // CURLE_HTTP_POST_ERROR
1532 2 : psResult->pszErrBuf = CPLStrdup(oPostFields.GetErrorMessage().c_str());
1533 2 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1534 2 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1535 2 : return psResult;
1536 : }
1537 :
1538 : /* -------------------------------------------------------------------- */
1539 : /* Depending on status code, retry this HTTP call until max */
1540 : /* retry has been reached */
1541 : /* -------------------------------------------------------------------- */
1542 1146 : const char *pszRetryDelay = CSLFetchNameValue(papszOptions, "RETRY_DELAY");
1543 1146 : if (pszRetryDelay == nullptr)
1544 1146 : pszRetryDelay = CPLGetConfigOption(
1545 : "GDAL_HTTP_RETRY_DELAY", CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY));
1546 1146 : const char *pszMaxRetries = CSLFetchNameValue(papszOptions, "MAX_RETRY");
1547 1146 : if (pszMaxRetries == nullptr)
1548 1082 : pszMaxRetries = CPLGetConfigOption(
1549 : "GDAL_HTTP_MAX_RETRY", CPLSPrintf("%d", CPL_HTTP_MAX_RETRY));
1550 : // coverity[tainted_data]
1551 1146 : double dfRetryDelaySecs = CPLAtof(pszRetryDelay);
1552 1146 : int nMaxRetries = atoi(pszMaxRetries);
1553 1146 : const char *pszRetryCodes = CSLFetchNameValue(papszOptions, "RETRY_CODES");
1554 1146 : if (!pszRetryCodes)
1555 1146 : pszRetryCodes = CPLGetConfigOption("GDAL_HTTP_RETRY_CODES", nullptr);
1556 1146 : int nRetryCount = 0;
1557 :
1558 : while (true)
1559 : {
1560 : /* --------------------------------------------------------------------
1561 : */
1562 : /* Execute the request, waiting for results. */
1563 : /* --------------------------------------------------------------------
1564 : */
1565 1146 : void *old_handler = CPLHTTPIgnoreSigPipe();
1566 1146 : psResult->nStatus = static_cast<int>(curl_easy_perform(http_handle));
1567 1146 : CPLHTTPRestoreSigPipeHandler(old_handler);
1568 :
1569 : /* --------------------------------------------------------------------
1570 : */
1571 : /* Fetch content-type if possible. */
1572 : /* --------------------------------------------------------------------
1573 : */
1574 1146 : psResult->pszContentType = nullptr;
1575 1146 : curl_easy_getinfo(http_handle, CURLINFO_CONTENT_TYPE,
1576 : &(psResult->pszContentType));
1577 1146 : if (psResult->pszContentType != nullptr)
1578 508 : psResult->pszContentType = CPLStrdup(psResult->pszContentType);
1579 :
1580 1146 : long response_code = 0;
1581 1146 : curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
1582 1146 : if (response_code != 200)
1583 : {
1584 394 : const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1585 : static_cast<int>(response_code), dfRetryDelaySecs,
1586 197 : reinterpret_cast<const char *>(psResult->pabyData),
1587 : szCurlErrBuf, pszRetryCodes);
1588 197 : if (dfNewRetryDelay > 0 && nRetryCount < nMaxRetries)
1589 : {
1590 0 : CPLError(CE_Warning, CPLE_AppDefined,
1591 : "HTTP error code: %d - %s. "
1592 : "Retrying again in %.1f secs",
1593 : static_cast<int>(response_code), pszURL,
1594 : dfRetryDelaySecs);
1595 0 : CPLSleep(dfRetryDelaySecs);
1596 0 : dfRetryDelaySecs = dfNewRetryDelay;
1597 0 : nRetryCount++;
1598 :
1599 0 : CPLFree(psResult->pszContentType);
1600 0 : psResult->pszContentType = nullptr;
1601 0 : CSLDestroy(psResult->papszHeaders);
1602 0 : psResult->papszHeaders = nullptr;
1603 0 : CPLFree(psResult->pabyData);
1604 0 : psResult->pabyData = nullptr;
1605 0 : psResult->nDataLen = 0;
1606 0 : psResult->nDataAlloc = 0;
1607 :
1608 0 : continue;
1609 : }
1610 : }
1611 :
1612 : /* --------------------------------------------------------------------
1613 : */
1614 : /* Have we encountered some sort of error? */
1615 : /* --------------------------------------------------------------------
1616 : */
1617 1146 : if (strlen(szCurlErrBuf) > 0)
1618 : {
1619 8 : bool bSkipError = false;
1620 : const char *pszContentLength =
1621 8 : CSLFetchNameValue(psResult->papszHeaders, "Content-Length");
1622 : // Some servers such as
1623 : // http://115.113.193.14/cgi-bin/world/qgis_mapserv.fcgi?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetCapabilities
1624 : // invalidly return Content-Length as the uncompressed size, with
1625 : // makes curl to wait for more data and time-out finally. If we got
1626 : // the expected data size, then we don't emit an error but turn off
1627 : // GZip requests.
1628 8 : if (bGZipRequested &&
1629 8 : strstr(szCurlErrBuf, "transfer closed with") &&
1630 0 : strstr(szCurlErrBuf, "bytes remaining to read"))
1631 : {
1632 0 : if (pszContentLength && psResult->nDataLen != 0 &&
1633 0 : atoi(pszContentLength) == psResult->nDataLen)
1634 : {
1635 : const char *pszCurlGZIPOption =
1636 0 : CPLGetConfigOption("CPL_CURL_GZIP", nullptr);
1637 0 : if (pszCurlGZIPOption == nullptr)
1638 : {
1639 0 : CPLSetConfigOption("CPL_CURL_GZIP", "NO");
1640 0 : CPLDebug("HTTP",
1641 : "Disabling CPL_CURL_GZIP, "
1642 : "because %s doesn't support it properly",
1643 : pszURL);
1644 : }
1645 0 : psResult->nStatus = 0;
1646 0 : bSkipError = true;
1647 0 : }
1648 : }
1649 :
1650 : // Ignore SSL errors about non-properly terminated connection,
1651 : // often due to HTTP proxies
1652 8 : else if (pszContentLength == nullptr &&
1653 : // Cf https://github.com/curl/curl/pull/3148
1654 8 : (strstr(szCurlErrBuf,
1655 : "GnuTLS recv error (-110): The TLS connection was "
1656 8 : "non-properly terminated") != nullptr ||
1657 : // Cf https://github.com/curl/curl/issues/9024
1658 8 : strstr(szCurlErrBuf,
1659 : "SSL_read: error:0A000126:SSL "
1660 : "routines::unexpected eof while reading") !=
1661 : nullptr))
1662 : {
1663 0 : psResult->nStatus = 0;
1664 0 : bSkipError = true;
1665 : }
1666 8 : else if (CPLTestBool(
1667 : CPLGetConfigOption("CPL_CURL_IGNORE_ERROR", "NO")))
1668 : {
1669 0 : psResult->nStatus = 0;
1670 0 : bSkipError = true;
1671 : }
1672 :
1673 8 : if (!bSkipError)
1674 : {
1675 8 : psResult->pszErrBuf = CPLStrdup(szCurlErrBuf);
1676 8 : if (psResult->nDataLen > 0)
1677 : {
1678 1 : CPLError(CE_Failure, CPLE_AppDefined,
1679 : "%s. You may set the CPL_CURL_IGNORE_ERROR "
1680 : "configuration option to YES to try to ignore it.",
1681 : szCurlErrBuf);
1682 : }
1683 : else
1684 : {
1685 7 : CPLError(CE_Failure, CPLE_AppDefined, "%s", szCurlErrBuf);
1686 : }
1687 : }
1688 : }
1689 : else
1690 : {
1691 1138 : if (response_code >= 400 && response_code < 600)
1692 : {
1693 186 : psResult->pszErrBuf = CPLStrdup(CPLSPrintf(
1694 : "HTTP error code : %d", static_cast<int>(response_code)));
1695 186 : CPLError(CE_Failure, CPLE_AppDefined, "%s",
1696 : psResult->pszErrBuf);
1697 : }
1698 : }
1699 1146 : break;
1700 0 : }
1701 :
1702 1146 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1703 :
1704 1146 : return psResult;
1705 : #endif /* def HAVE_CURL */
1706 : }
1707 :
1708 : #ifdef HAVE_CURL
1709 : /************************************************************************/
1710 : /* CPLMultiPerformWait() */
1711 : /************************************************************************/
1712 :
1713 2927 : bool CPLMultiPerformWait(void *hCurlMultiHandleIn, int & /*repeats*/)
1714 : {
1715 2927 : CURLM *hCurlMultiHandle = static_cast<CURLM *>(hCurlMultiHandleIn);
1716 :
1717 : // Wait for events on the sockets
1718 :
1719 : // Using curl_multi_poll() is preferred to avoid hitting the 1024 file
1720 : // descriptor limit
1721 :
1722 2927 : int numfds = 0;
1723 2927 : if (curl_multi_poll(hCurlMultiHandle, nullptr, 0, 1000, &numfds) !=
1724 : CURLM_OK)
1725 : {
1726 0 : CPLError(CE_Failure, CPLE_AppDefined, "curl_multi_poll() failed");
1727 0 : return false;
1728 : }
1729 2927 : return true;
1730 : }
1731 :
1732 : class CPLHTTPErrorBuffer
1733 : {
1734 : public:
1735 : char szBuffer[CURL_ERROR_SIZE + 1];
1736 :
1737 0 : CPLHTTPErrorBuffer()
1738 0 : {
1739 0 : szBuffer[0] = '\0';
1740 0 : }
1741 : };
1742 :
1743 : #endif // HAVE_CURL
1744 :
1745 : /************************************************************************/
1746 : /* CPLHTTPMultiFetch() */
1747 : /************************************************************************/
1748 :
1749 : /**
1750 : * \brief Fetch several documents at once
1751 : *
1752 : * @param papszURL array of valid URLs recognized by underlying download library
1753 : * (libcurl)
1754 : * @param nURLCount number of URLs of papszURL
1755 : * @param nMaxSimultaneous maximum number of downloads to issue simultaneously.
1756 : * Any negative or zer value means unlimited.
1757 : * @param papszOptions option list as a NULL-terminated array of strings. May be
1758 : * NULL. Refer to CPLHTTPFetch() for valid options.
1759 : * @return an array of CPLHTTPResult* structures that must be freed by
1760 : * CPLHTTPDestroyMultiResult() or NULL if libcurl support is disabled
1761 : *
1762 : * @since GDAL 2.3
1763 : */
1764 0 : CPLHTTPResult **CPLHTTPMultiFetch(const char *const *papszURL, int nURLCount,
1765 : int nMaxSimultaneous,
1766 : CSLConstList papszOptions)
1767 : {
1768 : #ifndef HAVE_CURL
1769 : (void)papszURL;
1770 : (void)nURLCount;
1771 : (void)nMaxSimultaneous;
1772 : (void)papszOptions;
1773 :
1774 : CPLError(CE_Failure, CPLE_NotSupported,
1775 : "GDAL/OGR not compiled with libcurl support, "
1776 : "remote requests not supported.");
1777 : return nullptr;
1778 : #else /* def HAVE_CURL */
1779 :
1780 : /* -------------------------------------------------------------------- */
1781 : /* Are we using a persistent named session? If so, search for */
1782 : /* or create it. */
1783 : /* */
1784 : /* Currently this code does not attempt to protect against */
1785 : /* multiple threads asking for the same named session. If that */
1786 : /* occurs it will be in use in multiple threads at once, which */
1787 : /* will lead to potential crashes in libcurl. */
1788 : /* -------------------------------------------------------------------- */
1789 0 : CURLM *hCurlMultiHandle = nullptr;
1790 :
1791 0 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1792 : const char *pszClosePersistent =
1793 0 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1794 0 : if (pszPersistent)
1795 : {
1796 0 : CPLString osSessionName = pszPersistent;
1797 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1798 :
1799 0 : if (poSessionMultiMap == nullptr)
1800 0 : poSessionMultiMap = new std::map<CPLString, CURLM *>;
1801 0 : if (poSessionMultiMap->count(osSessionName) == 0)
1802 : {
1803 0 : (*poSessionMultiMap)[osSessionName] = curl_multi_init();
1804 0 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1805 : osSessionName.c_str());
1806 : }
1807 :
1808 0 : hCurlMultiHandle = (*poSessionMultiMap)[osSessionName];
1809 : }
1810 : /* -------------------------------------------------------------------- */
1811 : /* Are we requested to close a persistent named session? */
1812 : /* -------------------------------------------------------------------- */
1813 0 : else if (pszClosePersistent)
1814 : {
1815 0 : CPLString osSessionName = pszClosePersistent;
1816 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1817 :
1818 0 : if (poSessionMultiMap)
1819 : {
1820 0 : auto oIter = poSessionMultiMap->find(osSessionName);
1821 0 : if (oIter != poSessionMultiMap->end())
1822 : {
1823 0 : VSICURLMultiCleanup(oIter->second);
1824 0 : poSessionMultiMap->erase(oIter);
1825 0 : if (poSessionMultiMap->empty())
1826 : {
1827 0 : delete poSessionMultiMap;
1828 0 : poSessionMultiMap = nullptr;
1829 : }
1830 0 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1831 : osSessionName.c_str());
1832 : }
1833 : else
1834 : {
1835 0 : CPLDebug("HTTP",
1836 : "Could not find persistent session named '%s'.",
1837 : osSessionName.c_str());
1838 : }
1839 : }
1840 :
1841 0 : return nullptr;
1842 : }
1843 : else
1844 : {
1845 0 : hCurlMultiHandle = curl_multi_init();
1846 : }
1847 :
1848 : CPLHTTPResult **papsResults = static_cast<CPLHTTPResult **>(
1849 0 : CPLCalloc(nURLCount, sizeof(CPLHTTPResult *)));
1850 0 : std::vector<CURL *> asHandles;
1851 0 : std::vector<CPLHTTPResultWithLimit> asResults;
1852 0 : asResults.resize(nURLCount);
1853 0 : std::vector<struct curl_slist *> aHeaders;
1854 0 : aHeaders.resize(nURLCount);
1855 0 : std::vector<CPLHTTPErrorBuffer> asErrorBuffers;
1856 0 : asErrorBuffers.resize(nURLCount);
1857 :
1858 0 : for (int i = 0; i < nURLCount; i++)
1859 : {
1860 0 : papsResults[i] =
1861 0 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1862 :
1863 0 : const char *pszURL = papszURL[i];
1864 0 : CURL *http_handle = curl_easy_init();
1865 :
1866 0 : aHeaders[i] = reinterpret_cast<struct curl_slist *>(
1867 0 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1868 :
1869 : // Set Headers.
1870 0 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1871 0 : if (pszHeaders != nullptr)
1872 : {
1873 : char **papszTokensHeaders =
1874 0 : CSLTokenizeString2(pszHeaders, "\r\n", 0);
1875 0 : for (int j = 0; papszTokensHeaders[j] != nullptr; ++j)
1876 0 : aHeaders[i] =
1877 0 : curl_slist_append(aHeaders[i], papszTokensHeaders[j]);
1878 0 : CSLDestroy(papszTokensHeaders);
1879 : }
1880 :
1881 0 : if (aHeaders[i] != nullptr)
1882 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER,
1883 : aHeaders[i]);
1884 :
1885 : // Capture response headers.
1886 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA,
1887 : papsResults[i]);
1888 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1889 : CPLHdrWriteFct);
1890 :
1891 0 : asResults[i].psResult = papsResults[i];
1892 : const char *pszMaxFileSize =
1893 0 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1894 0 : if (pszMaxFileSize != nullptr)
1895 : {
1896 0 : asResults[i].nMaxFileSize = atoi(pszMaxFileSize);
1897 : // Only useful if size is returned by server before actual download.
1898 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1899 : asResults[i].nMaxFileSize);
1900 : }
1901 :
1902 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA,
1903 : &asResults[i]);
1904 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION,
1905 : CPLWriteFct);
1906 :
1907 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER,
1908 : asErrorBuffers[i].szBuffer);
1909 :
1910 0 : if (bSupportGZip &&
1911 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1912 : {
1913 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1914 : }
1915 :
1916 0 : asHandles.push_back(http_handle);
1917 : }
1918 :
1919 0 : int iCurRequest = 0;
1920 0 : for (;
1921 0 : iCurRequest <
1922 0 : std::min(nURLCount, nMaxSimultaneous > 0 ? nMaxSimultaneous : INT_MAX);
1923 : iCurRequest++)
1924 : {
1925 0 : CPLHTTPEmitFetchDebug(papszURL[iCurRequest],
1926 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1927 0 : curl_multi_add_handle(hCurlMultiHandle, asHandles[iCurRequest]);
1928 : }
1929 :
1930 0 : int repeats = 0;
1931 0 : void *old_handler = CPLHTTPIgnoreSigPipe();
1932 : while (true)
1933 : {
1934 0 : int still_running = 0;
1935 0 : while (curl_multi_perform(hCurlMultiHandle, &still_running) ==
1936 : CURLM_CALL_MULTI_PERFORM)
1937 : {
1938 : // loop
1939 : }
1940 0 : if (!still_running && iCurRequest == nURLCount)
1941 : {
1942 0 : break;
1943 : }
1944 :
1945 0 : bool bRequestsAdded = false;
1946 : CURLMsg *msg;
1947 0 : do
1948 : {
1949 0 : int msgq = 0;
1950 0 : msg = curl_multi_info_read(hCurlMultiHandle, &msgq);
1951 0 : if (msg && (msg->msg == CURLMSG_DONE))
1952 : {
1953 0 : if (iCurRequest < nURLCount)
1954 : {
1955 0 : CPLHTTPEmitFetchDebug(
1956 0 : papszURL[iCurRequest],
1957 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1958 0 : curl_multi_add_handle(hCurlMultiHandle,
1959 0 : asHandles[iCurRequest]);
1960 0 : iCurRequest++;
1961 0 : bRequestsAdded = true;
1962 : }
1963 : }
1964 0 : } while (msg);
1965 :
1966 0 : if (!bRequestsAdded)
1967 0 : CPLMultiPerformWait(hCurlMultiHandle, repeats);
1968 0 : }
1969 0 : CPLHTTPRestoreSigPipeHandler(old_handler);
1970 :
1971 0 : for (int i = 0; i < nURLCount; i++)
1972 : {
1973 0 : if (asErrorBuffers[i].szBuffer[0] != '\0')
1974 0 : papsResults[i]->pszErrBuf = CPLStrdup(asErrorBuffers[i].szBuffer);
1975 : else
1976 : {
1977 0 : long response_code = 0;
1978 0 : curl_easy_getinfo(asHandles[i], CURLINFO_RESPONSE_CODE,
1979 : &response_code);
1980 :
1981 0 : if (response_code >= 400 && response_code < 600)
1982 : {
1983 0 : papsResults[i]->pszErrBuf = CPLStrdup(CPLSPrintf(
1984 : "HTTP error code : %d", static_cast<int>(response_code)));
1985 : }
1986 : }
1987 :
1988 0 : curl_easy_getinfo(asHandles[i], CURLINFO_CONTENT_TYPE,
1989 : &(papsResults[i]->pszContentType));
1990 0 : if (papsResults[i]->pszContentType != nullptr)
1991 0 : papsResults[i]->pszContentType =
1992 0 : CPLStrdup(papsResults[i]->pszContentType);
1993 :
1994 0 : curl_multi_remove_handle(hCurlMultiHandle, asHandles[i]);
1995 0 : curl_easy_cleanup(asHandles[i]);
1996 : }
1997 :
1998 0 : if (!pszPersistent)
1999 0 : VSICURLMultiCleanup(hCurlMultiHandle);
2000 :
2001 0 : for (size_t i = 0; i < aHeaders.size(); i++)
2002 0 : curl_slist_free_all(aHeaders[i]);
2003 :
2004 0 : return papsResults;
2005 : #endif /* def HAVE_CURL */
2006 : }
2007 :
2008 : /************************************************************************/
2009 : /* CPLHTTPDestroyMultiResult() */
2010 : /************************************************************************/
2011 : /**
2012 : * \brief Clean the memory associated with the return value of
2013 : * CPLHTTPMultiFetch()
2014 : *
2015 : * @param papsResults pointer to the return value of CPLHTTPMultiFetch()
2016 : * @param nCount value of the nURLCount parameter passed to CPLHTTPMultiFetch()
2017 : * @since GDAL 2.3
2018 : */
2019 0 : void CPLHTTPDestroyMultiResult(CPLHTTPResult **papsResults, int nCount)
2020 : {
2021 0 : if (papsResults)
2022 : {
2023 0 : for (int i = 0; i < nCount; i++)
2024 : {
2025 0 : CPLHTTPDestroyResult(papsResults[i]);
2026 : }
2027 0 : CPLFree(papsResults);
2028 : }
2029 0 : }
2030 :
2031 : #ifdef HAVE_CURL
2032 :
2033 : #ifdef _WIN32
2034 :
2035 : #include <windows.h>
2036 :
2037 : /************************************************************************/
2038 : /* CPLFindWin32CurlCaBundleCrt() */
2039 : /************************************************************************/
2040 :
2041 : static const char *CPLFindWin32CurlCaBundleCrt()
2042 : {
2043 : DWORD nResLen;
2044 : const DWORD nBufSize = MAX_PATH + 1;
2045 : char *pszFilePart = nullptr;
2046 :
2047 : char *pszPath = static_cast<char *>(CPLCalloc(1, nBufSize));
2048 :
2049 : nResLen = SearchPathA(nullptr, "curl-ca-bundle.crt", nullptr, nBufSize,
2050 : pszPath, &pszFilePart);
2051 : if (nResLen > 0)
2052 : {
2053 : const char *pszRet = CPLSPrintf("%s", pszPath);
2054 : CPLFree(pszPath);
2055 : return pszRet;
2056 : }
2057 : CPLFree(pszPath);
2058 : return nullptr;
2059 : }
2060 :
2061 : #endif // WIN32
2062 :
2063 : /************************************************************************/
2064 : /* CPLHTTPCurlDebugFunction() */
2065 : /************************************************************************/
2066 :
2067 16 : static int CPLHTTPCurlDebugFunction(CURL *handle, curl_infotype type,
2068 : char *data, size_t size, void *userp)
2069 : {
2070 : (void)handle;
2071 : (void)userp;
2072 :
2073 16 : const char *pszDebugKey = nullptr;
2074 16 : if (type == CURLINFO_TEXT)
2075 : {
2076 10 : pszDebugKey = "CURL_INFO_TEXT";
2077 : }
2078 6 : else if (type == CURLINFO_HEADER_OUT)
2079 : {
2080 1 : pszDebugKey = "CURL_INFO_HEADER_OUT";
2081 : }
2082 5 : else if (type == CURLINFO_HEADER_IN)
2083 : {
2084 5 : pszDebugKey = "CURL_INFO_HEADER_IN";
2085 : }
2086 0 : else if (type == CURLINFO_DATA_IN &&
2087 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE_DATA_IN", "NO")))
2088 : {
2089 0 : pszDebugKey = "CURL_INFO_DATA_IN";
2090 : }
2091 :
2092 16 : if (pszDebugKey)
2093 : {
2094 32 : std::string osMsg(data, size);
2095 16 : if (!osMsg.empty() && osMsg.back() == '\n')
2096 16 : osMsg.pop_back();
2097 16 : CPLDebug(pszDebugKey, "%s", osMsg.c_str());
2098 : }
2099 16 : return 0;
2100 : }
2101 :
2102 : /************************************************************************/
2103 : /* CPLHTTPSetOptions() */
2104 : /************************************************************************/
2105 :
2106 : // Note: papszOptions must be kept alive until curl_easy/multi_perform()
2107 : // has completed, and we must be careful not to set short lived strings
2108 : // with unchecked_curl_easy_setopt(), as long as we need to support curl < 7.17
2109 : // see https://curl.haxx.se/libcurl/c/unchecked_curl_easy_setopt.html
2110 : // caution: if we remove that assumption, we'll needto use
2111 : // CURLOPT_COPYPOSTFIELDS
2112 :
2113 2549 : void *CPLHTTPSetOptions(void *pcurl, const char *pszURL,
2114 : const char *const *papszOptions)
2115 : {
2116 2549 : CheckCurlFeatures();
2117 :
2118 2549 : CURL *http_handle = reinterpret_cast<CURL *>(pcurl);
2119 :
2120 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_URL, pszURL);
2121 :
2122 2549 : if (CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")))
2123 : {
2124 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1);
2125 :
2126 1 : if (CPLIsDebugEnabled())
2127 : {
2128 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_DEBUGFUNCTION,
2129 : CPLHTTPCurlDebugFunction);
2130 : }
2131 : }
2132 :
2133 : const char *pszHttpVersion =
2134 2549 : CSLFetchNameValue(papszOptions, "HTTP_VERSION");
2135 2549 : if (pszHttpVersion == nullptr)
2136 2549 : pszHttpVersion = CPLGetConfigOption("GDAL_HTTP_VERSION", nullptr);
2137 2549 : if (pszHttpVersion && strcmp(pszHttpVersion, "1.0") == 0)
2138 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2139 : CURL_HTTP_VERSION_1_0);
2140 2549 : else if (pszHttpVersion && strcmp(pszHttpVersion, "1.1") == 0)
2141 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2142 : CURL_HTTP_VERSION_1_1);
2143 2549 : else if (pszHttpVersion && (strcmp(pszHttpVersion, "2") == 0 ||
2144 0 : strcmp(pszHttpVersion, "2.0") == 0))
2145 : {
2146 0 : if (bSupportHTTP2)
2147 : {
2148 : // Try HTTP/2 both for HTTP and HTTPS. With fallback to HTTP/1.1
2149 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2150 : CURL_HTTP_VERSION_2_0);
2151 : }
2152 : }
2153 2549 : else if (pszHttpVersion && strcmp(pszHttpVersion, "2PRIOR_KNOWLEDGE") == 0)
2154 : {
2155 0 : if (bSupportHTTP2)
2156 : {
2157 : // Assume HTTP/2 is supported by the server. The cURL docs indicate
2158 : // that it makes no difference for HTTPS, but it does seem to work
2159 : // in practice.
2160 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2161 : CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE);
2162 : }
2163 : }
2164 2549 : else if (pszHttpVersion == nullptr || strcmp(pszHttpVersion, "2TLS") == 0)
2165 : {
2166 2549 : if (bSupportHTTP2)
2167 : {
2168 : // Only enable this mode if explicitly required, or if the
2169 : // machine is a GCE instance. On other networks, requesting a
2170 : // file in HTTP/2 is found to be significantly slower than HTTP/1.1
2171 : // for unknown reasons.
2172 2549 : if (pszHttpVersion != nullptr || CPLIsMachineForSureGCEInstance())
2173 : {
2174 : static bool bDebugEmitted = false;
2175 0 : if (!bDebugEmitted)
2176 : {
2177 0 : CPLDebug("HTTP", "Using HTTP/2 for HTTPS when possible");
2178 0 : bDebugEmitted = true;
2179 : }
2180 :
2181 : // CURL_HTTP_VERSION_2TLS means for HTTPS connection, try to
2182 : // negotiate HTTP/2 with the server (and fallback to HTTP/1.1
2183 : // otherwise), and for HTTP connection do HTTP/1
2184 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2185 : CURL_HTTP_VERSION_2TLS);
2186 : }
2187 2549 : }
2188 : }
2189 : else
2190 : {
2191 0 : CPLError(CE_Warning, CPLE_NotSupported, "HTTP_VERSION=%s not supported",
2192 : pszHttpVersion);
2193 : }
2194 :
2195 : // Default value is 1 since curl 7.50.2. But worth applying it on
2196 : // previous versions as well.
2197 : const char *pszTCPNoDelay =
2198 2549 : CSLFetchNameValueDef(papszOptions, "TCP_NODELAY", "1");
2199 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_NODELAY,
2200 : atoi(pszTCPNoDelay));
2201 :
2202 : /* Support control over HTTPAUTH */
2203 2549 : const char *pszHttpAuth = CSLFetchNameValue(papszOptions, "HTTPAUTH");
2204 2549 : if (pszHttpAuth == nullptr)
2205 2539 : pszHttpAuth = CPLGetConfigOption("GDAL_HTTP_AUTH", nullptr);
2206 2549 : if (pszHttpAuth == nullptr)
2207 : {
2208 : /* do nothing */;
2209 : }
2210 15 : else if (EQUAL(pszHttpAuth, "BASIC"))
2211 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2212 : CURLAUTH_BASIC);
2213 15 : else if (EQUAL(pszHttpAuth, "NTLM"))
2214 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2215 : CURLAUTH_NTLM);
2216 15 : else if (EQUAL(pszHttpAuth, "ANY"))
2217 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
2218 15 : else if (EQUAL(pszHttpAuth, "ANYSAFE"))
2219 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2220 : CURLAUTH_ANYSAFE);
2221 15 : else if (EQUAL(pszHttpAuth, "BEARER"))
2222 : {
2223 15 : const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef(
2224 : papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES");
2225 : const bool bAuthorizationHeaderAllowed =
2226 15 : CPLTestBool(pszAuthorizationHeaderAllowed);
2227 15 : if (bAuthorizationHeaderAllowed)
2228 : {
2229 : const char *pszBearer =
2230 12 : CSLFetchNameValue(papszOptions, "HTTP_BEARER");
2231 12 : if (pszBearer == nullptr)
2232 5 : pszBearer = CPLGetConfigOption("GDAL_HTTP_BEARER", nullptr);
2233 12 : if (pszBearer != nullptr)
2234 12 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XOAUTH2_BEARER,
2235 : pszBearer);
2236 12 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2237 : CURLAUTH_BEARER);
2238 : }
2239 : }
2240 0 : else if (EQUAL(pszHttpAuth, "NEGOTIATE"))
2241 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2242 : CURLAUTH_NEGOTIATE);
2243 : else
2244 : {
2245 0 : CPLError(CE_Warning, CPLE_AppDefined,
2246 : "Unsupported HTTPAUTH value '%s', ignored.", pszHttpAuth);
2247 : }
2248 :
2249 : const char *pszGssDelegation =
2250 2549 : CSLFetchNameValue(papszOptions, "GSSAPI_DELEGATION");
2251 2549 : if (pszGssDelegation == nullptr)
2252 : pszGssDelegation =
2253 2549 : CPLGetConfigOption("GDAL_GSSAPI_DELEGATION", nullptr);
2254 2549 : if (pszGssDelegation == nullptr)
2255 : {
2256 : }
2257 0 : else if (EQUAL(pszGssDelegation, "NONE"))
2258 : {
2259 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2260 : CURLGSSAPI_DELEGATION_NONE);
2261 : }
2262 0 : else if (EQUAL(pszGssDelegation, "POLICY"))
2263 : {
2264 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2265 : CURLGSSAPI_DELEGATION_POLICY_FLAG);
2266 : }
2267 0 : else if (EQUAL(pszGssDelegation, "ALWAYS"))
2268 : {
2269 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2270 : CURLGSSAPI_DELEGATION_FLAG);
2271 : }
2272 : else
2273 : {
2274 0 : CPLError(CE_Warning, CPLE_AppDefined,
2275 : "Unsupported GSSAPI_DELEGATION value '%s', ignored.",
2276 : pszGssDelegation);
2277 : }
2278 :
2279 : // Support use of .netrc - default enabled.
2280 2549 : const char *pszHttpNetrc = CSLFetchNameValue(papszOptions, "NETRC");
2281 2549 : if (pszHttpNetrc == nullptr)
2282 2549 : pszHttpNetrc = CPLGetConfigOption("GDAL_HTTP_NETRC", "YES");
2283 2549 : if (pszHttpNetrc == nullptr || CPLTestBool(pszHttpNetrc))
2284 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC, 1L);
2285 :
2286 : // Custom .netrc file location
2287 : const char *pszHttpNetrcFile =
2288 2549 : CSLFetchNameValue(papszOptions, "NETRC_FILE");
2289 2549 : if (pszHttpNetrcFile == nullptr)
2290 2549 : pszHttpNetrcFile = CPLGetConfigOption("GDAL_HTTP_NETRC_FILE", nullptr);
2291 2549 : if (pszHttpNetrcFile)
2292 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC_FILE,
2293 : pszHttpNetrcFile);
2294 :
2295 : // Support setting userid:password.
2296 2549 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
2297 2549 : if (pszUserPwd == nullptr)
2298 2511 : pszUserPwd = CPLGetConfigOption("GDAL_HTTP_USERPWD", nullptr);
2299 2549 : if (pszUserPwd != nullptr)
2300 38 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERPWD, pszUserPwd);
2301 :
2302 : // Set Proxy parameters.
2303 2549 : const char *pszProxy = CSLFetchNameValue(papszOptions, "PROXY");
2304 2549 : if (pszProxy == nullptr)
2305 2549 : pszProxy = CPLGetConfigOption("GDAL_HTTP_PROXY", nullptr);
2306 2549 : if (pszProxy)
2307 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszProxy);
2308 :
2309 2549 : const char *pszHttpsProxy = CSLFetchNameValue(papszOptions, "HTTPS_PROXY");
2310 2549 : if (pszHttpsProxy == nullptr)
2311 2549 : pszHttpsProxy = CPLGetConfigOption("GDAL_HTTPS_PROXY", nullptr);
2312 2549 : if (pszHttpsProxy && (STARTS_WITH(pszURL, "https")))
2313 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszHttpsProxy);
2314 :
2315 : const char *pszProxyUserPwd =
2316 2549 : CSLFetchNameValue(papszOptions, "PROXYUSERPWD");
2317 2549 : if (pszProxyUserPwd == nullptr)
2318 2549 : pszProxyUserPwd = CPLGetConfigOption("GDAL_HTTP_PROXYUSERPWD", nullptr);
2319 2549 : if (pszProxyUserPwd)
2320 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYUSERPWD,
2321 : pszProxyUserPwd);
2322 :
2323 : // Support control over PROXYAUTH.
2324 2549 : const char *pszProxyAuth = CSLFetchNameValue(papszOptions, "PROXYAUTH");
2325 2549 : if (pszProxyAuth == nullptr)
2326 2549 : pszProxyAuth = CPLGetConfigOption("GDAL_PROXY_AUTH", nullptr);
2327 2549 : if (pszProxyAuth == nullptr)
2328 : {
2329 : // Do nothing.
2330 : }
2331 0 : else if (EQUAL(pszProxyAuth, "BASIC"))
2332 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2333 : CURLAUTH_BASIC);
2334 0 : else if (EQUAL(pszProxyAuth, "NTLM"))
2335 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2336 : CURLAUTH_NTLM);
2337 0 : else if (EQUAL(pszProxyAuth, "DIGEST"))
2338 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2339 : CURLAUTH_DIGEST);
2340 0 : else if (EQUAL(pszProxyAuth, "ANY"))
2341 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2342 : CURLAUTH_ANY);
2343 0 : else if (EQUAL(pszProxyAuth, "ANYSAFE"))
2344 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2345 : CURLAUTH_ANYSAFE);
2346 0 : else if (EQUAL(pszProxyAuth, "NEGOTIATE"))
2347 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2348 : CURLAUTH_NEGOTIATE);
2349 : else
2350 : {
2351 0 : CPLError(CE_Warning, CPLE_AppDefined,
2352 : "Unsupported PROXYAUTH value '%s', ignored.", pszProxyAuth);
2353 : }
2354 :
2355 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SUPPRESS_CONNECT_HEADERS,
2356 : 1L);
2357 :
2358 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
2359 2549 : const char *pszUnrestrictedAuth = CPLGetConfigOption(
2360 : "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT",
2361 : "IF_SAME_HOST");
2362 2567 : if (!EQUAL(pszUnrestrictedAuth, "IF_SAME_HOST") &&
2363 18 : CPLTestBool(pszUnrestrictedAuth))
2364 : {
2365 9 : unchecked_curl_easy_setopt(http_handle, CURLOPT_UNRESTRICTED_AUTH, 1);
2366 : }
2367 :
2368 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXREDIRS, 10);
2369 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTREDIR,
2370 : CURL_REDIR_POST_ALL);
2371 :
2372 : // Set connect timeout.
2373 : const char *pszConnectTimeout =
2374 2549 : CSLFetchNameValue(papszOptions, "CONNECTTIMEOUT");
2375 2549 : if (pszConnectTimeout == nullptr)
2376 : pszConnectTimeout =
2377 2549 : CPLGetConfigOption("GDAL_HTTP_CONNECTTIMEOUT", nullptr);
2378 2549 : if (pszConnectTimeout != nullptr)
2379 : {
2380 : // coverity[tainted_data]
2381 1 : unchecked_curl_easy_setopt(
2382 : http_handle, CURLOPT_CONNECTTIMEOUT_MS,
2383 : static_cast<int>(1000 * CPLAtof(pszConnectTimeout)));
2384 : }
2385 :
2386 : // Set timeout.
2387 2549 : const char *pszTimeout = CSLFetchNameValue(papszOptions, "TIMEOUT");
2388 2549 : if (pszTimeout == nullptr)
2389 2259 : pszTimeout = CPLGetConfigOption("GDAL_HTTP_TIMEOUT", nullptr);
2390 2549 : if (pszTimeout != nullptr)
2391 : {
2392 : // coverity[tainted_data]
2393 299 : unchecked_curl_easy_setopt(
2394 : http_handle, CURLOPT_TIMEOUT_MS,
2395 : static_cast<int>(1000 * CPLAtof(pszTimeout)));
2396 : }
2397 :
2398 : // Set low speed time and limit.
2399 : const char *pszLowSpeedTime =
2400 2549 : CSLFetchNameValue(papszOptions, "LOW_SPEED_TIME");
2401 2549 : if (pszLowSpeedTime == nullptr)
2402 : pszLowSpeedTime =
2403 2549 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_TIME", nullptr);
2404 2549 : if (pszLowSpeedTime != nullptr)
2405 : {
2406 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_TIME,
2407 : atoi(pszLowSpeedTime));
2408 :
2409 : const char *pszLowSpeedLimit =
2410 0 : CSLFetchNameValue(papszOptions, "LOW_SPEED_LIMIT");
2411 0 : if (pszLowSpeedLimit == nullptr)
2412 : pszLowSpeedLimit =
2413 0 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_LIMIT", "1");
2414 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_LIMIT,
2415 : atoi(pszLowSpeedLimit));
2416 : }
2417 :
2418 : /* Disable some SSL verification */
2419 2549 : const char *pszUnsafeSSL = CSLFetchNameValue(papszOptions, "UNSAFESSL");
2420 2549 : if (pszUnsafeSSL == nullptr)
2421 2528 : pszUnsafeSSL = CPLGetConfigOption("GDAL_HTTP_UNSAFESSL", nullptr);
2422 2549 : if (pszUnsafeSSL != nullptr && CPLTestBool(pszUnsafeSSL))
2423 : {
2424 21 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYPEER, 0L);
2425 21 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYHOST, 0L);
2426 : }
2427 :
2428 : const char *pszUseCAPIStore =
2429 2549 : CSLFetchNameValue(papszOptions, "USE_CAPI_STORE");
2430 2549 : if (pszUseCAPIStore == nullptr)
2431 2549 : pszUseCAPIStore = CPLGetConfigOption("GDAL_HTTP_USE_CAPI_STORE", "NO");
2432 2549 : if (CPLTestBool(pszUseCAPIStore))
2433 : {
2434 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2435 : // Use certificates from Windows certificate store; requires
2436 : // crypt32.lib, OpenSSL crypto and ssl libraries.
2437 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_CTX_FUNCTION,
2438 : *CPL_ssl_ctx_callback);
2439 : #else // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2440 2 : CPLError(CE_Warning, CPLE_NotSupported,
2441 : "GDAL_HTTP_USE_CAPI_STORE requested, but libcurl too old, "
2442 : "non-Windows platform or OpenSSL missing.");
2443 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2444 : }
2445 :
2446 : // Enable OCSP stapling if requested.
2447 : const char *pszSSLVerifyStatus =
2448 2549 : CSLFetchNameValue(papszOptions, "SSL_VERIFYSTATUS");
2449 2549 : if (pszSSLVerifyStatus == nullptr)
2450 : pszSSLVerifyStatus =
2451 2549 : CPLGetConfigOption("GDAL_HTTP_SSL_VERIFYSTATUS", "NO");
2452 2549 : if (CPLTestBool(pszSSLVerifyStatus))
2453 : {
2454 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYSTATUS, 1L);
2455 : }
2456 :
2457 : // Custom path to SSL certificates.
2458 2549 : const char *pszCAInfo = CSLFetchNameValue(papszOptions, "CAINFO");
2459 2549 : if (pszCAInfo == nullptr)
2460 : // Name of GDAL environment variable for the CA Bundle path
2461 2549 : pszCAInfo = CPLGetConfigOption("GDAL_CURL_CA_BUNDLE", nullptr);
2462 2549 : if (pszCAInfo == nullptr)
2463 : // Name of environment variable used by the curl binary
2464 2549 : pszCAInfo = CPLGetConfigOption("CURL_CA_BUNDLE", nullptr);
2465 2549 : if (pszCAInfo == nullptr)
2466 : // Name of environment variable used by the curl binary (tested
2467 : // after CURL_CA_BUNDLE
2468 2549 : pszCAInfo = CPLGetConfigOption("SSL_CERT_FILE", nullptr);
2469 : #ifdef _WIN32
2470 : if (pszCAInfo == nullptr)
2471 : {
2472 : pszCAInfo = CPLFindWin32CurlCaBundleCrt();
2473 : }
2474 : #endif
2475 2549 : if (pszCAInfo != nullptr)
2476 : {
2477 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAINFO, pszCAInfo);
2478 : }
2479 :
2480 2549 : const char *pszCAPath = CSLFetchNameValue(papszOptions, "CAPATH");
2481 2549 : if (pszCAPath != nullptr)
2482 : {
2483 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAPATH, pszCAPath);
2484 : }
2485 :
2486 : // Support for SSL client certificates
2487 :
2488 : // Filename of the the client certificate
2489 2549 : const char *pszSSLCert = CSLFetchNameValue(papszOptions, "SSLCERT");
2490 2549 : if (!pszSSLCert)
2491 2549 : pszSSLCert = CPLGetConfigOption("GDAL_HTTP_SSLCERT", nullptr);
2492 2549 : if (pszSSLCert)
2493 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERT, pszSSLCert);
2494 :
2495 : // private key file for TLS and SSL client cert
2496 2549 : const char *pszSSLKey = CSLFetchNameValue(papszOptions, "SSLKEY");
2497 2549 : if (!pszSSLKey)
2498 2549 : pszSSLKey = CPLGetConfigOption("GDAL_HTTP_SSLKEY", nullptr);
2499 2549 : if (pszSSLKey)
2500 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLKEY, pszSSLKey);
2501 :
2502 : // type of client SSL certificate ("PEM", "DER", ...)
2503 2549 : const char *pszSSLCertType = CSLFetchNameValue(papszOptions, "SSLCERTTYPE");
2504 2549 : if (!pszSSLCertType)
2505 2549 : pszSSLCertType = CPLGetConfigOption("GDAL_HTTP_SSLCERTTYPE", nullptr);
2506 2549 : if (pszSSLCertType)
2507 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERTTYPE,
2508 : pszSSLCertType);
2509 :
2510 : // passphrase to private key
2511 2549 : const char *pszKeyPasswd = CSLFetchNameValue(papszOptions, "KEYPASSWD");
2512 2549 : if (!pszKeyPasswd)
2513 2549 : pszKeyPasswd = CPLGetConfigOption("GDAL_HTTP_KEYPASSWD", nullptr);
2514 2549 : if (pszKeyPasswd)
2515 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_KEYPASSWD,
2516 : pszKeyPasswd);
2517 :
2518 : /* Set Referer */
2519 2549 : const char *pszReferer = CSLFetchNameValue(papszOptions, "REFERER");
2520 2549 : if (pszReferer != nullptr)
2521 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_REFERER, pszReferer);
2522 :
2523 : /* Set User-Agent */
2524 2549 : const char *pszUserAgent = CSLFetchNameValue(papszOptions, "USERAGENT");
2525 2549 : if (pszUserAgent == nullptr)
2526 2504 : pszUserAgent = CPLGetConfigOption("GDAL_HTTP_USERAGENT",
2527 : gosDefaultUserAgent.c_str());
2528 2549 : if (pszUserAgent != nullptr && !EQUAL(pszUserAgent, ""))
2529 : {
2530 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERAGENT,
2531 : pszUserAgent);
2532 : }
2533 :
2534 : /* NOSIGNAL should be set to true for timeout to work in multithread
2535 : * environments on Unix, requires libcurl 7.10 or more recent.
2536 : * (this force avoiding the use of signal handlers)
2537 : */
2538 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOSIGNAL, 1);
2539 :
2540 : const char *pszFormFilePath =
2541 2549 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
2542 : const char *pszParametersCount =
2543 2549 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
2544 2549 : if (pszFormFilePath == nullptr && pszParametersCount == nullptr)
2545 : {
2546 : /* Set POST mode */
2547 2547 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
2548 2547 : if (pszPost != nullptr)
2549 : {
2550 159 : CPLDebug("HTTP", "These POSTFIELDS were sent:%.4000s", pszPost);
2551 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 1);
2552 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS,
2553 : pszPost);
2554 : }
2555 : }
2556 :
2557 : const char *pszCustomRequest =
2558 2549 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
2559 2549 : if (pszCustomRequest != nullptr)
2560 : {
2561 138 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CUSTOMREQUEST,
2562 : pszCustomRequest);
2563 : }
2564 :
2565 2549 : const char *pszCookie = CSLFetchNameValue(papszOptions, "COOKIE");
2566 2549 : if (pszCookie == nullptr)
2567 2549 : pszCookie = CPLGetConfigOption("GDAL_HTTP_COOKIE", nullptr);
2568 2549 : if (pszCookie != nullptr)
2569 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIE, pszCookie);
2570 :
2571 2549 : const char *pszCookieFile = CSLFetchNameValue(papszOptions, "COOKIEFILE");
2572 2549 : if (pszCookieFile == nullptr)
2573 2549 : pszCookieFile = CPLGetConfigOption("GDAL_HTTP_COOKIEFILE", nullptr);
2574 2549 : if (pszCookieFile != nullptr)
2575 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEFILE,
2576 : pszCookieFile);
2577 :
2578 2549 : const char *pszCookieJar = CSLFetchNameValue(papszOptions, "COOKIEJAR");
2579 2549 : if (pszCookieJar == nullptr)
2580 2549 : pszCookieJar = CPLGetConfigOption("GDAL_HTTP_COOKIEJAR", nullptr);
2581 2549 : if (pszCookieJar != nullptr)
2582 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEJAR,
2583 : pszCookieJar);
2584 :
2585 : // TCP keep-alive
2586 : const char *pszTCPKeepAlive =
2587 2549 : CSLFetchNameValue(papszOptions, "TCP_KEEPALIVE");
2588 2549 : if (pszTCPKeepAlive == nullptr)
2589 2549 : pszTCPKeepAlive = CPLGetConfigOption("GDAL_HTTP_TCP_KEEPALIVE", "YES");
2590 2549 : if (pszTCPKeepAlive != nullptr && CPLTestBool(pszTCPKeepAlive))
2591 : {
2592 : // Set keep-alive interval.
2593 2549 : int nKeepAliveInterval = 60;
2594 : const char *pszKeepAliveInterval =
2595 2549 : CSLFetchNameValue(papszOptions, "TCP_KEEPINTVL");
2596 2549 : if (pszKeepAliveInterval == nullptr)
2597 : pszKeepAliveInterval =
2598 2549 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPINTVL", nullptr);
2599 2549 : if (pszKeepAliveInterval != nullptr)
2600 1 : nKeepAliveInterval = atoi(pszKeepAliveInterval);
2601 :
2602 : // Set keep-alive idle wait time.
2603 2549 : int nKeepAliveIdle = 60;
2604 : const char *pszKeepAliveIdle =
2605 2549 : CSLFetchNameValue(papszOptions, "TCP_KEEPIDLE");
2606 2549 : if (pszKeepAliveIdle == nullptr)
2607 : pszKeepAliveIdle =
2608 2549 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPIDLE", nullptr);
2609 2549 : if (pszKeepAliveIdle != nullptr)
2610 1 : nKeepAliveIdle = atoi(pszKeepAliveIdle);
2611 :
2612 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPALIVE, 1L);
2613 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPINTVL,
2614 : nKeepAliveInterval);
2615 2549 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPIDLE,
2616 : nKeepAliveIdle);
2617 : }
2618 :
2619 2549 : struct curl_slist *headers = nullptr;
2620 2549 : const char *pszHeaderFile = CSLFetchNameValue(papszOptions, "HEADER_FILE");
2621 2549 : if (pszHeaderFile == nullptr)
2622 2547 : pszHeaderFile = CPLGetConfigOption("GDAL_HTTP_HEADER_FILE", nullptr);
2623 2549 : if (pszHeaderFile != nullptr)
2624 : {
2625 2 : VSILFILE *fp = nullptr;
2626 : // Do not allow /vsicurl/ access from /vsicurl because of
2627 : // GetCurlHandleFor() e.g. "/vsicurl/,HEADER_FILE=/vsicurl/,url= " would
2628 : // cause use of memory after free
2629 2 : if (!STARTS_WITH(pszHeaderFile, "/vsi") ||
2630 2 : STARTS_WITH(pszHeaderFile, "/vsimem/"))
2631 : {
2632 2 : fp = VSIFOpenL(pszHeaderFile, "rb");
2633 : }
2634 2 : if (fp == nullptr)
2635 : {
2636 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot read %s", pszHeaderFile);
2637 : }
2638 : else
2639 : {
2640 2 : const char *pszLine = nullptr;
2641 4 : while ((pszLine = CPLReadLineL(fp)) != nullptr)
2642 : {
2643 2 : headers = curl_slist_append(headers, pszLine);
2644 : }
2645 2 : VSIFCloseL(fp);
2646 : }
2647 : }
2648 :
2649 2549 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
2650 2549 : if (pszHeaders == nullptr)
2651 1955 : pszHeaders = CPLGetConfigOption("GDAL_HTTP_HEADERS", nullptr);
2652 2549 : if (pszHeaders)
2653 : {
2654 602 : bool bHeadersDone = false;
2655 : // Compatibility hack for "HEADERS=Accept: text/plain, application/json"
2656 602 : if (strstr(pszHeaders, "\r\n") == nullptr)
2657 : {
2658 543 : const char *pszComma = strchr(pszHeaders, ',');
2659 543 : if (pszComma != nullptr && strchr(pszComma, ':') == nullptr)
2660 : {
2661 208 : headers = curl_slist_append(headers, pszHeaders);
2662 208 : bHeadersDone = true;
2663 : }
2664 : }
2665 602 : if (!bHeadersDone)
2666 : {
2667 394 : const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef(
2668 : papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES");
2669 : const bool bAuthorizationHeaderAllowed =
2670 394 : CPLTestBool(pszAuthorizationHeaderAllowed);
2671 :
2672 : // We accept both raw headers with \r\n as a separator, or as
2673 : // a comma separated list of foo: bar values.
2674 : const CPLStringList aosTokens(
2675 394 : strstr(pszHeaders, "\r\n")
2676 59 : ? CSLTokenizeString2(pszHeaders, "\r\n", 0)
2677 847 : : CSLTokenizeString2(pszHeaders, ",", CSLT_HONOURSTRINGS));
2678 855 : for (int i = 0; i < aosTokens.size(); ++i)
2679 : {
2680 465 : if (bAuthorizationHeaderAllowed ||
2681 4 : !STARTS_WITH_CI(aosTokens[i], "Authorization:"))
2682 : {
2683 457 : headers = curl_slist_append(headers, aosTokens[i]);
2684 : }
2685 : }
2686 : }
2687 : }
2688 :
2689 2549 : return headers;
2690 : }
2691 :
2692 : /************************************************************************/
2693 : /* CPLHTTPIgnoreSigPipe() */
2694 : /************************************************************************/
2695 :
2696 : /* If using OpenSSL with Curl, openssl can cause SIGPIPE to be triggered */
2697 : /* As we set CURLOPT_NOSIGNAL = 1, we must manually handle this situation */
2698 :
2699 2755 : void *CPLHTTPIgnoreSigPipe()
2700 : {
2701 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2702 : struct sigaction old_pipe_act;
2703 : struct sigaction action;
2704 : /* Get previous handler */
2705 2755 : memset(&old_pipe_act, 0, sizeof(struct sigaction));
2706 2755 : sigaction(SIGPIPE, nullptr, &old_pipe_act);
2707 :
2708 : /* Install new handler */
2709 2755 : action = old_pipe_act;
2710 2755 : action.sa_handler = SIG_IGN;
2711 2755 : sigaction(SIGPIPE, &action, nullptr);
2712 :
2713 2755 : void *ret = CPLMalloc(sizeof(old_pipe_act));
2714 2755 : memcpy(ret, &old_pipe_act, sizeof(old_pipe_act));
2715 2755 : return ret;
2716 : #else
2717 : return nullptr;
2718 : #endif
2719 : }
2720 :
2721 : /************************************************************************/
2722 : /* CPLHTTPRestoreSigPipeHandler() */
2723 : /************************************************************************/
2724 :
2725 2755 : void CPLHTTPRestoreSigPipeHandler(void *old_handler)
2726 : {
2727 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2728 2755 : sigaction(SIGPIPE, static_cast<struct sigaction *>(old_handler), nullptr);
2729 2755 : CPLFree(old_handler);
2730 : #else
2731 : (void)old_handler;
2732 : #endif
2733 2755 : }
2734 :
2735 : #endif // def HAVE_CURL
2736 :
2737 : /************************************************************************/
2738 : /* CPLHTTPEnabled() */
2739 : /************************************************************************/
2740 :
2741 : /**
2742 : * \brief Return if CPLHTTP services can be useful
2743 : *
2744 : * Those services depend on GDAL being build with libcurl support.
2745 : *
2746 : * @return TRUE if libcurl support is enabled
2747 : */
2748 8 : int CPLHTTPEnabled()
2749 :
2750 : {
2751 : #ifdef HAVE_CURL
2752 8 : return TRUE;
2753 : #else
2754 : return FALSE;
2755 : #endif
2756 : }
2757 :
2758 : /************************************************************************/
2759 : /* CPLHTTPCleanup() */
2760 : /************************************************************************/
2761 :
2762 : /**
2763 : * \brief Cleanup function to call at application termination
2764 : */
2765 941 : void CPLHTTPCleanup()
2766 :
2767 : {
2768 : #ifdef HAVE_CURL
2769 941 : if (!hSessionMapMutex)
2770 940 : return;
2771 :
2772 : {
2773 2 : CPLMutexHolder oHolder(&hSessionMapMutex);
2774 1 : if (poSessionMap)
2775 : {
2776 0 : for (auto &kv : *poSessionMap)
2777 : {
2778 0 : curl_easy_cleanup(kv.second);
2779 : }
2780 0 : delete poSessionMap;
2781 0 : poSessionMap = nullptr;
2782 : }
2783 1 : if (poSessionMultiMap)
2784 : {
2785 0 : for (auto &kv : *poSessionMultiMap)
2786 : {
2787 0 : VSICURLMultiCleanup(kv.second);
2788 : }
2789 0 : delete poSessionMultiMap;
2790 0 : poSessionMultiMap = nullptr;
2791 : }
2792 : }
2793 :
2794 : // Not quite a safe sequence.
2795 1 : CPLDestroyMutex(hSessionMapMutex);
2796 1 : hSessionMapMutex = nullptr;
2797 :
2798 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2799 : // This cleanup must be absolutely done before CPLOpenSSLCleanup()
2800 : // for some unknown reason, but otherwise X509_free() in
2801 : // CPLWindowsCertificateListCleanup() will crash.
2802 : CPLWindowsCertificateListCleanup();
2803 : #endif
2804 :
2805 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
2806 : CPLOpenSSLCleanup();
2807 : #endif
2808 :
2809 : #endif
2810 : }
2811 :
2812 : /************************************************************************/
2813 : /* CPLHTTPDestroyResult() */
2814 : /************************************************************************/
2815 :
2816 : /**
2817 : * \brief Clean the memory associated with the return value of CPLHTTPFetch()
2818 : *
2819 : * @param psResult pointer to the return value of CPLHTTPFetch()
2820 : */
2821 2154 : void CPLHTTPDestroyResult(CPLHTTPResult *psResult)
2822 :
2823 : {
2824 2154 : if (psResult)
2825 : {
2826 1967 : CPLFree(psResult->pabyData);
2827 1967 : CPLFree(psResult->pszErrBuf);
2828 1967 : CPLFree(psResult->pszContentType);
2829 1967 : CSLDestroy(psResult->papszHeaders);
2830 :
2831 2024 : for (int i = 0; i < psResult->nMimePartCount; i++)
2832 : {
2833 57 : CSLDestroy(psResult->pasMimePart[i].papszHeaders);
2834 : }
2835 1967 : CPLFree(psResult->pasMimePart);
2836 :
2837 1967 : CPLFree(psResult);
2838 : }
2839 2154 : }
2840 :
2841 : /************************************************************************/
2842 : /* CPLHTTPParseMultipartMime() */
2843 : /************************************************************************/
2844 :
2845 : /**
2846 : * \brief Parses a MIME multipart message.
2847 : *
2848 : * This function will iterate over each part and put it in a separate
2849 : * element of the pasMimePart array of the provided psResult structure.
2850 : *
2851 : * @param psResult pointer to the return value of CPLHTTPFetch()
2852 : * @return TRUE if the message contains MIME multipart message.
2853 : */
2854 45 : int CPLHTTPParseMultipartMime(CPLHTTPResult *psResult)
2855 :
2856 : {
2857 : /* -------------------------------------------------------------------- */
2858 : /* Is it already done? */
2859 : /* -------------------------------------------------------------------- */
2860 45 : if (psResult->nMimePartCount > 0)
2861 5 : return TRUE;
2862 :
2863 : /* -------------------------------------------------------------------- */
2864 : /* Find the boundary setting in the content type. */
2865 : /* -------------------------------------------------------------------- */
2866 40 : const char *pszBound = nullptr;
2867 :
2868 40 : if (psResult->pszContentType != nullptr)
2869 38 : pszBound = strstr(psResult->pszContentType, "boundary=");
2870 :
2871 40 : if (pszBound == nullptr)
2872 : {
2873 2 : CPLError(CE_Failure, CPLE_AppDefined,
2874 : "Unable to parse multi-part mime, no boundary setting.");
2875 2 : return FALSE;
2876 : }
2877 :
2878 76 : CPLString osBoundary;
2879 : char **papszTokens =
2880 38 : CSLTokenizeStringComplex(pszBound + 9, "\n ;", TRUE, FALSE);
2881 :
2882 38 : if (CSLCount(papszTokens) == 0 || strlen(papszTokens[0]) == 0)
2883 : {
2884 1 : CPLError(CE_Failure, CPLE_AppDefined,
2885 : "Unable to parse multi-part mime, boundary not parsable.");
2886 1 : CSLDestroy(papszTokens);
2887 1 : return FALSE;
2888 : }
2889 :
2890 37 : osBoundary = "--";
2891 37 : osBoundary += papszTokens[0];
2892 37 : CSLDestroy(papszTokens);
2893 :
2894 : /* -------------------------------------------------------------------- */
2895 : /* Find the start of the first chunk. */
2896 : /* -------------------------------------------------------------------- */
2897 37 : char *pszNext = psResult->pabyData
2898 37 : ? strstr(reinterpret_cast<char *>(psResult->pabyData),
2899 : osBoundary.c_str())
2900 37 : : nullptr;
2901 :
2902 37 : if (pszNext == nullptr)
2903 : {
2904 1 : CPLError(CE_Failure, CPLE_AppDefined, "No parts found.");
2905 1 : return FALSE;
2906 : }
2907 :
2908 36 : pszNext += osBoundary.size();
2909 146 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2910 110 : pszNext++;
2911 36 : if (*pszNext == '\r')
2912 30 : pszNext++;
2913 36 : if (*pszNext == '\n')
2914 36 : pszNext++;
2915 :
2916 : /* -------------------------------------------------------------------- */
2917 : /* Loop over parts... */
2918 : /* -------------------------------------------------------------------- */
2919 : while (true)
2920 : {
2921 57 : psResult->nMimePartCount++;
2922 57 : psResult->pasMimePart = static_cast<CPLMimePart *>(
2923 114 : CPLRealloc(psResult->pasMimePart,
2924 57 : sizeof(CPLMimePart) * psResult->nMimePartCount));
2925 :
2926 57 : CPLMimePart *psPart =
2927 57 : psResult->pasMimePart + psResult->nMimePartCount - 1;
2928 :
2929 57 : memset(psPart, 0, sizeof(CPLMimePart));
2930 :
2931 : /* --------------------------------------------------------------------
2932 : */
2933 : /* Collect headers. */
2934 : /* --------------------------------------------------------------------
2935 : */
2936 145 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2937 : {
2938 90 : if (!STARTS_WITH(pszNext, "Content-"))
2939 : {
2940 1 : break;
2941 : }
2942 89 : char *pszEOL = strstr(pszNext, "\n");
2943 :
2944 89 : if (pszEOL == nullptr)
2945 : {
2946 1 : CPLError(CE_Failure, CPLE_AppDefined,
2947 : "Error while parsing multipart content (at line %d)",
2948 : __LINE__);
2949 1 : return FALSE;
2950 : }
2951 :
2952 88 : *pszEOL = '\0';
2953 88 : bool bRestoreAntislashR = false;
2954 88 : if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r')
2955 : {
2956 80 : bRestoreAntislashR = true;
2957 80 : pszEOL[-1] = '\0';
2958 : }
2959 88 : char *pszKey = nullptr;
2960 88 : const char *pszValue = CPLParseNameValue(pszNext, &pszKey);
2961 88 : if (pszKey && pszValue)
2962 : {
2963 88 : psPart->papszHeaders =
2964 88 : CSLSetNameValue(psPart->papszHeaders, pszKey, pszValue);
2965 : }
2966 88 : CPLFree(pszKey);
2967 88 : if (bRestoreAntislashR)
2968 80 : pszEOL[-1] = '\r';
2969 88 : *pszEOL = '\n';
2970 :
2971 88 : pszNext = pszEOL + 1;
2972 : }
2973 :
2974 56 : if (*pszNext == '\r')
2975 54 : pszNext++;
2976 56 : if (*pszNext == '\n')
2977 54 : pszNext++;
2978 :
2979 : /* --------------------------------------------------------------------
2980 : */
2981 : /* Work out the data block size. */
2982 : /* --------------------------------------------------------------------
2983 : */
2984 56 : psPart->pabyData = reinterpret_cast<GByte *>(pszNext);
2985 :
2986 56 : int nBytesAvail = psResult->nDataLen -
2987 56 : static_cast<int>(pszNext - reinterpret_cast<char *>(
2988 56 : psResult->pabyData));
2989 :
2990 6579040 : while (nBytesAvail > 0 &&
2991 3289520 : (*pszNext != '-' ||
2992 109 : strncmp(pszNext, osBoundary, osBoundary.size()) != 0))
2993 : {
2994 3289470 : pszNext++;
2995 3289470 : nBytesAvail--;
2996 : }
2997 :
2998 56 : if (nBytesAvail == 0)
2999 : {
3000 2 : CPLError(CE_Failure, CPLE_AppDefined,
3001 : "Error while parsing multipart content (at line %d)",
3002 : __LINE__);
3003 2 : return FALSE;
3004 : }
3005 :
3006 54 : psPart->nDataLen = static_cast<int>(
3007 54 : pszNext - reinterpret_cast<char *>(psPart->pabyData));
3008 : // Normally the part should end with "\r\n--boundary_marker"
3009 54 : if (psPart->nDataLen >= 2 && pszNext[-2] == '\r' && pszNext[-1] == '\n')
3010 : {
3011 46 : psPart->nDataLen -= 2;
3012 : }
3013 :
3014 54 : pszNext += osBoundary.size();
3015 :
3016 54 : if (STARTS_WITH(pszNext, "--"))
3017 : {
3018 31 : break;
3019 : }
3020 :
3021 23 : if (*pszNext == '\r')
3022 19 : pszNext++;
3023 23 : if (*pszNext == '\n')
3024 21 : pszNext++;
3025 : else
3026 : {
3027 2 : CPLError(CE_Failure, CPLE_AppDefined,
3028 : "Error while parsing multipart content (at line %d)",
3029 : __LINE__);
3030 2 : return FALSE;
3031 : }
3032 21 : }
3033 :
3034 31 : return TRUE;
3035 : }
3036 :
3037 : #if defined(__GNUC__)
3038 : #pragma GCC diagnostic pop
3039 : #endif
|
