Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: libcurl based HTTP client
4 : * Purpose: libcurl based HTTP client
5 : * Author: Frank Warmerdam, warmerdam@pobox.com
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2006, Frank Warmerdam
9 : * Copyright (c) 2008-2013, Even Rouault <even dot rouault at spatialys.com>
10 : *
11 : * SPDX-License-Identifier: MIT
12 : ****************************************************************************/
13 :
14 : #include "cpl_port.h"
15 : #include "cpl_http.h"
16 :
17 : #include <cstddef>
18 : #include <cstring>
19 :
20 : #include <algorithm>
21 : #include <array>
22 : #include <map>
23 : #include <mutex>
24 : #include <string>
25 : #include <vector>
26 :
27 : #include "cpl_http.h"
28 : #include "cpl_error.h"
29 : #include "cpl_multiproc.h"
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_vsil_curl_class.h"
32 :
33 : // gcc or clang complains about C-style cast in #define like
34 : // CURL_ZERO_TERMINATED
35 : #if defined(__GNUC__)
36 : #pragma GCC diagnostic push
37 : #pragma GCC diagnostic ignored "-Wold-style-cast"
38 : #endif
39 :
40 : #ifdef HAVE_CURL
41 :
42 : #include "cpl_curl_priv.h"
43 :
44 : #ifdef HAVE_OPENSSL_CRYPTO
45 : #include <openssl/err.h>
46 : #include <openssl/ssl.h>
47 : #include <openssl/x509v3.h>
48 :
49 : #if defined(_WIN32)
50 : #include <wincrypt.h>
51 : #endif
52 :
53 : #endif
54 :
55 : #ifdef HAVE_SIGACTION
56 : #include <signal.h>
57 : #endif
58 :
59 : #define unchecked_curl_easy_setopt(handle, opt, param) \
60 : CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param))
61 :
62 : #endif // HAVE_CURL
63 :
64 : // list of named persistent http sessions
65 :
66 : #ifdef HAVE_CURL
67 : static std::map<CPLString, CURL *> *poSessionMap = nullptr;
68 : static std::map<CPLString, CURLM *> *poSessionMultiMap = nullptr;
69 : static CPLMutex *hSessionMapMutex = nullptr;
70 : static bool bHasCheckVersion = false;
71 : static bool bSupportGZip = false;
72 : static bool bSupportHTTP2 = false;
73 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
74 : static std::vector<X509 *> *poWindowsCertificateList = nullptr;
75 :
76 : #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
77 : #define EVP_PKEY_get0_RSA(x) (x->pkey.rsa)
78 : #define EVP_PKEY_get0_DSA(x) (x->pkey.dsa)
79 : #define X509_get_extension_flags(x) (x->ex_flags)
80 : #define X509_get_key_usage(x) (x->ex_kusage)
81 : #define X509_get_extended_key_usage(x) (x->ex_xkusage)
82 : #endif
83 :
84 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
85 :
86 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
87 :
88 : // Ported from https://curl.haxx.se/libcurl/c/opensslthreadlock.html
89 : static CPLMutex **pahSSLMutex = nullptr;
90 :
91 : static void CPLOpenSSLLockingFunction(int mode, int n, const char * /*file*/,
92 : int /*line*/)
93 : {
94 : if (mode & CRYPTO_LOCK)
95 : {
96 : CPLAcquireMutex(pahSSLMutex[n], 3600.0);
97 : }
98 : else
99 : {
100 : CPLReleaseMutex(pahSSLMutex[n]);
101 : }
102 : }
103 :
104 : static unsigned long CPLOpenSSLIdCallback(void)
105 : {
106 : return static_cast<unsigned long>(CPLGetPID());
107 : }
108 :
109 : static void CPLOpenSSLInit()
110 : {
111 : if (strstr(curl_version(), "OpenSSL") &&
112 : CPLTestBool(CPLGetConfigOption("CPL_OPENSSL_INIT_ENABLED", "YES")) &&
113 : CRYPTO_get_id_callback() == nullptr)
114 : {
115 : pahSSLMutex = static_cast<CPLMutex **>(
116 : CPLMalloc(CRYPTO_num_locks() * sizeof(CPLMutex *)));
117 : for (int i = 0; i < CRYPTO_num_locks(); i++)
118 : {
119 : pahSSLMutex[i] = CPLCreateMutex();
120 : CPLReleaseMutex(pahSSLMutex[i]);
121 : }
122 : CRYPTO_set_id_callback(CPLOpenSSLIdCallback);
123 : CRYPTO_set_locking_callback(CPLOpenSSLLockingFunction);
124 : }
125 : }
126 :
127 : static void CPLOpenSSLCleanup()
128 : {
129 : if (pahSSLMutex)
130 : {
131 : for (int i = 0; i < CRYPTO_num_locks(); i++)
132 : {
133 : CPLDestroyMutex(pahSSLMutex[i]);
134 : }
135 : CPLFree(pahSSLMutex);
136 : pahSSLMutex = nullptr;
137 : CRYPTO_set_id_callback(nullptr);
138 : CRYPTO_set_locking_callback(nullptr);
139 : }
140 : }
141 :
142 : #endif
143 :
144 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
145 :
146 : /************************************************************************/
147 : /* CPLWindowsCertificateListCleanup() */
148 : /************************************************************************/
149 :
150 : static void CPLWindowsCertificateListCleanup()
151 : {
152 : if (poWindowsCertificateList)
153 : {
154 : for (auto &&pX509 : *poWindowsCertificateList)
155 : {
156 : X509_free(pX509);
157 : }
158 : delete poWindowsCertificateList;
159 : poWindowsCertificateList = nullptr;
160 : }
161 : }
162 :
163 : /************************************************************************/
164 : /* LoadCAPICertificates() */
165 : /************************************************************************/
166 :
167 : static CPLErr LoadCAPICertificates(const char *pszName,
168 : std::vector<X509 *> *poCertificateList)
169 : {
170 : CPLAssert(pszName);
171 : CPLAssert(poCertificateList);
172 :
173 : HCERTSTORE pCertStore = CertOpenSystemStore(
174 : reinterpret_cast<HCRYPTPROV_LEGACY>(nullptr), pszName);
175 : if (pCertStore == nullptr)
176 : {
177 : CPLError(CE_Failure, CPLE_AppDefined,
178 : "CPLLoadCAPICertificates(): Unable open system "
179 : "certificate store %s.",
180 : pszName);
181 : return CE_Failure;
182 : }
183 :
184 : PCCERT_CONTEXT pCertificate =
185 : CertEnumCertificatesInStore(pCertStore, nullptr);
186 : while (pCertificate != nullptr)
187 : {
188 : X509 *pX509 = d2i_X509(
189 : nullptr,
190 : const_cast<unsigned char const **>(&pCertificate->pbCertEncoded),
191 : pCertificate->cbCertEncoded);
192 : if (pX509 == nullptr)
193 : {
194 : CPLError(CE_Warning, CPLE_AppDefined,
195 : "CPLLoadCAPICertificates(): CertEnumCertificatesInStore() "
196 : "returned a null certificate, skipping.");
197 : }
198 : else
199 : {
200 : #ifdef DEBUG_VERBOSE
201 : char szSubject[256] = {0};
202 : CPLString osSubject;
203 : X509_NAME *pName = X509_get_subject_name(pX509);
204 : if (pName)
205 : {
206 : X509_NAME_oneline(pName, szSubject, sizeof(szSubject));
207 : osSubject = szSubject;
208 : }
209 : if (!osSubject.empty())
210 : CPLDebug("HTTP", "SSL Certificate: %s", osSubject.c_str());
211 : #endif
212 : poCertificateList->push_back(pX509);
213 : }
214 : pCertificate = CertEnumCertificatesInStore(pCertStore, pCertificate);
215 : }
216 : CertCloseStore(pCertStore, 0);
217 : return CE_None;
218 : }
219 :
220 : /************************************************************************/
221 : /* CPL_ssl_ctx_callback() */
222 : /************************************************************************/
223 :
224 : // Load certificates from Windows Crypto API store.
225 : static CURLcode CPL_ssl_ctx_callback(CURL *, void *pSSL, void *)
226 : {
227 : SSL_CTX *pSSL_CTX = static_cast<SSL_CTX *>(pSSL);
228 : if (pSSL_CTX == nullptr)
229 : {
230 : CPLError(CE_Failure, CPLE_AppDefined,
231 : "CPL_ssl_ctx_callback(): OpenSSL context pointer is NULL.");
232 : return CURLE_ABORTED_BY_CALLBACK;
233 : }
234 :
235 : static std::mutex goMutex;
236 : {
237 : std::lock_guard<std::mutex> oLock(goMutex);
238 : if (poWindowsCertificateList == nullptr)
239 : {
240 : poWindowsCertificateList = new std::vector<X509 *>();
241 : if (!poWindowsCertificateList)
242 : {
243 : CPLError(CE_Failure, CPLE_AppDefined,
244 : "CPL_ssl_ctx_callback(): Unable to allocate "
245 : "structure to hold certificates.");
246 : return CURLE_FAILED_INIT;
247 : }
248 :
249 : const std::array<const char *, 3> aszStores{
250 : {"CA", "AuthRoot", "ROOT"}};
251 : for (auto &&pszStore : aszStores)
252 : {
253 : if (LoadCAPICertificates(pszStore, poWindowsCertificateList) ==
254 : CE_Failure)
255 : {
256 : CPLError(
257 : CE_Failure, CPLE_AppDefined,
258 : "CPL_ssl_ctx_callback(): Unable to load certificates "
259 : "from '%s' store.",
260 : pszStore);
261 : return CURLE_FAILED_INIT;
262 : }
263 : }
264 :
265 : CPLDebug("HTTP", "Loading %d certificates from Windows store.",
266 : static_cast<int>(poWindowsCertificateList->size()));
267 : }
268 : }
269 :
270 : X509_STORE *pX509Store = SSL_CTX_get_cert_store(pSSL_CTX);
271 : for (X509 *x509 : *poWindowsCertificateList)
272 : X509_STORE_add_cert(pX509Store, x509);
273 :
274 : return CURLE_OK;
275 : }
276 :
277 : #endif // defined(_WIN32) && defined (HAVE_OPENSSL_CRYPTO)
278 :
279 : /************************************************************************/
280 : /* CheckCurlFeatures() */
281 : /************************************************************************/
282 :
283 2414 : static void CheckCurlFeatures()
284 : {
285 4828 : CPLMutexHolder oHolder(&hSessionMapMutex);
286 2414 : if (!bHasCheckVersion)
287 : {
288 6 : const char *pszVersion = curl_version();
289 6 : CPLDebug("HTTP", "%s", pszVersion);
290 6 : bSupportGZip = strstr(pszVersion, "zlib/") != nullptr;
291 6 : bSupportHTTP2 = strstr(curl_version(), "nghttp2/") != nullptr;
292 6 : bHasCheckVersion = true;
293 :
294 6 : curl_version_info_data *data = curl_version_info(CURLVERSION_NOW);
295 6 : if (data->version_num < LIBCURL_VERSION_NUM)
296 : {
297 0 : CPLError(CE_Warning, CPLE_AppDefined,
298 : "GDAL was built against curl %d.%d.%d, but is "
299 : "running against %s. Runtime failure is likely !",
300 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
301 : LIBCURL_VERSION_PATCH, data->version);
302 : }
303 6 : else if (data->version_num > LIBCURL_VERSION_NUM)
304 : {
305 0 : CPLDebug("HTTP",
306 : "GDAL was built against curl %d.%d.%d, but is "
307 : "running against %s.",
308 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
309 : LIBCURL_VERSION_PATCH, data->version);
310 : }
311 :
312 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
313 : CPLOpenSSLInit();
314 : #endif
315 : }
316 2414 : }
317 :
318 : /************************************************************************/
319 : /* CPLWriteFct() */
320 : /* */
321 : /* Append incoming text to our collection buffer, reallocating */
322 : /* it larger as needed. */
323 : /************************************************************************/
324 :
325 : class CPLHTTPResultWithLimit
326 : {
327 : public:
328 : CPLHTTPResult *psResult = nullptr;
329 : int nMaxFileSize = 0;
330 : };
331 :
332 12201 : static size_t CPLWriteFct(void *buffer, size_t size, size_t nmemb,
333 : void *reqInfo)
334 :
335 : {
336 12201 : CPLHTTPResultWithLimit *psResultWithLimit =
337 : static_cast<CPLHTTPResultWithLimit *>(reqInfo);
338 12201 : CPLHTTPResult *psResult = psResultWithLimit->psResult;
339 :
340 12201 : int nBytesToWrite = static_cast<int>(nmemb) * static_cast<int>(size);
341 12201 : int nNewSize = psResult->nDataLen + nBytesToWrite + 1;
342 12201 : if (nNewSize > psResult->nDataAlloc)
343 : {
344 1542 : psResult->nDataAlloc = static_cast<int>(nNewSize * 1.25 + 100);
345 : GByte *pabyNewData = static_cast<GByte *>(
346 1542 : VSIRealloc(psResult->pabyData, psResult->nDataAlloc));
347 1542 : if (pabyNewData == nullptr)
348 : {
349 0 : VSIFree(psResult->pabyData);
350 0 : psResult->pabyData = nullptr;
351 0 : psResult->pszErrBuf = CPLStrdup(CPLString().Printf(
352 : "Out of memory allocating %d bytes for HTTP data buffer.",
353 0 : psResult->nDataAlloc));
354 0 : psResult->nDataAlloc = psResult->nDataLen = 0;
355 :
356 0 : return 0;
357 : }
358 1542 : psResult->pabyData = pabyNewData;
359 : }
360 :
361 12201 : memcpy(psResult->pabyData + psResult->nDataLen, buffer, nBytesToWrite);
362 :
363 12201 : psResult->nDataLen += nBytesToWrite;
364 12201 : psResult->pabyData[psResult->nDataLen] = 0;
365 :
366 12201 : if (psResultWithLimit->nMaxFileSize > 0 &&
367 8 : psResult->nDataLen > psResultWithLimit->nMaxFileSize)
368 : {
369 1 : CPLError(CE_Failure, CPLE_AppDefined, "Maximum file size reached");
370 1 : return 0;
371 : }
372 :
373 12200 : return nmemb;
374 : }
375 :
376 : /************************************************************************/
377 : /* CPLHdrWriteFct() */
378 : /************************************************************************/
379 6896 : static size_t CPLHdrWriteFct(void *buffer, size_t size, size_t nmemb,
380 : void *reqInfo)
381 : {
382 6896 : CPLHTTPResult *psResult = static_cast<CPLHTTPResult *>(reqInfo);
383 : // Copy the buffer to a char* and initialize with zeros (zero
384 : // terminate as well).
385 6896 : size_t nBytes = size * nmemb;
386 6896 : char *pszHdr = static_cast<char *>(CPLCalloc(1, nBytes + 1));
387 6896 : memcpy(pszHdr, buffer, nBytes);
388 6896 : size_t nIdx = nBytes - 1;
389 : // Remove end of line characters
390 19350 : while (nIdx > 0 && (pszHdr[nIdx] == '\r' || pszHdr[nIdx] == '\n'))
391 : {
392 12454 : pszHdr[nIdx] = 0;
393 12454 : nIdx--;
394 : }
395 6896 : char *pszKey = nullptr;
396 6896 : const char *pszValue = CPLParseNameValue(pszHdr, &pszKey);
397 6896 : if (pszKey && pszValue)
398 : {
399 4700 : psResult->papszHeaders =
400 4700 : CSLAddNameValue(psResult->papszHeaders, pszKey, pszValue);
401 : }
402 6896 : CPLFree(pszHdr);
403 6896 : CPLFree(pszKey);
404 6896 : return nmemb;
405 : }
406 :
407 : /************************************************************************/
408 : /* CPLHTTPReadFunction() */
409 : /************************************************************************/
410 0 : static size_t CPLHTTPReadFunction(char *buffer, size_t size, size_t nitems,
411 : void *arg)
412 : {
413 0 : return VSIFReadL(buffer, size, nitems, static_cast<VSILFILE *>(arg));
414 : }
415 :
416 : /************************************************************************/
417 : /* CPLHTTPSeekFunction() */
418 : /************************************************************************/
419 0 : static int CPLHTTPSeekFunction(void *arg, curl_off_t offset, int origin)
420 : {
421 0 : if (VSIFSeekL(static_cast<VSILFILE *>(arg), offset, origin) == 0)
422 0 : return CURL_SEEKFUNC_OK;
423 : else
424 0 : return CURL_SEEKFUNC_FAIL;
425 : }
426 :
427 : /************************************************************************/
428 : /* CPLHTTPFreeFunction() */
429 : /************************************************************************/
430 0 : static void CPLHTTPFreeFunction(void *arg)
431 : {
432 0 : VSIFCloseL(static_cast<VSILFILE *>(arg));
433 0 : }
434 :
435 : typedef struct
436 : {
437 : GDALProgressFunc pfnProgress;
438 : void *pProgressArg;
439 : } CurlProcessData, *CurlProcessDataL;
440 :
441 394 : static int NewProcessFunction(void *p, curl_off_t dltotal, curl_off_t dlnow,
442 : curl_off_t ultotal, curl_off_t ulnow)
443 : {
444 394 : CurlProcessDataL pData = static_cast<CurlProcessDataL>(p);
445 394 : if (nullptr != pData && pData->pfnProgress)
446 : {
447 394 : if (dltotal > 0)
448 : {
449 0 : const double dfDone = double(dlnow) / dltotal;
450 0 : return pData->pfnProgress(dfDone, "Downloading ...",
451 : pData->pProgressArg) == TRUE
452 0 : ? 0
453 0 : : 1;
454 : }
455 394 : else if (ultotal > 0)
456 : {
457 0 : const double dfDone = double(ulnow) / ultotal;
458 0 : return pData->pfnProgress(dfDone, "Uploading ...",
459 : pData->pProgressArg) == TRUE
460 0 : ? 0
461 0 : : 1;
462 : }
463 : }
464 394 : return 0;
465 : }
466 :
467 : #endif /* def HAVE_CURL */
468 :
469 : /************************************************************************/
470 : /* CPLHTTPSetDefaultUserAgent() */
471 : /************************************************************************/
472 :
473 : static std::string gosDefaultUserAgent;
474 :
475 : /**
476 : * \brief Set the default user agent.
477 : *
478 : * GDAL core will by default call this method with "GDAL/x.y.z" where x.y.z
479 : * is the GDAL version number (during driver initialization). Applications may
480 : * override it.
481 : *
482 : * @param pszUserAgent String (or nullptr to cancel the default user agent)
483 : *
484 : * @since GDAL 3.7
485 : */
486 1295 : void CPLHTTPSetDefaultUserAgent(const char *pszUserAgent)
487 : {
488 1295 : gosDefaultUserAgent = pszUserAgent ? pszUserAgent : "";
489 1295 : }
490 :
491 : /************************************************************************/
492 : /* CPLHTTPGetOptionsFromEnv() */
493 : /************************************************************************/
494 :
495 : typedef struct
496 : {
497 : const char *pszEnvVar;
498 : const char *pszOptionName;
499 : } TupleEnvVarOptionName;
500 :
501 : constexpr TupleEnvVarOptionName asAssocEnvVarOptionName[] = {
502 : {"GDAL_HTTP_VERSION", "HTTP_VERSION"},
503 : {"GDAL_HTTP_CONNECTTIMEOUT", "CONNECTTIMEOUT"},
504 : {"GDAL_HTTP_TIMEOUT", "TIMEOUT"},
505 : {"GDAL_HTTP_LOW_SPEED_TIME", "LOW_SPEED_TIME"},
506 : {"GDAL_HTTP_LOW_SPEED_LIMIT", "LOW_SPEED_LIMIT"},
507 : {"GDAL_HTTP_USERPWD", "USERPWD"},
508 : {"GDAL_HTTP_PROXY", "PROXY"},
509 : {"GDAL_HTTPS_PROXY", "HTTPS_PROXY"},
510 : {"GDAL_HTTP_PROXYUSERPWD", "PROXYUSERPWD"},
511 : {"GDAL_PROXY_AUTH", "PROXYAUTH"},
512 : {"GDAL_HTTP_NETRC", "NETRC"},
513 : {"GDAL_HTTP_NETRC_FILE", "NETRC_FILE"},
514 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
515 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
516 : {"GDAL_HTTP_RETRY_CODES", "RETRY_CODES"},
517 : {"GDAL_CURL_CA_BUNDLE", "CAINFO"},
518 : {"CURL_CA_BUNDLE", "CAINFO"},
519 : {"SSL_CERT_FILE", "CAINFO"},
520 : {"GDAL_HTTP_HEADER_FILE", "HEADER_FILE"},
521 : {"GDAL_HTTP_CAPATH", "CAPATH"},
522 : {"GDAL_HTTP_SSL_VERIFYSTATUS", "SSL_VERIFYSTATUS"},
523 : {"GDAL_HTTP_USE_CAPI_STORE", "USE_CAPI_STORE"},
524 : {"GDAL_HTTP_HEADERS", "HEADERS"},
525 : {"GDAL_HTTP_HEADER_FILE", "HEADER_FILE"},
526 : {"GDAL_HTTP_AUTH", "HTTPAUTH"},
527 : {"GDAL_GSSAPI_DELEGATION", "GSSAPI_DELEGATION"},
528 : {"GDAL_HTTP_BEARER", "HTTP_BEARER"},
529 : {"GDAL_HTTP_COOKIE", "COOKIE"},
530 : {"GDAL_HTTP_COOKIEFILE", "COOKIEFILE"},
531 : {"GDAL_HTTP_COOKIEJAR", "COOKIEJAR"},
532 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
533 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
534 : {"GDAL_HTTP_TCP_KEEPALIVE", "TCP_KEEPALIVE"},
535 : {"GDAL_HTTP_TCP_KEEPIDLE", "TCP_KEEPIDLE"},
536 : {"GDAL_HTTP_TCP_KEEPINTVL", "TCP_KEEPINTVL"},
537 : };
538 :
539 1316 : char **CPLHTTPGetOptionsFromEnv(const char *pszFilename)
540 : {
541 2632 : CPLStringList aosOptions;
542 2632 : std::string osNonStreamingFilename;
543 1316 : if (pszFilename && STARTS_WITH(pszFilename, "/vsi"))
544 : {
545 : VSIFilesystemHandler *poFSHandler =
546 1314 : VSIFileManager::GetHandler(pszFilename);
547 : osNonStreamingFilename =
548 1314 : poFSHandler->GetNonStreamingFilename(pszFilename);
549 1314 : if (osNonStreamingFilename == pszFilename)
550 : {
551 1250 : osNonStreamingFilename.clear();
552 : }
553 : else
554 : {
555 : // CPLDebug("HTTP", "Non-streaming filename for %s: %s", pszFilename, osNonStreamingFilename.c_str());
556 : }
557 : }
558 47375 : for (const auto &sTuple : asAssocEnvVarOptionName)
559 : {
560 46059 : const char *pszVal = nullptr;
561 46059 : if (pszFilename)
562 : {
563 46056 : pszVal = VSIGetPathSpecificOption(pszFilename, sTuple.pszEnvVar,
564 : nullptr);
565 46060 : if (!pszVal && !osNonStreamingFilename.empty())
566 : {
567 2228 : pszVal = VSIGetPathSpecificOption(
568 2228 : osNonStreamingFilename.c_str(), sTuple.pszEnvVar, nullptr);
569 : }
570 : }
571 46062 : if (!pszVal)
572 : {
573 45900 : pszVal = CPLGetConfigOption(sTuple.pszEnvVar, nullptr);
574 : }
575 46061 : if (pszVal)
576 : {
577 160 : aosOptions.AddNameValue(sTuple.pszOptionName, pszVal);
578 : }
579 : }
580 2632 : return aosOptions.StealList();
581 : }
582 :
583 : /************************************************************************/
584 : /* CPLHTTPGetNewRetryDelay() */
585 : /************************************************************************/
586 :
587 : /** Return the new retry delay.
588 : *
589 : * This takes into account the HTTP response code, the previous delay, the
590 : * HTTP payload error message, the Curl error message and a potential list of
591 : * retriable HTTP codes.
592 : *
593 : * @param response_code HTTP response code (e.g. 400)
594 : * @param dfOldDelay Previous delay (nominally in second)
595 : * @param pszErrBuf HTTP response body of the failed request (may be NULL)
596 : * @param pszCurlError Curl error as returned by CURLOPT_ERRORBUFFER (may be NULL)
597 : * @param pszRetriableCodes nullptr to limit to the default hard-coded scenarios,
598 : * "ALL" to ask to retry for all non-200 codes, or a comma-separated list of
599 : * HTTP codes (e.g. "400,500") that are accepted for retry.
600 : * @return the new delay, or 0 if no retry should be attempted.
601 : */
602 215 : static double CPLHTTPGetNewRetryDelay(int response_code, double dfOldDelay,
603 : const char *pszErrBuf,
604 : const char *pszCurlError,
605 : const char *pszRetriableCodes)
606 : {
607 215 : bool bRetry = false;
608 215 : if (pszRetriableCodes && pszRetriableCodes[0])
609 : {
610 5 : bRetry = EQUAL(pszRetriableCodes, "ALL") ||
611 2 : strstr(pszRetriableCodes, CPLSPrintf("%d", response_code));
612 : }
613 212 : else if (response_code == 429 || response_code == 500 ||
614 208 : (response_code >= 502 && response_code <= 504) ||
615 : // S3 sends some client timeout errors as 400 Client Error
616 6 : (response_code == 400 && pszErrBuf &&
617 193 : strstr(pszErrBuf, "RequestTimeout")) ||
618 193 : (pszCurlError &&
619 193 : (strstr(pszCurlError, "Connection timed out") ||
620 193 : strstr(pszCurlError, "Operation timed out") ||
621 193 : strstr(pszCurlError, "Connection reset by peer") ||
622 193 : strstr(pszCurlError, "Connection was reset"))))
623 : {
624 23 : bRetry = true;
625 : }
626 215 : if (bRetry)
627 : {
628 : // 'Operation timed out': seen during some long running operation 'hang'
629 : // no error but no response from server and we are in the cURL loop
630 : // infinitely.
631 :
632 : // 'Connection was reset': was found with Azure: server resets
633 : // connection during TLS handshake (10054 error code). It seems like
634 : // the server process crashed or something forced TCP reset;
635 : // the request succeeds on retry.
636 :
637 : // Use an exponential backoff factor of 2 plus some random jitter
638 : // We don't care about cryptographic quality randomness, hence:
639 : // coverity[dont_call]
640 25 : return dfOldDelay * (2 + rand() * 0.5 / RAND_MAX);
641 : }
642 : else
643 : {
644 190 : return 0;
645 : }
646 : }
647 :
648 : /*! @cond Doxygen_Suppress */
649 :
650 : /************************************************************************/
651 : /* CPLHTTPRetryParameters() */
652 : /************************************************************************/
653 :
654 : /** Constructs a CPLHTTPRetryParameters instance from configuration
655 : * options or path-specific options.
656 : *
657 : * @param aosHTTPOptions HTTP options returned by CPLHTTPGetOptionsFromEnv()
658 : */
659 1125 : CPLHTTPRetryParameters::CPLHTTPRetryParameters(
660 1125 : const CPLStringList &aosHTTPOptions)
661 1125 : : nMaxRetry(atoi(aosHTTPOptions.FetchNameValueDef(
662 : "MAX_RETRY", CPLSPrintf("%d", CPL_HTTP_MAX_RETRY)))),
663 1125 : dfInitialDelay(CPLAtof(aosHTTPOptions.FetchNameValueDef(
664 : "RETRY_DELAY", CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)))),
665 2250 : osRetryCodes(aosHTTPOptions.FetchNameValueDef("RETRY_CODES", ""))
666 : {
667 1124 : }
668 :
669 : /************************************************************************/
670 : /* CPLHTTPRetryContext() */
671 : /************************************************************************/
672 :
673 : /** Constructor */
674 999 : CPLHTTPRetryContext::CPLHTTPRetryContext(const CPLHTTPRetryParameters &oParams)
675 999 : : m_oParameters(oParams), m_dfNextDelay(oParams.dfInitialDelay)
676 : {
677 999 : }
678 :
679 : /************************************************************************/
680 : /* CPLHTTPRetryContext::CanRetry() */
681 : /************************************************************************/
682 :
683 : /** Returns whether we can attempt a new retry, based on the retry counter,
684 : * and increment that counter.
685 : */
686 0 : bool CPLHTTPRetryContext::CanRetry()
687 : {
688 0 : if (m_nRetryCount >= m_oParameters.nMaxRetry)
689 0 : return false;
690 0 : m_nRetryCount++;
691 0 : return true;
692 : }
693 :
694 : /** Returns whether we can attempt a new retry, based on the retry counter,
695 : * the response code, payload and curl error buffers.
696 : *
697 : * If successful, the retry counter is incremented, and GetCurrentDelay()
698 : * returns the delay to apply with CPLSleep().
699 : */
700 197 : bool CPLHTTPRetryContext::CanRetry(int response_code, const char *pszErrBuf,
701 : const char *pszCurlError)
702 : {
703 197 : if (m_nRetryCount >= m_oParameters.nMaxRetry)
704 179 : return false;
705 18 : m_dfCurDelay = m_dfNextDelay;
706 18 : m_dfNextDelay = CPLHTTPGetNewRetryDelay(response_code, m_dfNextDelay,
707 : pszErrBuf, pszCurlError,
708 : m_oParameters.osRetryCodes.c_str());
709 18 : if (m_dfNextDelay == 0.0)
710 2 : return false;
711 16 : m_nRetryCount++;
712 16 : return true;
713 : }
714 :
715 : /************************************************************************/
716 : /* CPLHTTPRetryContext::GetCurrentDelay() */
717 : /************************************************************************/
718 :
719 : /** Returns the delay to apply. Only valid after a successful call to CanRetry() */
720 32 : double CPLHTTPRetryContext::GetCurrentDelay() const
721 : {
722 32 : if (m_nRetryCount == 0)
723 0 : CPLDebug("CPL",
724 : "GetCurrentDelay() should only be called after CanRetry()");
725 32 : return m_dfCurDelay;
726 : }
727 :
728 : /*! @endcond Doxygen_Suppress */
729 :
730 : #ifdef HAVE_CURL
731 :
732 : /************************************************************************/
733 : /* CPLHTTPEmitFetchDebug() */
734 : /************************************************************************/
735 :
736 1110 : static void CPLHTTPEmitFetchDebug(const char *pszURL,
737 : const char *pszExtraDebug = "")
738 : {
739 1110 : const char *pszArobase = strchr(pszURL, '@');
740 1110 : const char *pszSlash = strchr(pszURL, '/');
741 1110 : const char *pszColon = (pszSlash) ? strchr(pszSlash, ':') : nullptr;
742 1110 : if (pszArobase != nullptr && pszColon != nullptr &&
743 0 : pszArobase - pszColon > 0)
744 : {
745 : /* http://user:password@www.example.com */
746 0 : char *pszSanitizedURL = CPLStrdup(pszURL);
747 0 : pszSanitizedURL[pszColon - pszURL] = 0;
748 0 : CPLDebug("HTTP", "Fetch(%s:#password#%s%s)", pszSanitizedURL,
749 : pszArobase, pszExtraDebug);
750 0 : CPLFree(pszSanitizedURL);
751 : }
752 : else
753 : {
754 1110 : CPLDebug("HTTP", "Fetch(%s%s)", pszURL, pszExtraDebug);
755 : }
756 1110 : }
757 :
758 : #endif
759 :
760 : #ifdef HAVE_CURL
761 :
762 : /************************************************************************/
763 : /* class CPLHTTPPostFields */
764 : /************************************************************************/
765 :
766 : class CPLHTTPPostFields
767 : {
768 : public:
769 1110 : CPLHTTPPostFields() = default;
770 : CPLHTTPPostFields &operator=(const CPLHTTPPostFields &) = delete;
771 : CPLHTTPPostFields(const CPLHTTPPostFields &) = delete;
772 :
773 1110 : CPLErr Fill(CURL *http_handle, CSLConstList papszOptions)
774 : {
775 : // Fill POST form if present
776 : const char *pszFormFilePath =
777 1110 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
778 : const char *pszParametersCount =
779 1110 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
780 :
781 1110 : if (pszFormFilePath != nullptr || pszParametersCount != nullptr)
782 : {
783 2 : mime = curl_mime_init(http_handle);
784 2 : curl_mimepart *mimepart = curl_mime_addpart(mime);
785 2 : if (pszFormFilePath != nullptr)
786 : {
787 : const char *pszFormFileName =
788 1 : CSLFetchNameValue(papszOptions, "FORM_FILE_NAME");
789 1 : const char *pszFilename = CPLGetFilename(pszFormFilePath);
790 1 : if (pszFormFileName == nullptr)
791 : {
792 1 : pszFormFileName = pszFilename;
793 : }
794 :
795 : VSIStatBufL sStat;
796 1 : if (VSIStatL(pszFormFilePath, &sStat) == 0)
797 : {
798 0 : VSILFILE *mime_fp = VSIFOpenL(pszFormFilePath, "rb");
799 0 : if (mime_fp != nullptr)
800 : {
801 0 : curl_mime_name(mimepart, pszFormFileName);
802 0 : CPL_IGNORE_RET_VAL(
803 0 : curl_mime_filename(mimepart, pszFilename));
804 0 : curl_mime_data_cb(
805 : mimepart, sStat.st_size, CPLHTTPReadFunction,
806 : CPLHTTPSeekFunction, CPLHTTPFreeFunction, mime_fp);
807 : }
808 : else
809 : {
810 : osErrMsg = CPLSPrintf("Failed to open file %s",
811 0 : pszFormFilePath);
812 1 : return CE_Failure;
813 : }
814 :
815 0 : CPLDebug("HTTP", "Send file: %s, COPYNAME: %s",
816 : pszFormFilePath, pszFormFileName);
817 : }
818 : else
819 : {
820 : osErrMsg =
821 1 : CPLSPrintf("File '%s' not found", pszFormFilePath);
822 1 : return CE_Failure;
823 : }
824 : }
825 :
826 1 : int nParametersCount = 0;
827 1 : if (pszParametersCount != nullptr)
828 : {
829 1 : nParametersCount = atoi(pszParametersCount);
830 : }
831 :
832 2 : for (int i = 0; i < nParametersCount; ++i)
833 : {
834 2 : const char *pszKey = CSLFetchNameValue(
835 : papszOptions, CPLSPrintf("FORM_KEY_%d", i));
836 2 : const char *pszValue = CSLFetchNameValue(
837 : papszOptions, CPLSPrintf("FORM_VALUE_%d", i));
838 :
839 2 : if (nullptr == pszKey)
840 : {
841 : osErrMsg = CPLSPrintf("Key #%d is not exists. Maybe wrong "
842 : "count of form items",
843 1 : i);
844 1 : return CE_Failure;
845 : }
846 :
847 1 : if (nullptr == pszValue)
848 : {
849 : osErrMsg = CPLSPrintf("Value #%d is not exists. Maybe "
850 : "wrong count of form items",
851 0 : i);
852 0 : return CE_Failure;
853 : }
854 :
855 1 : mimepart = curl_mime_addpart(mime);
856 1 : curl_mime_name(mimepart, pszKey);
857 1 : CPL_IGNORE_RET_VAL(
858 1 : curl_mime_data(mimepart, pszValue, CURL_ZERO_TERMINATED));
859 :
860 1 : CPLDebug("HTTP", "COPYNAME: %s, COPYCONTENTS: %s", pszKey,
861 : pszValue);
862 : }
863 :
864 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MIMEPOST, mime);
865 : }
866 1108 : return CE_None;
867 : }
868 :
869 1110 : ~CPLHTTPPostFields()
870 1110 : {
871 1110 : if (mime != nullptr)
872 : {
873 2 : curl_mime_free(mime);
874 : }
875 1110 : }
876 :
877 2 : std::string GetErrorMessage() const
878 : {
879 2 : return osErrMsg;
880 : }
881 :
882 : private:
883 : curl_mime *mime = nullptr;
884 : std::string osErrMsg{};
885 : };
886 :
887 : /************************************************************************/
888 : /* CPLHTTPFetchCleanup() */
889 : /************************************************************************/
890 :
891 1110 : static void CPLHTTPFetchCleanup(CURL *http_handle, struct curl_slist *headers,
892 : const char *pszPersistent,
893 : CSLConstList papszOptions)
894 : {
895 1110 : if (CSLFetchNameValue(papszOptions, "POSTFIELDS"))
896 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 0);
897 1110 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, nullptr);
898 :
899 1110 : if (!pszPersistent)
900 613 : curl_easy_cleanup(http_handle);
901 :
902 1110 : curl_slist_free_all(headers);
903 1110 : }
904 : #endif // HAVE_CURL
905 :
906 : struct CPLHTTPFetchContext
907 : {
908 : std::vector<std::pair<CPLHTTPFetchCallbackFunc, void *>> stack{};
909 : };
910 :
911 : /************************************************************************/
912 : /* GetHTTPFetchContext() */
913 : /************************************************************************/
914 :
915 1302 : static CPLHTTPFetchContext *GetHTTPFetchContext(bool bAlloc)
916 : {
917 1302 : int bError = FALSE;
918 : CPLHTTPFetchContext *psCtx = static_cast<CPLHTTPFetchContext *>(
919 1302 : CPLGetTLSEx(CTLS_HTTPFETCHCALLBACK, &bError));
920 1302 : if (bError)
921 0 : return nullptr;
922 :
923 1302 : if (psCtx == nullptr && bAlloc)
924 : {
925 1 : const auto FreeFunc = [](void *pData)
926 1 : { delete static_cast<CPLHTTPFetchContext *>(pData); };
927 1 : psCtx = new CPLHTTPFetchContext();
928 1 : CPLSetTLSWithFreeFuncEx(CTLS_HTTPFETCHCALLBACK, psCtx, FreeFunc,
929 : &bError);
930 1 : if (bError)
931 : {
932 0 : delete psCtx;
933 0 : psCtx = nullptr;
934 : }
935 : }
936 1302 : return psCtx;
937 : }
938 :
939 : /************************************************************************/
940 : /* CPLHTTPSetFetchCallback() */
941 : /************************************************************************/
942 :
943 : static CPLHTTPFetchCallbackFunc gpsHTTPFetchCallbackFunc = nullptr;
944 : static void *gpHTTPFetchCallbackUserData = nullptr;
945 :
946 : /** Installs an alternate callback to the default implementation of
947 : * CPLHTTPFetchEx().
948 : *
949 : * This callback will be used by all threads, unless contextual callbacks are
950 : * installed with CPLHTTPPushFetchCallback().
951 : *
952 : * It is the responsibility of the caller to make sure this function is not
953 : * called concurrently, or during CPLHTTPFetchEx() execution.
954 : *
955 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called
956 : * (or NULL to restore default handler)
957 : * @param pUserData Last argument to provide to the pFunc callback.
958 : *
959 : * @since GDAL 3.2
960 : */
961 2 : void CPLHTTPSetFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
962 : {
963 2 : gpsHTTPFetchCallbackFunc = pFunc;
964 2 : gpHTTPFetchCallbackUserData = pUserData;
965 2 : }
966 :
967 : /************************************************************************/
968 : /* CPLHTTPPushFetchCallback() */
969 : /************************************************************************/
970 :
971 : /** Installs an alternate callback to the default implementation of
972 : * CPLHTTPFetchEx().
973 : *
974 : * This callback will only be used in the thread where this function has been
975 : * called. It must be un-installed by CPLHTTPPopFetchCallback(), which must also
976 : * be called from the same thread.
977 : *
978 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called.
979 : * @param pUserData Last argument to provide to the pFunc callback.
980 : * @return TRUE in case of success.
981 : *
982 : * @since GDAL 3.2
983 : */
984 1 : int CPLHTTPPushFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
985 : {
986 1 : auto psCtx = GetHTTPFetchContext(true);
987 1 : if (psCtx == nullptr)
988 0 : return false;
989 : psCtx->stack.emplace_back(
990 1 : std::pair<CPLHTTPFetchCallbackFunc, void *>(pFunc, pUserData));
991 1 : return true;
992 : }
993 :
994 : /************************************************************************/
995 : /* CPLHTTPPopFetchCallback() */
996 : /************************************************************************/
997 :
998 : /** Uninstalls a callback set by CPLHTTPPushFetchCallback().
999 : *
1000 : * @see CPLHTTPPushFetchCallback()
1001 : * @return TRUE in case of success.
1002 : * @since GDAL 3.2
1003 : */
1004 2 : int CPLHTTPPopFetchCallback(void)
1005 : {
1006 2 : auto psCtx = GetHTTPFetchContext(false);
1007 2 : if (psCtx == nullptr || psCtx->stack.empty())
1008 : {
1009 1 : CPLError(
1010 : CE_Failure, CPLE_AppDefined,
1011 : "CPLHTTPPushFetchCallback / CPLHTTPPopFetchCallback not balanced");
1012 1 : return false;
1013 : }
1014 : else
1015 : {
1016 1 : psCtx->stack.pop_back();
1017 1 : return true;
1018 : }
1019 : }
1020 :
1021 : /************************************************************************/
1022 : /* CPLHTTPFetch() */
1023 : /************************************************************************/
1024 :
1025 : // NOTE: when adding an option below, add it in asAssocEnvVarOptionName[]
1026 :
1027 : // clang-format off
1028 : /**
1029 : * \brief Fetch a document from an url and return in a string.
1030 : *
1031 : * Different options may be passed through the papszOptions function parameter,
1032 : * or for most of them through a configuration option:
1033 : * <ul>
1034 : * <li>CONNECTTIMEOUT=val, where
1035 : * val is in seconds (possibly with decimals). This is the maximum delay for the
1036 : * connection to be established before being aborted (GDAL >= 2.2).
1037 : * Corresponding configuration option: GDAL_HTTP_CONNECTTIMEOUT.
1038 : * </li>
1039 : * <li>TIMEOUT=val, where val is in seconds. This is the maximum delay for the
1040 : * whole request to complete before being aborted.
1041 : * Corresponding configuration option: GDAL_HTTP_TIMEOUT.
1042 : * </li>
1043 : * <li>LOW_SPEED_TIME=val,
1044 : * where val is in seconds. This is the maximum time where the transfer speed
1045 : * should be below the LOW_SPEED_LIMIT (if not specified 1b/s), before the
1046 : * transfer to be considered too slow and aborted. (GDAL >= 2.1).
1047 : * Corresponding configuration option: GDAL_HTTP_LOW_SPEED_TIME.
1048 : * </li>
1049 : * <li>LOW_SPEED_LIMIT=val, where val is in bytes/second. See LOW_SPEED_TIME.
1050 : * Has only effect if LOW_SPEED_TIME is specified too. (GDAL >= 2.1).
1051 : * Corresponding configuration option: GDAL_HTTP_LOW_SPEED_LIMIT.
1052 : * </li>
1053 : * <li>HEADERS=val, where val is an extra header to use when getting a web page.
1054 : * For example "Accept: application/x-ogcwkt"
1055 : * Corresponding configuration option: GDAL_HTTP_HEADERS.
1056 : * Starting with GDAL 3.6, the GDAL_HTTP_HEADERS configuration option can also
1057 : * be used to specify a comma separated list of key: value pairs. This is an
1058 : * alternative to the GDAL_HTTP_HEADER_FILE mechanism. If a comma or a
1059 : * double-quote character is needed in the value, then the key: value pair must
1060 : * be enclosed in double-quote characters. In that situation, backslash and
1061 : * double quote character must be backslash-escaped. e.g GDAL_HTTP_HEADERS=Foo:
1062 : * Bar,"Baz: escaped backslash \\, escaped double-quote \", end of
1063 : * value",Another: Header
1064 : * </li>
1065 : * <li>HEADER_FILE=filename: filename of a text file with "key: value" headers.
1066 : * The content of the file is not cached, and thus it is read again before
1067 : * issuing each HTTP request. (GDAL >= 2.2)
1068 : * Corresponding configuration option: GDAL_HTTP_HEADER_FILE.
1069 : * </li>
1070 : * <li>HTTPAUTH=[BASIC/NTLM/NEGOTIATE/ANY/ANYSAFE/BEARER] to specify an
1071 : * authentication scheme to use.
1072 : * Corresponding configuration option: GDAL_HTTP_AUTH.
1073 : * </li>
1074 : * <li>USERPWD=userid:password to specify a user and password for
1075 : * authentication.
1076 : * Corresponding configuration option: GDAL_HTTP_USERPWD.
1077 : * </li>
1078 : * <li>GSSAPI_DELEGATION=[NONE/POLICY/ALWAYS] set allowed
1079 : * GSS-API delegation. Relevant only with HTTPAUTH=NEGOTIATE (GDAL >= 3.3).
1080 : * Corresponding configuration option: GDAL_GSSAPI_DELEGATION (note: no "HTTP_" in the name)
1081 : * </li>
1082 : * <li>HTTP_BEARER=val set OAuth 2.0 Bearer Access Token.
1083 : * Relevant only with HTTPAUTH=BEARER (GDAL >= 3.9).
1084 : * Corresponding configuration option: GDAL_HTTP_BEARER
1085 : * </li>
1086 : * <li>POSTFIELDS=val, where val is a nul-terminated string to be passed to the
1087 : * server with a POST request.
1088 : * No Corresponding configuration option.
1089 : * </li>
1090 : * <li>PROXY=val, to make requests go through a
1091 : * proxy server, where val is of the form proxy.server.com:port_number. This
1092 : * option affects both HTTP and HTTPS URLs.
1093 : * Corresponding configuration option: GDAL_HTTP_PROXY.
1094 : * </li>
1095 : * <li>HTTPS_PROXY=val (GDAL
1096 : * >= 2.4), the same meaning as PROXY, but this option is taken into account
1097 : * only for HTTPS URLs.
1098 : * Corresponding configuration option: GDAL_HTTPS_PROXY.
1099 : * </li>
1100 : * <li>PROXYUSERPWD=val, where val is of the form username:password.
1101 : * Corresponding configuration option: GDAL_HTTP_PROXYUSERPWD
1102 : * </li>
1103 : * <li>PROXYAUTH=[BASIC/NTLM/DIGEST/NEGOTIATE/ANY/ANYSAFE] to
1104 : * specify an proxy authentication scheme to use..
1105 : * Corresponding configuration option: GDAL_PROXYAUTH (note: no "HTTP_" in the name)
1106 : * </li>
1107 : * <li>NETRC=[YES/NO] to
1108 : * enable or disable use of $HOME/.netrc (or NETRC_FILE), default YES.
1109 : * Corresponding configuration option: GDAL_HTTP_NETRC.
1110 : * </li>
1111 : * <li>NETRC_FILE=file name to read .netrc info from (GDAL >= 3.7).
1112 : * Corresponding configuration option: GDAL_HTTP_NETRC_FILE.
1113 : * </li>
1114 : * <li>CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc...
1115 : * No corresponding configuration option.
1116 : * </li>
1117 : * <li>FORM_FILE_NAME=val, where val is upload file name. If this
1118 : * option and FORM_FILE_PATH present, request type will set to POST.
1119 : * No corresponding configuration option.
1120 : * </li>
1121 : * <li>FORM_FILE_PATH=val, where val is upload file path.
1122 : * No corresponding configuration option.
1123 : * </li>
1124 : * <li>FORM_KEY_0=val...FORM_KEY_N, where val is name of form item.
1125 : * No corresponding configuration option.
1126 : * </li>
1127 : * <li>FORM_VALUE_0=val...FORM_VALUE_N, where val is value of the form
1128 : * item.
1129 : * No corresponding configuration option.
1130 : * </li>
1131 : * <li>FORM_ITEM_COUNT=val, where val is count of form items.
1132 : * No corresponding configuration option.
1133 : * </li>
1134 : * <li>COOKIE=val, where val is formatted as COOKIE1=VALUE1; COOKIE2=VALUE2;...
1135 : * Corresponding configuration option: GDAL_HTTP_COOKIE.</li>
1136 : * <li>COOKIEFILE=val, where val is file name to read cookies from
1137 : * (GDAL >= 2.4).
1138 : * Corresponding configuration option: GDAL_HTTP_COOKIEFILE.</li>
1139 : * <li>COOKIEJAR=val, where val is file name to store cookies to (GDAL >= 2.4).
1140 : * Corresponding configuration option: GDAL_HTTP_COOKIEJAR.</li>
1141 : * <li>MAX_RETRY=val, where val is the maximum number of
1142 : * retry attempts, when a retry is allowed (cf RETRY_CODES option).
1143 : * Default is 0, meaning no retry.
1144 : * Corresponding configuration option: GDAL_HTTP_MAX_RETRY.
1145 : * </li>
1146 : * <li>RETRY_DELAY=val, where val is the number of seconds
1147 : * between retry attempts. Default is 30.
1148 : * Corresponding configuration option: GDAL_HTTP_RETRY_DELAY.
1149 : * <li>RETRY_CODES=val, where val is "ALL" or a comma-separated list of HTTP
1150 : * codes that are considered for retry. By default, 429, 500, 502, 503 or 504
1151 : * HTTP errors are considered, as well as other situations with a particular
1152 : * HTTP or Curl error message. (GDAL >= 3.10).
1153 : * Corresponding configuration option: GDAL_HTTP_RETRY_CODES.
1154 : * </li>
1155 : * <li>MAX_FILE_SIZE=val, where val is a number of bytes (GDAL >= 2.2)
1156 : * No corresponding configuration option.
1157 : * </li>
1158 : * <li>CAINFO=/path/to/bundle.crt. This is path to Certificate Authority (CA)
1159 : * bundle file. By default, it will be looked for in a system location. If
1160 : * the CAINFO option is not defined, GDAL will also look in the
1161 : * CURL_CA_BUNDLE and SSL_CERT_FILE environment variables respectively
1162 : * and use the first one found as the CAINFO value (GDAL >= 2.1.3). The
1163 : * GDAL_CURL_CA_BUNDLE environment variable may also be used to set the
1164 : * CAINFO value in GDAL >= 3.2.</li>
1165 : * <li>HTTP_VERSION=1.0/1.1/2/2TLS (GDAL >= 2.3)/2PRIOR_KNOWLEDGE (GDAL >= 3.10).
1166 : * Specify HTTP version to use.
1167 : * Will default to 1.1 generally (except on some controlled environments,
1168 : * like Google Compute Engine VMs, where 2TLS will be the default).
1169 : * Support for HTTP/2 requires curl 7.33 or later, built against nghttp2.
1170 : * "2TLS" means that HTTP/2 will be attempted for HTTPS connections only.
1171 : * Whereas "2" means that HTTP/2 will be attempted for HTTP or HTTPS.
1172 : * "2PRIOR_KNOWLEDGE" means that the server will be assumed to support
1173 : * HTTP/2.
1174 : * Corresponding configuration option: GDAL_HTTP_VERSION.
1175 : * </li>
1176 : * <li>SSL_VERIFYSTATUS=YES/NO (GDAL >= 2.3, and curl >= 7.41): determines
1177 : * whether the status of the server cert using the "Certificate Status Request"
1178 : * TLS extension (aka. OCSP stapling) should be checked. If this option is
1179 : * enabled but the server does not support the TLS extension, the verification
1180 : * will fail. Default to NO.
1181 : * Corresponding configuration option: GDAL_HTTP_SSL_VERIFYSTATUS.
1182 : * </li>
1183 : * <li>USE_CAPI_STORE=YES/NO (GDAL >= 2.3,
1184 : * Windows only): whether CA certificates from the Windows certificate store.
1185 : * Defaults to NO.
1186 : * Corresponding configuration option: GDAL_HTTP_USE_CAPI_STORE.
1187 : * </li>
1188 : * <li>TCP_KEEPALIVE=YES/NO (GDAL >= 3.6): whether to
1189 : * enable TCP keep-alive. Defaults to NO.
1190 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPALIVE.
1191 : * </li>
1192 : * <li>TCP_KEEPIDLE=integer, in
1193 : * seconds (GDAL >= 3.6): keep-alive idle time. Defaults to 60. Only taken into
1194 : * account if TCP_KEEPALIVE=YES.
1195 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPIDLE.
1196 : * </li>
1197 : * <li>TCP_KEEPINTVL=integer, in seconds
1198 : * (GDAL >= 3.6): interval time between keep-alive probes. Defaults to 60. Only
1199 : * taken into account if TCP_KEEPALIVE=YES.
1200 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPINTVL.
1201 : * </li>
1202 : * <li>USERAGENT=string: value of User-Agent header. Starting with GDAL 3.7,
1203 : * GDAL core sets it by default (during driver initialization) to GDAL/x.y.z
1204 : * where x.y.z is the GDAL version number. Applications may override it with the
1205 : * CPLHTTPSetDefaultUserAgent() function.
1206 : * Corresponding configuration option: GDAL_HTTP_USERAGENT.
1207 : * </li>
1208 : * <li>SSLCERT=filename (GDAL >= 3.7): Filename of the the SSL client certificate.
1209 : * Cf https://curl.se/libcurl/c/CURLOPT_SSLCERT.html.
1210 : * Corresponding configuration option: GDAL_HTTP_SSLCERT.
1211 : * </li>
1212 : * <li>SSLCERTTYPE=string (GDAL >= 3.7): Format of the SSL certificate: "PEM"
1213 : * or "DER". Cf https://curl.se/libcurl/c/CURLOPT_SSLCERTTYPE.html.
1214 : * Corresponding configuration option: GDAL_HTTP_SSLCERTTYPE.
1215 : * </li>
1216 : * <li>SSLKEY=filename (GDAL >= 3.7): Private key file for TLS and SSL client
1217 : * certificate. Cf https://curl.se/libcurl/c/CURLOPT_SSLKEY.html.
1218 : * Corresponding configuration option: GDAL_HTTP_SSLKEY.
1219 : * </li>
1220 : * <li>KEYPASSWD=string (GDAL >= 3.7): Passphrase to private key.
1221 : * Cf https://curl.se/libcurl/c/CURLOPT_KEYPASSWD.html.
1222 : * Corresponding configuration option: GDAL_HTTP_KEYPASSWD.
1223 : * </ul>
1224 : *
1225 : * If an option is specified through papszOptions and as a configuration option,
1226 : * the former takes precedence over the later.
1227 : *
1228 : * Starting with GDAL 3.7, the above configuration options can also be specified
1229 : * as path-specific options with VSISetPathSpecificOption().
1230 : *
1231 : * @param pszURL valid URL recognized by underlying download library (libcurl)
1232 : * @param papszOptions option list as a NULL-terminated array of strings. May be
1233 : * NULL.
1234 : *
1235 : * @return a CPLHTTPResult* structure that must be freed by
1236 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled
1237 : */
1238 : // clang-format on
1239 2044 : CPLHTTPResult *CPLHTTPFetch(const char *pszURL, CSLConstList papszOptions)
1240 : {
1241 2044 : return CPLHTTPFetchEx(pszURL, papszOptions, nullptr, nullptr, nullptr,
1242 2044 : nullptr);
1243 : }
1244 :
1245 : /**
1246 : * Fetch a document from an url and return in a string.
1247 : * @param pszURL Url to fetch document from web.
1248 : * @param papszOptions Option list as a NULL-terminated array of strings.
1249 : * Available keys see in CPLHTTPFetch.
1250 : * @param pfnProgress Callback for reporting algorithm progress matching the
1251 : * GDALProgressFunc() semantics. May be NULL.
1252 : * @param pProgressArg Callback argument passed to pfnProgress.
1253 : * @param pfnWrite Write function pointer matching the CPLHTTPWriteFunc()
1254 : * semantics. May be NULL.
1255 : * @param pWriteArg Argument which will pass to a write function.
1256 : * @return A CPLHTTPResult* structure that must be freed by
1257 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled.
1258 : */
1259 2074 : CPLHTTPResult *CPLHTTPFetchEx(const char *pszURL, CSLConstList papszOptions,
1260 : GDALProgressFunc pfnProgress, void *pProgressArg,
1261 : CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg)
1262 :
1263 : {
1264 2874 : if (STARTS_WITH(pszURL, "/vsimem/") &&
1265 : // Disabled by default for potential security issues.
1266 800 : CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")))
1267 : {
1268 775 : CPLString osURL(pszURL);
1269 : const char *pszCustomRequest =
1270 775 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
1271 775 : if (pszCustomRequest != nullptr)
1272 : {
1273 5 : osURL += "&CUSTOMREQUEST=";
1274 5 : osURL += pszCustomRequest;
1275 : }
1276 775 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
1277 775 : if (pszUserPwd != nullptr)
1278 : {
1279 1 : osURL += "&USERPWD=";
1280 1 : osURL += pszUserPwd;
1281 : }
1282 775 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
1283 775 : if (pszPost != nullptr) // Hack: We append post content to filename.
1284 : {
1285 281 : osURL += "&POSTFIELDS=";
1286 281 : osURL += pszPost;
1287 : }
1288 775 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1289 907 : if (pszHeaders != nullptr &&
1290 132 : CPLTestBool(
1291 : CPLGetConfigOption("CPL_CURL_VSIMEM_PRINT_HEADERS", "FALSE")))
1292 : {
1293 5 : osURL += "&HEADERS=";
1294 5 : osURL += pszHeaders;
1295 : }
1296 775 : vsi_l_offset nLength = 0;
1297 : CPLHTTPResult *psResult =
1298 775 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1299 775 : GByte *pabyData = VSIGetMemFileBuffer(osURL, &nLength, FALSE);
1300 775 : if (pabyData == nullptr)
1301 : {
1302 119 : CPLDebug("HTTP", "Cannot find %s", osURL.c_str());
1303 119 : psResult->nStatus = 1;
1304 119 : psResult->pszErrBuf =
1305 119 : CPLStrdup(CPLSPrintf("HTTP error code : %d", 404));
1306 119 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1307 : }
1308 656 : else if (nLength != 0)
1309 : {
1310 613 : psResult->nDataLen = static_cast<int>(nLength);
1311 613 : psResult->pabyData = static_cast<GByte *>(
1312 613 : CPLMalloc(static_cast<size_t>(nLength) + 1));
1313 613 : memcpy(psResult->pabyData, pabyData, static_cast<size_t>(nLength));
1314 613 : psResult->pabyData[static_cast<size_t>(nLength)] = 0;
1315 : }
1316 :
1317 775 : if (psResult->pabyData != nullptr &&
1318 613 : STARTS_WITH(reinterpret_cast<char *>(psResult->pabyData),
1319 : "Content-Type: "))
1320 : {
1321 7 : const char *pszContentType =
1322 7 : reinterpret_cast<char *>(psResult->pabyData) +
1323 : strlen("Content-type: ");
1324 7 : const char *pszEOL = strchr(pszContentType, '\r');
1325 7 : if (pszEOL)
1326 7 : pszEOL = strchr(pszContentType, '\n');
1327 7 : if (pszEOL)
1328 : {
1329 7 : size_t nContentLength = pszEOL - pszContentType;
1330 7 : psResult->pszContentType =
1331 7 : static_cast<char *>(CPLMalloc(nContentLength + 1));
1332 7 : memcpy(psResult->pszContentType, pszContentType,
1333 : nContentLength);
1334 7 : psResult->pszContentType[nContentLength] = 0;
1335 : }
1336 : }
1337 :
1338 775 : return psResult;
1339 : }
1340 :
1341 : // Try to user alternate network layer if set.
1342 1299 : auto pCtx = GetHTTPFetchContext(false);
1343 1299 : if (pCtx)
1344 : {
1345 3 : for (size_t i = pCtx->stack.size(); i > 0;)
1346 : {
1347 1 : --i;
1348 1 : const auto &cbk = pCtx->stack[i];
1349 1 : auto cbkFunc = cbk.first;
1350 1 : auto pUserData = cbk.second;
1351 1 : auto res = cbkFunc(pszURL, papszOptions, pfnProgress, pProgressArg,
1352 : pfnWrite, pWriteArg, pUserData);
1353 1 : if (res)
1354 : {
1355 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1356 : {
1357 0 : CPLHTTPDestroyResult(res);
1358 0 : res = nullptr;
1359 : }
1360 1 : return res;
1361 : }
1362 : }
1363 : }
1364 :
1365 1298 : if (gpsHTTPFetchCallbackFunc)
1366 : {
1367 1 : auto res = gpsHTTPFetchCallbackFunc(pszURL, papszOptions, pfnProgress,
1368 : pProgressArg, pfnWrite, pWriteArg,
1369 : gpHTTPFetchCallbackUserData);
1370 1 : if (res)
1371 : {
1372 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1373 : {
1374 0 : CPLHTTPDestroyResult(res);
1375 0 : res = nullptr;
1376 : }
1377 1 : return res;
1378 : }
1379 : }
1380 :
1381 : #ifndef HAVE_CURL
1382 : CPLError(CE_Failure, CPLE_NotSupported,
1383 : "GDAL/OGR not compiled with libcurl support, "
1384 : "remote requests not supported.");
1385 : return nullptr;
1386 : #else
1387 :
1388 : /* -------------------------------------------------------------------- */
1389 : /* Are we using a persistent named session? If so, search for */
1390 : /* or create it. */
1391 : /* */
1392 : /* Currently this code does not attempt to protect against */
1393 : /* multiple threads asking for the same named session. If that */
1394 : /* occurs it will be in use in multiple threads at once, which */
1395 : /* will lead to potential crashes in libcurl. */
1396 : /* -------------------------------------------------------------------- */
1397 1297 : CURL *http_handle = nullptr;
1398 :
1399 1297 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1400 : const char *pszClosePersistent =
1401 1297 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1402 1297 : if (pszPersistent)
1403 : {
1404 994 : CPLString osSessionName = pszPersistent;
1405 497 : CPLMutexHolder oHolder(&hSessionMapMutex);
1406 :
1407 497 : if (poSessionMap == nullptr)
1408 134 : poSessionMap = new std::map<CPLString, CURL *>;
1409 497 : if (poSessionMap->count(osSessionName) == 0)
1410 : {
1411 157 : (*poSessionMap)[osSessionName] = curl_easy_init();
1412 157 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1413 : osSessionName.c_str());
1414 : }
1415 :
1416 497 : http_handle = (*poSessionMap)[osSessionName];
1417 : }
1418 : /* -------------------------------------------------------------------- */
1419 : /* Are we requested to close a persistent named session? */
1420 : /* -------------------------------------------------------------------- */
1421 800 : else if (pszClosePersistent)
1422 : {
1423 374 : CPLString osSessionName = pszClosePersistent;
1424 187 : CPLMutexHolder oHolder(&hSessionMapMutex);
1425 :
1426 187 : if (poSessionMap)
1427 : {
1428 : std::map<CPLString, CURL *>::iterator oIter =
1429 157 : poSessionMap->find(osSessionName);
1430 157 : if (oIter != poSessionMap->end())
1431 : {
1432 157 : curl_easy_cleanup(oIter->second);
1433 157 : poSessionMap->erase(oIter);
1434 157 : if (poSessionMap->empty())
1435 : {
1436 134 : delete poSessionMap;
1437 134 : poSessionMap = nullptr;
1438 : }
1439 157 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1440 : osSessionName.c_str());
1441 : }
1442 : else
1443 : {
1444 0 : CPLDebug("HTTP",
1445 : "Could not find persistent session named '%s'.",
1446 : osSessionName.c_str());
1447 : }
1448 : }
1449 :
1450 187 : return nullptr;
1451 : }
1452 : else
1453 613 : http_handle = curl_easy_init();
1454 :
1455 : /* -------------------------------------------------------------------- */
1456 : /* Setup the request. */
1457 : /* -------------------------------------------------------------------- */
1458 1110 : char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
1459 :
1460 1110 : CPLHTTPEmitFetchDebug(pszURL);
1461 :
1462 : CPLHTTPResult *psResult =
1463 1110 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1464 :
1465 : struct curl_slist *headers = reinterpret_cast<struct curl_slist *>(
1466 1110 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1467 1110 : if (headers != nullptr)
1468 570 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers);
1469 :
1470 : // Are we making a head request.
1471 1110 : const char *pszNoBody = nullptr;
1472 1110 : if ((pszNoBody = CSLFetchNameValue(papszOptions, "NO_BODY")) != nullptr)
1473 : {
1474 0 : if (CPLTestBool(pszNoBody))
1475 : {
1476 0 : CPLDebug("HTTP", "HEAD Request: %s", pszURL);
1477 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOBODY, 1L);
1478 : }
1479 : }
1480 :
1481 : // Capture response headers.
1482 1110 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA, psResult);
1483 1110 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1484 : CPLHdrWriteFct);
1485 :
1486 1110 : CPLHTTPResultWithLimit sResultWithLimit;
1487 1110 : if (nullptr == pfnWrite)
1488 : {
1489 1109 : pfnWrite = CPLWriteFct;
1490 :
1491 1109 : sResultWithLimit.psResult = psResult;
1492 1109 : sResultWithLimit.nMaxFileSize = 0;
1493 : const char *pszMaxFileSize =
1494 1109 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1495 1109 : if (pszMaxFileSize != nullptr)
1496 : {
1497 10 : sResultWithLimit.nMaxFileSize = atoi(pszMaxFileSize);
1498 : // Only useful if size is returned by server before actual download.
1499 10 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1500 : sResultWithLimit.nMaxFileSize);
1501 : }
1502 1109 : pWriteArg = &sResultWithLimit;
1503 : }
1504 :
1505 1110 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, pWriteArg);
1506 1110 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, pfnWrite);
1507 :
1508 1110 : CurlProcessData stProcessData = {pfnProgress, pProgressArg};
1509 1110 : if (nullptr != pfnProgress)
1510 : {
1511 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFOFUNCTION,
1512 : NewProcessFunction);
1513 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFODATA,
1514 : &stProcessData);
1515 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOPROGRESS, 0L);
1516 : }
1517 :
1518 1110 : szCurlErrBuf[0] = '\0';
1519 :
1520 1110 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER, szCurlErrBuf);
1521 :
1522 1110 : bool bGZipRequested = false;
1523 1110 : if (bSupportGZip && CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1524 : {
1525 1110 : bGZipRequested = true;
1526 1110 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1527 : }
1528 :
1529 2220 : CPLHTTPPostFields oPostFields;
1530 1110 : if (oPostFields.Fill(http_handle, papszOptions) != CE_None)
1531 : {
1532 2 : psResult->nStatus = 34; // CURLE_HTTP_POST_ERROR
1533 2 : psResult->pszErrBuf = CPLStrdup(oPostFields.GetErrorMessage().c_str());
1534 2 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1535 2 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1536 2 : return psResult;
1537 : }
1538 :
1539 : /* -------------------------------------------------------------------- */
1540 : /* Depending on status code, retry this HTTP call until max */
1541 : /* retry has been reached */
1542 : /* -------------------------------------------------------------------- */
1543 1108 : const char *pszRetryDelay = CSLFetchNameValue(papszOptions, "RETRY_DELAY");
1544 1108 : if (pszRetryDelay == nullptr)
1545 1108 : pszRetryDelay = CPLGetConfigOption(
1546 : "GDAL_HTTP_RETRY_DELAY", CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY));
1547 1108 : const char *pszMaxRetries = CSLFetchNameValue(papszOptions, "MAX_RETRY");
1548 1108 : if (pszMaxRetries == nullptr)
1549 1044 : pszMaxRetries = CPLGetConfigOption(
1550 : "GDAL_HTTP_MAX_RETRY", CPLSPrintf("%d", CPL_HTTP_MAX_RETRY));
1551 : // coverity[tainted_data]
1552 1108 : double dfRetryDelaySecs = CPLAtof(pszRetryDelay);
1553 1108 : int nMaxRetries = atoi(pszMaxRetries);
1554 1108 : const char *pszRetryCodes = CSLFetchNameValue(papszOptions, "RETRY_CODES");
1555 1108 : if (!pszRetryCodes)
1556 1108 : pszRetryCodes = CPLGetConfigOption("GDAL_HTTP_RETRY_CODES", nullptr);
1557 1108 : int nRetryCount = 0;
1558 :
1559 : while (true)
1560 : {
1561 : /* --------------------------------------------------------------------
1562 : */
1563 : /* Execute the request, waiting for results. */
1564 : /* --------------------------------------------------------------------
1565 : */
1566 1108 : void *old_handler = CPLHTTPIgnoreSigPipe();
1567 1108 : psResult->nStatus = static_cast<int>(curl_easy_perform(http_handle));
1568 1108 : CPLHTTPRestoreSigPipeHandler(old_handler);
1569 :
1570 : /* --------------------------------------------------------------------
1571 : */
1572 : /* Fetch content-type if possible. */
1573 : /* --------------------------------------------------------------------
1574 : */
1575 1108 : psResult->pszContentType = nullptr;
1576 1108 : curl_easy_getinfo(http_handle, CURLINFO_CONTENT_TYPE,
1577 : &(psResult->pszContentType));
1578 1108 : if (psResult->pszContentType != nullptr)
1579 475 : psResult->pszContentType = CPLStrdup(psResult->pszContentType);
1580 :
1581 1108 : long response_code = 0;
1582 1108 : curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
1583 1108 : if (response_code != 200)
1584 : {
1585 394 : const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1586 : static_cast<int>(response_code), dfRetryDelaySecs,
1587 197 : reinterpret_cast<const char *>(psResult->pabyData),
1588 : szCurlErrBuf, pszRetryCodes);
1589 197 : if (dfNewRetryDelay > 0 && nRetryCount < nMaxRetries)
1590 : {
1591 0 : CPLError(CE_Warning, CPLE_AppDefined,
1592 : "HTTP error code: %d - %s. "
1593 : "Retrying again in %.1f secs",
1594 : static_cast<int>(response_code), pszURL,
1595 : dfRetryDelaySecs);
1596 0 : CPLSleep(dfRetryDelaySecs);
1597 0 : dfRetryDelaySecs = dfNewRetryDelay;
1598 0 : nRetryCount++;
1599 :
1600 0 : CPLFree(psResult->pszContentType);
1601 0 : psResult->pszContentType = nullptr;
1602 0 : CSLDestroy(psResult->papszHeaders);
1603 0 : psResult->papszHeaders = nullptr;
1604 0 : CPLFree(psResult->pabyData);
1605 0 : psResult->pabyData = nullptr;
1606 0 : psResult->nDataLen = 0;
1607 0 : psResult->nDataAlloc = 0;
1608 :
1609 0 : continue;
1610 : }
1611 : }
1612 :
1613 : /* --------------------------------------------------------------------
1614 : */
1615 : /* Have we encountered some sort of error? */
1616 : /* --------------------------------------------------------------------
1617 : */
1618 1108 : if (strlen(szCurlErrBuf) > 0)
1619 : {
1620 8 : bool bSkipError = false;
1621 : const char *pszContentLength =
1622 8 : CSLFetchNameValue(psResult->papszHeaders, "Content-Length");
1623 : // Some servers such as
1624 : // http://115.113.193.14/cgi-bin/world/qgis_mapserv.fcgi?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetCapabilities
1625 : // invalidly return Content-Length as the uncompressed size, with
1626 : // makes curl to wait for more data and time-out finally. If we got
1627 : // the expected data size, then we don't emit an error but turn off
1628 : // GZip requests.
1629 8 : if (bGZipRequested &&
1630 8 : strstr(szCurlErrBuf, "transfer closed with") &&
1631 0 : strstr(szCurlErrBuf, "bytes remaining to read"))
1632 : {
1633 0 : if (pszContentLength && psResult->nDataLen != 0 &&
1634 0 : atoi(pszContentLength) == psResult->nDataLen)
1635 : {
1636 : const char *pszCurlGZIPOption =
1637 0 : CPLGetConfigOption("CPL_CURL_GZIP", nullptr);
1638 0 : if (pszCurlGZIPOption == nullptr)
1639 : {
1640 0 : CPLSetConfigOption("CPL_CURL_GZIP", "NO");
1641 0 : CPLDebug("HTTP",
1642 : "Disabling CPL_CURL_GZIP, "
1643 : "because %s doesn't support it properly",
1644 : pszURL);
1645 : }
1646 0 : psResult->nStatus = 0;
1647 0 : bSkipError = true;
1648 0 : }
1649 : }
1650 :
1651 : // Ignore SSL errors about non-properly terminated connection,
1652 : // often due to HTTP proxies
1653 8 : else if (pszContentLength == nullptr &&
1654 : // Cf https://github.com/curl/curl/pull/3148
1655 7 : (strstr(szCurlErrBuf,
1656 : "GnuTLS recv error (-110): The TLS connection was "
1657 7 : "non-properly terminated") != nullptr ||
1658 : // Cf https://github.com/curl/curl/issues/9024
1659 7 : strstr(szCurlErrBuf,
1660 : "SSL_read: error:0A000126:SSL "
1661 : "routines::unexpected eof while reading") !=
1662 : nullptr))
1663 : {
1664 0 : psResult->nStatus = 0;
1665 0 : bSkipError = true;
1666 : }
1667 8 : else if (CPLTestBool(
1668 : CPLGetConfigOption("CPL_CURL_IGNORE_ERROR", "NO")))
1669 : {
1670 0 : psResult->nStatus = 0;
1671 0 : bSkipError = true;
1672 : }
1673 :
1674 8 : if (!bSkipError)
1675 : {
1676 8 : psResult->pszErrBuf = CPLStrdup(szCurlErrBuf);
1677 8 : if (psResult->nDataLen > 0)
1678 : {
1679 1 : CPLError(CE_Failure, CPLE_AppDefined,
1680 : "%s. You may set the CPL_CURL_IGNORE_ERROR "
1681 : "configuration option to YES to try to ignore it.",
1682 : szCurlErrBuf);
1683 : }
1684 : else
1685 : {
1686 7 : CPLError(CE_Failure, CPLE_AppDefined, "%s", szCurlErrBuf);
1687 : }
1688 : }
1689 : }
1690 : else
1691 : {
1692 1100 : if (response_code >= 400 && response_code < 600)
1693 : {
1694 186 : psResult->pszErrBuf = CPLStrdup(CPLSPrintf(
1695 : "HTTP error code : %d", static_cast<int>(response_code)));
1696 186 : CPLError(CE_Failure, CPLE_AppDefined, "%s",
1697 : psResult->pszErrBuf);
1698 : }
1699 : }
1700 1108 : break;
1701 0 : }
1702 :
1703 1108 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1704 :
1705 1108 : return psResult;
1706 : #endif /* def HAVE_CURL */
1707 : }
1708 :
1709 : #ifdef HAVE_CURL
1710 : /************************************************************************/
1711 : /* CPLMultiPerformWait() */
1712 : /************************************************************************/
1713 :
1714 3328 : bool CPLMultiPerformWait(void *hCurlMultiHandleIn, int & /*repeats*/)
1715 : {
1716 3328 : CURLM *hCurlMultiHandle = static_cast<CURLM *>(hCurlMultiHandleIn);
1717 :
1718 : // Wait for events on the sockets
1719 :
1720 : // Using curl_multi_poll() is preferred to avoid hitting the 1024 file
1721 : // descriptor limit
1722 :
1723 3328 : int numfds = 0;
1724 3328 : if (curl_multi_poll(hCurlMultiHandle, nullptr, 0, 1000, &numfds) !=
1725 : CURLM_OK)
1726 : {
1727 0 : CPLError(CE_Failure, CPLE_AppDefined, "curl_multi_poll() failed");
1728 0 : return false;
1729 : }
1730 3328 : return true;
1731 : }
1732 :
1733 : class CPLHTTPErrorBuffer
1734 : {
1735 : public:
1736 : char szBuffer[CURL_ERROR_SIZE + 1];
1737 :
1738 0 : CPLHTTPErrorBuffer()
1739 0 : {
1740 0 : szBuffer[0] = '\0';
1741 0 : }
1742 : };
1743 :
1744 : #endif // HAVE_CURL
1745 :
1746 : /************************************************************************/
1747 : /* CPLHTTPMultiFetch() */
1748 : /************************************************************************/
1749 :
1750 : /**
1751 : * \brief Fetch several documents at once
1752 : *
1753 : * @param papszURL array of valid URLs recognized by underlying download library
1754 : * (libcurl)
1755 : * @param nURLCount number of URLs of papszURL
1756 : * @param nMaxSimultaneous maximum number of downloads to issue simultaneously.
1757 : * Any negative or zer value means unlimited.
1758 : * @param papszOptions option list as a NULL-terminated array of strings. May be
1759 : * NULL. Refer to CPLHTTPFetch() for valid options.
1760 : * @return an array of CPLHTTPResult* structures that must be freed by
1761 : * CPLHTTPDestroyMultiResult() or NULL if libcurl support is disabled
1762 : *
1763 : * @since GDAL 2.3
1764 : */
1765 0 : CPLHTTPResult **CPLHTTPMultiFetch(const char *const *papszURL, int nURLCount,
1766 : int nMaxSimultaneous,
1767 : CSLConstList papszOptions)
1768 : {
1769 : #ifndef HAVE_CURL
1770 : (void)papszURL;
1771 : (void)nURLCount;
1772 : (void)nMaxSimultaneous;
1773 : (void)papszOptions;
1774 :
1775 : CPLError(CE_Failure, CPLE_NotSupported,
1776 : "GDAL/OGR not compiled with libcurl support, "
1777 : "remote requests not supported.");
1778 : return nullptr;
1779 : #else /* def HAVE_CURL */
1780 :
1781 : /* -------------------------------------------------------------------- */
1782 : /* Are we using a persistent named session? If so, search for */
1783 : /* or create it. */
1784 : /* */
1785 : /* Currently this code does not attempt to protect against */
1786 : /* multiple threads asking for the same named session. If that */
1787 : /* occurs it will be in use in multiple threads at once, which */
1788 : /* will lead to potential crashes in libcurl. */
1789 : /* -------------------------------------------------------------------- */
1790 0 : CURLM *hCurlMultiHandle = nullptr;
1791 :
1792 0 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1793 : const char *pszClosePersistent =
1794 0 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1795 0 : if (pszPersistent)
1796 : {
1797 0 : CPLString osSessionName = pszPersistent;
1798 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1799 :
1800 0 : if (poSessionMultiMap == nullptr)
1801 0 : poSessionMultiMap = new std::map<CPLString, CURLM *>;
1802 0 : if (poSessionMultiMap->count(osSessionName) == 0)
1803 : {
1804 0 : (*poSessionMultiMap)[osSessionName] = curl_multi_init();
1805 0 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1806 : osSessionName.c_str());
1807 : }
1808 :
1809 0 : hCurlMultiHandle = (*poSessionMultiMap)[osSessionName];
1810 : }
1811 : /* -------------------------------------------------------------------- */
1812 : /* Are we requested to close a persistent named session? */
1813 : /* -------------------------------------------------------------------- */
1814 0 : else if (pszClosePersistent)
1815 : {
1816 0 : CPLString osSessionName = pszClosePersistent;
1817 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1818 :
1819 0 : if (poSessionMultiMap)
1820 : {
1821 0 : auto oIter = poSessionMultiMap->find(osSessionName);
1822 0 : if (oIter != poSessionMultiMap->end())
1823 : {
1824 0 : VSICURLMultiCleanup(oIter->second);
1825 0 : poSessionMultiMap->erase(oIter);
1826 0 : if (poSessionMultiMap->empty())
1827 : {
1828 0 : delete poSessionMultiMap;
1829 0 : poSessionMultiMap = nullptr;
1830 : }
1831 0 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1832 : osSessionName.c_str());
1833 : }
1834 : else
1835 : {
1836 0 : CPLDebug("HTTP",
1837 : "Could not find persistent session named '%s'.",
1838 : osSessionName.c_str());
1839 : }
1840 : }
1841 :
1842 0 : return nullptr;
1843 : }
1844 : else
1845 : {
1846 0 : hCurlMultiHandle = curl_multi_init();
1847 : }
1848 :
1849 : CPLHTTPResult **papsResults = static_cast<CPLHTTPResult **>(
1850 0 : CPLCalloc(nURLCount, sizeof(CPLHTTPResult *)));
1851 0 : std::vector<CURL *> asHandles;
1852 0 : std::vector<CPLHTTPResultWithLimit> asResults;
1853 0 : asResults.resize(nURLCount);
1854 0 : std::vector<struct curl_slist *> aHeaders;
1855 0 : aHeaders.resize(nURLCount);
1856 0 : std::vector<CPLHTTPErrorBuffer> asErrorBuffers;
1857 0 : asErrorBuffers.resize(nURLCount);
1858 :
1859 0 : for (int i = 0; i < nURLCount; i++)
1860 : {
1861 0 : papsResults[i] =
1862 0 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1863 :
1864 0 : const char *pszURL = papszURL[i];
1865 0 : CURL *http_handle = curl_easy_init();
1866 :
1867 0 : aHeaders[i] = reinterpret_cast<struct curl_slist *>(
1868 0 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1869 :
1870 : // Set Headers.
1871 0 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1872 0 : if (pszHeaders != nullptr)
1873 : {
1874 : char **papszTokensHeaders =
1875 0 : CSLTokenizeString2(pszHeaders, "\r\n", 0);
1876 0 : for (int j = 0; papszTokensHeaders[j] != nullptr; ++j)
1877 0 : aHeaders[i] =
1878 0 : curl_slist_append(aHeaders[i], papszTokensHeaders[j]);
1879 0 : CSLDestroy(papszTokensHeaders);
1880 : }
1881 :
1882 0 : if (aHeaders[i] != nullptr)
1883 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER,
1884 : aHeaders[i]);
1885 :
1886 : // Capture response headers.
1887 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA,
1888 : papsResults[i]);
1889 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1890 : CPLHdrWriteFct);
1891 :
1892 0 : asResults[i].psResult = papsResults[i];
1893 : const char *pszMaxFileSize =
1894 0 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1895 0 : if (pszMaxFileSize != nullptr)
1896 : {
1897 0 : asResults[i].nMaxFileSize = atoi(pszMaxFileSize);
1898 : // Only useful if size is returned by server before actual download.
1899 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1900 : asResults[i].nMaxFileSize);
1901 : }
1902 :
1903 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA,
1904 : &asResults[i]);
1905 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION,
1906 : CPLWriteFct);
1907 :
1908 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER,
1909 : asErrorBuffers[i].szBuffer);
1910 :
1911 0 : if (bSupportGZip &&
1912 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1913 : {
1914 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1915 : }
1916 :
1917 0 : asHandles.push_back(http_handle);
1918 : }
1919 :
1920 0 : int iCurRequest = 0;
1921 0 : for (;
1922 0 : iCurRequest <
1923 0 : std::min(nURLCount, nMaxSimultaneous > 0 ? nMaxSimultaneous : INT_MAX);
1924 : iCurRequest++)
1925 : {
1926 0 : CPLHTTPEmitFetchDebug(papszURL[iCurRequest],
1927 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1928 0 : curl_multi_add_handle(hCurlMultiHandle, asHandles[iCurRequest]);
1929 : }
1930 :
1931 0 : int repeats = 0;
1932 0 : void *old_handler = CPLHTTPIgnoreSigPipe();
1933 : while (true)
1934 : {
1935 0 : int still_running = 0;
1936 0 : while (curl_multi_perform(hCurlMultiHandle, &still_running) ==
1937 : CURLM_CALL_MULTI_PERFORM)
1938 : {
1939 : // loop
1940 : }
1941 0 : if (!still_running && iCurRequest == nURLCount)
1942 : {
1943 0 : break;
1944 : }
1945 :
1946 0 : bool bRequestsAdded = false;
1947 : CURLMsg *msg;
1948 0 : do
1949 : {
1950 0 : int msgq = 0;
1951 0 : msg = curl_multi_info_read(hCurlMultiHandle, &msgq);
1952 0 : if (msg && (msg->msg == CURLMSG_DONE))
1953 : {
1954 0 : if (iCurRequest < nURLCount)
1955 : {
1956 0 : CPLHTTPEmitFetchDebug(
1957 0 : papszURL[iCurRequest],
1958 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1959 0 : curl_multi_add_handle(hCurlMultiHandle,
1960 0 : asHandles[iCurRequest]);
1961 0 : iCurRequest++;
1962 0 : bRequestsAdded = true;
1963 : }
1964 : }
1965 0 : } while (msg);
1966 :
1967 0 : if (!bRequestsAdded)
1968 0 : CPLMultiPerformWait(hCurlMultiHandle, repeats);
1969 0 : }
1970 0 : CPLHTTPRestoreSigPipeHandler(old_handler);
1971 :
1972 0 : for (int i = 0; i < nURLCount; i++)
1973 : {
1974 0 : if (asErrorBuffers[i].szBuffer[0] != '\0')
1975 0 : papsResults[i]->pszErrBuf = CPLStrdup(asErrorBuffers[i].szBuffer);
1976 : else
1977 : {
1978 0 : long response_code = 0;
1979 0 : curl_easy_getinfo(asHandles[i], CURLINFO_RESPONSE_CODE,
1980 : &response_code);
1981 :
1982 0 : if (response_code >= 400 && response_code < 600)
1983 : {
1984 0 : papsResults[i]->pszErrBuf = CPLStrdup(CPLSPrintf(
1985 : "HTTP error code : %d", static_cast<int>(response_code)));
1986 : }
1987 : }
1988 :
1989 0 : curl_easy_getinfo(asHandles[i], CURLINFO_CONTENT_TYPE,
1990 : &(papsResults[i]->pszContentType));
1991 0 : if (papsResults[i]->pszContentType != nullptr)
1992 0 : papsResults[i]->pszContentType =
1993 0 : CPLStrdup(papsResults[i]->pszContentType);
1994 :
1995 0 : curl_multi_remove_handle(hCurlMultiHandle, asHandles[i]);
1996 0 : curl_easy_cleanup(asHandles[i]);
1997 : }
1998 :
1999 0 : if (!pszPersistent)
2000 0 : VSICURLMultiCleanup(hCurlMultiHandle);
2001 :
2002 0 : for (size_t i = 0; i < aHeaders.size(); i++)
2003 0 : curl_slist_free_all(aHeaders[i]);
2004 :
2005 0 : return papsResults;
2006 : #endif /* def HAVE_CURL */
2007 : }
2008 :
2009 : /************************************************************************/
2010 : /* CPLHTTPDestroyMultiResult() */
2011 : /************************************************************************/
2012 : /**
2013 : * \brief Clean the memory associated with the return value of
2014 : * CPLHTTPMultiFetch()
2015 : *
2016 : * @param papsResults pointer to the return value of CPLHTTPMultiFetch()
2017 : * @param nCount value of the nURLCount parameter passed to CPLHTTPMultiFetch()
2018 : * @since GDAL 2.3
2019 : */
2020 0 : void CPLHTTPDestroyMultiResult(CPLHTTPResult **papsResults, int nCount)
2021 : {
2022 0 : if (papsResults)
2023 : {
2024 0 : for (int i = 0; i < nCount; i++)
2025 : {
2026 0 : CPLHTTPDestroyResult(papsResults[i]);
2027 : }
2028 0 : CPLFree(papsResults);
2029 : }
2030 0 : }
2031 :
2032 : #ifdef HAVE_CURL
2033 :
2034 : #ifdef _WIN32
2035 :
2036 : #include <windows.h>
2037 :
2038 : /************************************************************************/
2039 : /* CPLFindWin32CurlCaBundleCrt() */
2040 : /************************************************************************/
2041 :
2042 : static const char *CPLFindWin32CurlCaBundleCrt()
2043 : {
2044 : DWORD nResLen;
2045 : const DWORD nBufSize = MAX_PATH + 1;
2046 : char *pszFilePart = nullptr;
2047 :
2048 : char *pszPath = static_cast<char *>(CPLCalloc(1, nBufSize));
2049 :
2050 : nResLen = SearchPathA(nullptr, "curl-ca-bundle.crt", nullptr, nBufSize,
2051 : pszPath, &pszFilePart);
2052 : if (nResLen > 0)
2053 : {
2054 : const char *pszRet = CPLSPrintf("%s", pszPath);
2055 : CPLFree(pszPath);
2056 : return pszRet;
2057 : }
2058 : CPLFree(pszPath);
2059 : return nullptr;
2060 : }
2061 :
2062 : #endif // WIN32
2063 :
2064 : /************************************************************************/
2065 : /* CPLHTTPCurlDebugFunction() */
2066 : /************************************************************************/
2067 :
2068 16 : static int CPLHTTPCurlDebugFunction(CURL *handle, curl_infotype type,
2069 : char *data, size_t size, void *userp)
2070 : {
2071 : (void)handle;
2072 : (void)userp;
2073 :
2074 16 : const char *pszDebugKey = nullptr;
2075 16 : if (type == CURLINFO_TEXT)
2076 : {
2077 10 : pszDebugKey = "CURL_INFO_TEXT";
2078 : }
2079 6 : else if (type == CURLINFO_HEADER_OUT)
2080 : {
2081 1 : pszDebugKey = "CURL_INFO_HEADER_OUT";
2082 : }
2083 5 : else if (type == CURLINFO_HEADER_IN)
2084 : {
2085 5 : pszDebugKey = "CURL_INFO_HEADER_IN";
2086 : }
2087 0 : else if (type == CURLINFO_DATA_IN &&
2088 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE_DATA_IN", "NO")))
2089 : {
2090 0 : pszDebugKey = "CURL_INFO_DATA_IN";
2091 : }
2092 :
2093 16 : if (pszDebugKey)
2094 : {
2095 32 : std::string osMsg(data, size);
2096 16 : if (!osMsg.empty() && osMsg.back() == '\n')
2097 16 : osMsg.pop_back();
2098 16 : CPLDebug(pszDebugKey, "%s", osMsg.c_str());
2099 : }
2100 16 : return 0;
2101 : }
2102 :
2103 : /************************************************************************/
2104 : /* CPLHTTPSetOptions() */
2105 : /************************************************************************/
2106 :
2107 : // Note: papszOptions must be kept alive until curl_easy/multi_perform()
2108 : // has completed, and we must be careful not to set short lived strings
2109 : // with unchecked_curl_easy_setopt(), as long as we need to support curl < 7.17
2110 : // see https://curl.haxx.se/libcurl/c/unchecked_curl_easy_setopt.html
2111 : // caution: if we remove that assumption, we'll needto use
2112 : // CURLOPT_COPYPOSTFIELDS
2113 :
2114 2414 : void *CPLHTTPSetOptions(void *pcurl, const char *pszURL,
2115 : const char *const *papszOptions)
2116 : {
2117 2414 : CheckCurlFeatures();
2118 :
2119 2414 : CURL *http_handle = reinterpret_cast<CURL *>(pcurl);
2120 :
2121 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_URL, pszURL);
2122 :
2123 2414 : if (CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")))
2124 : {
2125 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1);
2126 :
2127 1 : if (CPLGetConfigOption("CPL_DEBUG", nullptr) != nullptr)
2128 : {
2129 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_DEBUGFUNCTION,
2130 : CPLHTTPCurlDebugFunction);
2131 : }
2132 : }
2133 :
2134 : const char *pszHttpVersion =
2135 2414 : CSLFetchNameValue(papszOptions, "HTTP_VERSION");
2136 2414 : if (pszHttpVersion == nullptr)
2137 2414 : pszHttpVersion = CPLGetConfigOption("GDAL_HTTP_VERSION", nullptr);
2138 2414 : if (pszHttpVersion && strcmp(pszHttpVersion, "1.0") == 0)
2139 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2140 : CURL_HTTP_VERSION_1_0);
2141 2414 : else if (pszHttpVersion && strcmp(pszHttpVersion, "1.1") == 0)
2142 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2143 : CURL_HTTP_VERSION_1_1);
2144 2414 : else if (pszHttpVersion && (strcmp(pszHttpVersion, "2") == 0 ||
2145 0 : strcmp(pszHttpVersion, "2.0") == 0))
2146 : {
2147 0 : if (bSupportHTTP2)
2148 : {
2149 : // Try HTTP/2 both for HTTP and HTTPS. With fallback to HTTP/1.1
2150 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2151 : CURL_HTTP_VERSION_2_0);
2152 : }
2153 : }
2154 2414 : else if (pszHttpVersion && strcmp(pszHttpVersion, "2PRIOR_KNOWLEDGE") == 0)
2155 : {
2156 0 : if (bSupportHTTP2)
2157 : {
2158 : // Assume HTTP/2 is supported by the server. The cURL docs indicate
2159 : // that it makes no difference for HTTPS, but it does seem to work
2160 : // in practice.
2161 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2162 : CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE);
2163 : }
2164 : }
2165 2414 : else if (pszHttpVersion == nullptr || strcmp(pszHttpVersion, "2TLS") == 0)
2166 : {
2167 2414 : if (bSupportHTTP2)
2168 : {
2169 : // Only enable this mode if explicitly required, or if the
2170 : // machine is a GCE instance. On other networks, requesting a
2171 : // file in HTTP/2 is found to be significantly slower than HTTP/1.1
2172 : // for unknown reasons.
2173 2414 : if (pszHttpVersion != nullptr || CPLIsMachineForSureGCEInstance())
2174 : {
2175 : static bool bDebugEmitted = false;
2176 0 : if (!bDebugEmitted)
2177 : {
2178 0 : CPLDebug("HTTP", "Using HTTP/2 for HTTPS when possible");
2179 0 : bDebugEmitted = true;
2180 : }
2181 :
2182 : // CURL_HTTP_VERSION_2TLS means for HTTPS connection, try to
2183 : // negotiate HTTP/2 with the server (and fallback to HTTP/1.1
2184 : // otherwise), and for HTTP connection do HTTP/1
2185 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2186 : CURL_HTTP_VERSION_2TLS);
2187 : }
2188 2414 : }
2189 : }
2190 : else
2191 : {
2192 0 : CPLError(CE_Warning, CPLE_NotSupported, "HTTP_VERSION=%s not supported",
2193 : pszHttpVersion);
2194 : }
2195 :
2196 : // Default value is 1 since curl 7.50.2. But worth applying it on
2197 : // previous versions as well.
2198 : const char *pszTCPNoDelay =
2199 2414 : CSLFetchNameValueDef(papszOptions, "TCP_NODELAY", "1");
2200 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_NODELAY,
2201 : atoi(pszTCPNoDelay));
2202 :
2203 : /* Support control over HTTPAUTH */
2204 2414 : const char *pszHttpAuth = CSLFetchNameValue(papszOptions, "HTTPAUTH");
2205 2414 : if (pszHttpAuth == nullptr)
2206 2404 : pszHttpAuth = CPLGetConfigOption("GDAL_HTTP_AUTH", nullptr);
2207 2414 : if (pszHttpAuth == nullptr)
2208 : {
2209 : /* do nothing */;
2210 : }
2211 15 : else if (EQUAL(pszHttpAuth, "BASIC"))
2212 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2213 : CURLAUTH_BASIC);
2214 15 : else if (EQUAL(pszHttpAuth, "NTLM"))
2215 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2216 : CURLAUTH_NTLM);
2217 15 : else if (EQUAL(pszHttpAuth, "ANY"))
2218 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
2219 15 : else if (EQUAL(pszHttpAuth, "ANYSAFE"))
2220 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2221 : CURLAUTH_ANYSAFE);
2222 15 : else if (EQUAL(pszHttpAuth, "BEARER"))
2223 : {
2224 15 : const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef(
2225 : papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES");
2226 : const bool bAuthorizationHeaderAllowed =
2227 15 : CPLTestBool(pszAuthorizationHeaderAllowed);
2228 15 : if (bAuthorizationHeaderAllowed)
2229 : {
2230 : const char *pszBearer =
2231 12 : CSLFetchNameValue(papszOptions, "HTTP_BEARER");
2232 12 : if (pszBearer == nullptr)
2233 5 : pszBearer = CPLGetConfigOption("GDAL_HTTP_BEARER", nullptr);
2234 12 : if (pszBearer != nullptr)
2235 12 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XOAUTH2_BEARER,
2236 : pszBearer);
2237 12 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2238 : CURLAUTH_BEARER);
2239 : }
2240 : }
2241 0 : else if (EQUAL(pszHttpAuth, "NEGOTIATE"))
2242 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2243 : CURLAUTH_NEGOTIATE);
2244 : else
2245 : {
2246 0 : CPLError(CE_Warning, CPLE_AppDefined,
2247 : "Unsupported HTTPAUTH value '%s', ignored.", pszHttpAuth);
2248 : }
2249 :
2250 : const char *pszGssDelegation =
2251 2414 : CSLFetchNameValue(papszOptions, "GSSAPI_DELEGATION");
2252 2414 : if (pszGssDelegation == nullptr)
2253 : pszGssDelegation =
2254 2414 : CPLGetConfigOption("GDAL_GSSAPI_DELEGATION", nullptr);
2255 2414 : if (pszGssDelegation == nullptr)
2256 : {
2257 : }
2258 0 : else if (EQUAL(pszGssDelegation, "NONE"))
2259 : {
2260 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2261 : CURLGSSAPI_DELEGATION_NONE);
2262 : }
2263 0 : else if (EQUAL(pszGssDelegation, "POLICY"))
2264 : {
2265 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2266 : CURLGSSAPI_DELEGATION_POLICY_FLAG);
2267 : }
2268 0 : else if (EQUAL(pszGssDelegation, "ALWAYS"))
2269 : {
2270 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2271 : CURLGSSAPI_DELEGATION_FLAG);
2272 : }
2273 : else
2274 : {
2275 0 : CPLError(CE_Warning, CPLE_AppDefined,
2276 : "Unsupported GSSAPI_DELEGATION value '%s', ignored.",
2277 : pszGssDelegation);
2278 : }
2279 :
2280 : // Support use of .netrc - default enabled.
2281 2414 : const char *pszHttpNetrc = CSLFetchNameValue(papszOptions, "NETRC");
2282 2414 : if (pszHttpNetrc == nullptr)
2283 2414 : pszHttpNetrc = CPLGetConfigOption("GDAL_HTTP_NETRC", "YES");
2284 2414 : if (pszHttpNetrc == nullptr || CPLTestBool(pszHttpNetrc))
2285 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC, 1L);
2286 :
2287 : // Custom .netrc file location
2288 : const char *pszHttpNetrcFile =
2289 2414 : CSLFetchNameValue(papszOptions, "NETRC_FILE");
2290 2414 : if (pszHttpNetrcFile == nullptr)
2291 2414 : pszHttpNetrcFile = CPLGetConfigOption("GDAL_HTTP_NETRC_FILE", nullptr);
2292 2414 : if (pszHttpNetrcFile)
2293 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC_FILE,
2294 : pszHttpNetrcFile);
2295 :
2296 : // Support setting userid:password.
2297 2414 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
2298 2414 : if (pszUserPwd == nullptr)
2299 2376 : pszUserPwd = CPLGetConfigOption("GDAL_HTTP_USERPWD", nullptr);
2300 2414 : if (pszUserPwd != nullptr)
2301 38 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERPWD, pszUserPwd);
2302 :
2303 : // Set Proxy parameters.
2304 2414 : const char *pszProxy = CSLFetchNameValue(papszOptions, "PROXY");
2305 2414 : if (pszProxy == nullptr)
2306 2414 : pszProxy = CPLGetConfigOption("GDAL_HTTP_PROXY", nullptr);
2307 2414 : if (pszProxy)
2308 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszProxy);
2309 :
2310 2414 : const char *pszHttpsProxy = CSLFetchNameValue(papszOptions, "HTTPS_PROXY");
2311 2414 : if (pszHttpsProxy == nullptr)
2312 2414 : pszHttpsProxy = CPLGetConfigOption("GDAL_HTTPS_PROXY", nullptr);
2313 2414 : if (pszHttpsProxy && (STARTS_WITH(pszURL, "https")))
2314 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszHttpsProxy);
2315 :
2316 : const char *pszProxyUserPwd =
2317 2414 : CSLFetchNameValue(papszOptions, "PROXYUSERPWD");
2318 2414 : if (pszProxyUserPwd == nullptr)
2319 2414 : pszProxyUserPwd = CPLGetConfigOption("GDAL_HTTP_PROXYUSERPWD", nullptr);
2320 2414 : if (pszProxyUserPwd)
2321 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYUSERPWD,
2322 : pszProxyUserPwd);
2323 :
2324 : // Support control over PROXYAUTH.
2325 2414 : const char *pszProxyAuth = CSLFetchNameValue(papszOptions, "PROXYAUTH");
2326 2414 : if (pszProxyAuth == nullptr)
2327 2414 : pszProxyAuth = CPLGetConfigOption("GDAL_PROXY_AUTH", nullptr);
2328 2414 : if (pszProxyAuth == nullptr)
2329 : {
2330 : // Do nothing.
2331 : }
2332 0 : else if (EQUAL(pszProxyAuth, "BASIC"))
2333 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2334 : CURLAUTH_BASIC);
2335 0 : else if (EQUAL(pszProxyAuth, "NTLM"))
2336 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2337 : CURLAUTH_NTLM);
2338 0 : else if (EQUAL(pszProxyAuth, "DIGEST"))
2339 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2340 : CURLAUTH_DIGEST);
2341 0 : else if (EQUAL(pszProxyAuth, "ANY"))
2342 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2343 : CURLAUTH_ANY);
2344 0 : else if (EQUAL(pszProxyAuth, "ANYSAFE"))
2345 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2346 : CURLAUTH_ANYSAFE);
2347 0 : else if (EQUAL(pszProxyAuth, "NEGOTIATE"))
2348 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2349 : CURLAUTH_NEGOTIATE);
2350 : else
2351 : {
2352 0 : CPLError(CE_Warning, CPLE_AppDefined,
2353 : "Unsupported PROXYAUTH value '%s', ignored.", pszProxyAuth);
2354 : }
2355 :
2356 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SUPPRESS_CONNECT_HEADERS,
2357 : 1L);
2358 :
2359 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
2360 2414 : const char *pszUnrestrictedAuth = CPLGetConfigOption(
2361 : "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT",
2362 : "IF_SAME_HOST");
2363 2432 : if (!EQUAL(pszUnrestrictedAuth, "IF_SAME_HOST") &&
2364 18 : CPLTestBool(pszUnrestrictedAuth))
2365 : {
2366 9 : unchecked_curl_easy_setopt(http_handle, CURLOPT_UNRESTRICTED_AUTH, 1);
2367 : }
2368 :
2369 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXREDIRS, 10);
2370 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTREDIR,
2371 : CURL_REDIR_POST_ALL);
2372 :
2373 : // Set connect timeout.
2374 : const char *pszConnectTimeout =
2375 2414 : CSLFetchNameValue(papszOptions, "CONNECTTIMEOUT");
2376 2414 : if (pszConnectTimeout == nullptr)
2377 : pszConnectTimeout =
2378 2414 : CPLGetConfigOption("GDAL_HTTP_CONNECTTIMEOUT", nullptr);
2379 2414 : if (pszConnectTimeout != nullptr)
2380 : {
2381 : // coverity[tainted_data]
2382 1 : unchecked_curl_easy_setopt(
2383 : http_handle, CURLOPT_CONNECTTIMEOUT_MS,
2384 : static_cast<int>(1000 * CPLAtof(pszConnectTimeout)));
2385 : }
2386 :
2387 : // Set timeout.
2388 2414 : const char *pszTimeout = CSLFetchNameValue(papszOptions, "TIMEOUT");
2389 2414 : if (pszTimeout == nullptr)
2390 2124 : pszTimeout = CPLGetConfigOption("GDAL_HTTP_TIMEOUT", nullptr);
2391 2414 : if (pszTimeout != nullptr)
2392 : {
2393 : // coverity[tainted_data]
2394 299 : unchecked_curl_easy_setopt(
2395 : http_handle, CURLOPT_TIMEOUT_MS,
2396 : static_cast<int>(1000 * CPLAtof(pszTimeout)));
2397 : }
2398 :
2399 : // Set low speed time and limit.
2400 : const char *pszLowSpeedTime =
2401 2414 : CSLFetchNameValue(papszOptions, "LOW_SPEED_TIME");
2402 2414 : if (pszLowSpeedTime == nullptr)
2403 : pszLowSpeedTime =
2404 2414 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_TIME", nullptr);
2405 2414 : if (pszLowSpeedTime != nullptr)
2406 : {
2407 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_TIME,
2408 : atoi(pszLowSpeedTime));
2409 :
2410 : const char *pszLowSpeedLimit =
2411 0 : CSLFetchNameValue(papszOptions, "LOW_SPEED_LIMIT");
2412 0 : if (pszLowSpeedLimit == nullptr)
2413 : pszLowSpeedLimit =
2414 0 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_LIMIT", "1");
2415 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_LIMIT,
2416 : atoi(pszLowSpeedLimit));
2417 : }
2418 :
2419 : /* Disable some SSL verification */
2420 2414 : const char *pszUnsafeSSL = CSLFetchNameValue(papszOptions, "UNSAFESSL");
2421 2414 : if (pszUnsafeSSL == nullptr)
2422 2393 : pszUnsafeSSL = CPLGetConfigOption("GDAL_HTTP_UNSAFESSL", nullptr);
2423 2414 : if (pszUnsafeSSL != nullptr && CPLTestBool(pszUnsafeSSL))
2424 : {
2425 21 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYPEER, 0L);
2426 21 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYHOST, 0L);
2427 : }
2428 :
2429 : const char *pszUseCAPIStore =
2430 2414 : CSLFetchNameValue(papszOptions, "USE_CAPI_STORE");
2431 2414 : if (pszUseCAPIStore == nullptr)
2432 2414 : pszUseCAPIStore = CPLGetConfigOption("GDAL_HTTP_USE_CAPI_STORE", "NO");
2433 2414 : if (CPLTestBool(pszUseCAPIStore))
2434 : {
2435 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2436 : // Use certificates from Windows certificate store; requires
2437 : // crypt32.lib, OpenSSL crypto and ssl libraries.
2438 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_CTX_FUNCTION,
2439 : *CPL_ssl_ctx_callback);
2440 : #else // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2441 2 : CPLError(CE_Warning, CPLE_NotSupported,
2442 : "GDAL_HTTP_USE_CAPI_STORE requested, but libcurl too old, "
2443 : "non-Windows platform or OpenSSL missing.");
2444 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2445 : }
2446 :
2447 : // Enable OCSP stapling if requested.
2448 : const char *pszSSLVerifyStatus =
2449 2414 : CSLFetchNameValue(papszOptions, "SSL_VERIFYSTATUS");
2450 2414 : if (pszSSLVerifyStatus == nullptr)
2451 : pszSSLVerifyStatus =
2452 2414 : CPLGetConfigOption("GDAL_HTTP_SSL_VERIFYSTATUS", "NO");
2453 2414 : if (CPLTestBool(pszSSLVerifyStatus))
2454 : {
2455 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYSTATUS, 1L);
2456 : }
2457 :
2458 : // Custom path to SSL certificates.
2459 2414 : const char *pszCAInfo = CSLFetchNameValue(papszOptions, "CAINFO");
2460 2414 : if (pszCAInfo == nullptr)
2461 : // Name of GDAL environment variable for the CA Bundle path
2462 2414 : pszCAInfo = CPLGetConfigOption("GDAL_CURL_CA_BUNDLE", nullptr);
2463 2414 : if (pszCAInfo == nullptr)
2464 : // Name of environment variable used by the curl binary
2465 2414 : pszCAInfo = CPLGetConfigOption("CURL_CA_BUNDLE", nullptr);
2466 2414 : if (pszCAInfo == nullptr)
2467 : // Name of environment variable used by the curl binary (tested
2468 : // after CURL_CA_BUNDLE
2469 2414 : pszCAInfo = CPLGetConfigOption("SSL_CERT_FILE", nullptr);
2470 : #ifdef _WIN32
2471 : if (pszCAInfo == nullptr)
2472 : {
2473 : pszCAInfo = CPLFindWin32CurlCaBundleCrt();
2474 : }
2475 : #endif
2476 2414 : if (pszCAInfo != nullptr)
2477 : {
2478 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAINFO, pszCAInfo);
2479 : }
2480 :
2481 2414 : const char *pszCAPath = CSLFetchNameValue(papszOptions, "CAPATH");
2482 2414 : if (pszCAPath != nullptr)
2483 : {
2484 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAPATH, pszCAPath);
2485 : }
2486 :
2487 : // Support for SSL client certificates
2488 :
2489 : // Filename of the the client certificate
2490 2414 : const char *pszSSLCert = CSLFetchNameValue(papszOptions, "SSLCERT");
2491 2414 : if (!pszSSLCert)
2492 2414 : pszSSLCert = CPLGetConfigOption("GDAL_HTTP_SSLCERT", nullptr);
2493 2414 : if (pszSSLCert)
2494 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERT, pszSSLCert);
2495 :
2496 : // private key file for TLS and SSL client cert
2497 2414 : const char *pszSSLKey = CSLFetchNameValue(papszOptions, "SSLKEY");
2498 2414 : if (!pszSSLKey)
2499 2414 : pszSSLKey = CPLGetConfigOption("GDAL_HTTP_SSLKEY", nullptr);
2500 2414 : if (pszSSLKey)
2501 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLKEY, pszSSLKey);
2502 :
2503 : // type of client SSL certificate ("PEM", "DER", ...)
2504 2414 : const char *pszSSLCertType = CSLFetchNameValue(papszOptions, "SSLCERTTYPE");
2505 2414 : if (!pszSSLCertType)
2506 2414 : pszSSLCertType = CPLGetConfigOption("GDAL_HTTP_SSLCERTTYPE", nullptr);
2507 2414 : if (pszSSLCertType)
2508 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERTTYPE,
2509 : pszSSLCertType);
2510 :
2511 : // passphrase to private key
2512 2414 : const char *pszKeyPasswd = CSLFetchNameValue(papszOptions, "KEYPASSWD");
2513 2414 : if (!pszKeyPasswd)
2514 2414 : pszKeyPasswd = CPLGetConfigOption("GDAL_HTTP_KEYPASSWD", nullptr);
2515 2414 : if (pszKeyPasswd)
2516 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_KEYPASSWD,
2517 : pszKeyPasswd);
2518 :
2519 : /* Set Referer */
2520 2414 : const char *pszReferer = CSLFetchNameValue(papszOptions, "REFERER");
2521 2414 : if (pszReferer != nullptr)
2522 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_REFERER, pszReferer);
2523 :
2524 : /* Set User-Agent */
2525 2414 : const char *pszUserAgent = CSLFetchNameValue(papszOptions, "USERAGENT");
2526 2414 : if (pszUserAgent == nullptr)
2527 2369 : pszUserAgent = CPLGetConfigOption("GDAL_HTTP_USERAGENT",
2528 : gosDefaultUserAgent.c_str());
2529 2414 : if (pszUserAgent != nullptr && !EQUAL(pszUserAgent, ""))
2530 : {
2531 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERAGENT,
2532 : pszUserAgent);
2533 : }
2534 :
2535 : /* NOSIGNAL should be set to true for timeout to work in multithread
2536 : * environments on Unix, requires libcurl 7.10 or more recent.
2537 : * (this force avoiding the use of signal handlers)
2538 : */
2539 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOSIGNAL, 1);
2540 :
2541 : const char *pszFormFilePath =
2542 2414 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
2543 : const char *pszParametersCount =
2544 2414 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
2545 2414 : if (pszFormFilePath == nullptr && pszParametersCount == nullptr)
2546 : {
2547 : /* Set POST mode */
2548 2412 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
2549 2412 : if (pszPost != nullptr)
2550 : {
2551 159 : CPLDebug("HTTP", "These POSTFIELDS were sent:%.4000s", pszPost);
2552 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 1);
2553 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS,
2554 : pszPost);
2555 : }
2556 : }
2557 :
2558 : const char *pszCustomRequest =
2559 2414 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
2560 2414 : if (pszCustomRequest != nullptr)
2561 : {
2562 138 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CUSTOMREQUEST,
2563 : pszCustomRequest);
2564 : }
2565 :
2566 2414 : const char *pszCookie = CSLFetchNameValue(papszOptions, "COOKIE");
2567 2414 : if (pszCookie == nullptr)
2568 2414 : pszCookie = CPLGetConfigOption("GDAL_HTTP_COOKIE", nullptr);
2569 2414 : if (pszCookie != nullptr)
2570 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIE, pszCookie);
2571 :
2572 2414 : const char *pszCookieFile = CSLFetchNameValue(papszOptions, "COOKIEFILE");
2573 2414 : if (pszCookieFile == nullptr)
2574 2414 : pszCookieFile = CPLGetConfigOption("GDAL_HTTP_COOKIEFILE", nullptr);
2575 2414 : if (pszCookieFile != nullptr)
2576 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEFILE,
2577 : pszCookieFile);
2578 :
2579 2414 : const char *pszCookieJar = CSLFetchNameValue(papszOptions, "COOKIEJAR");
2580 2414 : if (pszCookieJar == nullptr)
2581 2414 : pszCookieJar = CPLGetConfigOption("GDAL_HTTP_COOKIEJAR", nullptr);
2582 2414 : if (pszCookieJar != nullptr)
2583 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEJAR,
2584 : pszCookieJar);
2585 :
2586 : // TCP keep-alive
2587 : const char *pszTCPKeepAlive =
2588 2414 : CSLFetchNameValue(papszOptions, "TCP_KEEPALIVE");
2589 2414 : if (pszTCPKeepAlive == nullptr)
2590 2414 : pszTCPKeepAlive = CPLGetConfigOption("GDAL_HTTP_TCP_KEEPALIVE", "YES");
2591 2414 : if (pszTCPKeepAlive != nullptr && CPLTestBool(pszTCPKeepAlive))
2592 : {
2593 : // Set keep-alive interval.
2594 2414 : int nKeepAliveInterval = 60;
2595 : const char *pszKeepAliveInterval =
2596 2414 : CSLFetchNameValue(papszOptions, "TCP_KEEPINTVL");
2597 2414 : if (pszKeepAliveInterval == nullptr)
2598 : pszKeepAliveInterval =
2599 2414 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPINTVL", nullptr);
2600 2414 : if (pszKeepAliveInterval != nullptr)
2601 1 : nKeepAliveInterval = atoi(pszKeepAliveInterval);
2602 :
2603 : // Set keep-alive idle wait time.
2604 2414 : int nKeepAliveIdle = 60;
2605 : const char *pszKeepAliveIdle =
2606 2414 : CSLFetchNameValue(papszOptions, "TCP_KEEPIDLE");
2607 2414 : if (pszKeepAliveIdle == nullptr)
2608 : pszKeepAliveIdle =
2609 2414 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPIDLE", nullptr);
2610 2414 : if (pszKeepAliveIdle != nullptr)
2611 1 : nKeepAliveIdle = atoi(pszKeepAliveIdle);
2612 :
2613 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPALIVE, 1L);
2614 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPINTVL,
2615 : nKeepAliveInterval);
2616 2414 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPIDLE,
2617 : nKeepAliveIdle);
2618 : }
2619 :
2620 2414 : struct curl_slist *headers = nullptr;
2621 2414 : const char *pszHeaderFile = CSLFetchNameValue(papszOptions, "HEADER_FILE");
2622 2414 : if (pszHeaderFile == nullptr)
2623 2412 : pszHeaderFile = CPLGetConfigOption("GDAL_HTTP_HEADER_FILE", nullptr);
2624 2414 : if (pszHeaderFile != nullptr)
2625 : {
2626 2 : VSILFILE *fp = nullptr;
2627 : // Do not allow /vsicurl/ access from /vsicurl because of
2628 : // GetCurlHandleFor() e.g. "/vsicurl/,HEADER_FILE=/vsicurl/,url= " would
2629 : // cause use of memory after free
2630 2 : if (!STARTS_WITH(pszHeaderFile, "/vsi") ||
2631 2 : STARTS_WITH(pszHeaderFile, "/vsimem/"))
2632 : {
2633 2 : fp = VSIFOpenL(pszHeaderFile, "rb");
2634 : }
2635 2 : if (fp == nullptr)
2636 : {
2637 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot read %s", pszHeaderFile);
2638 : }
2639 : else
2640 : {
2641 2 : const char *pszLine = nullptr;
2642 4 : while ((pszLine = CPLReadLineL(fp)) != nullptr)
2643 : {
2644 2 : headers = curl_slist_append(headers, pszLine);
2645 : }
2646 2 : VSIFCloseL(fp);
2647 : }
2648 : }
2649 :
2650 2414 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
2651 2414 : if (pszHeaders == nullptr)
2652 1825 : pszHeaders = CPLGetConfigOption("GDAL_HTTP_HEADERS", nullptr);
2653 2414 : if (pszHeaders)
2654 : {
2655 597 : bool bHeadersDone = false;
2656 : // Compatibility hack for "HEADERS=Accept: text/plain, application/json"
2657 597 : if (strstr(pszHeaders, "\r\n") == nullptr)
2658 : {
2659 541 : const char *pszComma = strchr(pszHeaders, ',');
2660 541 : if (pszComma != nullptr && strchr(pszComma, ':') == nullptr)
2661 : {
2662 207 : headers = curl_slist_append(headers, pszHeaders);
2663 207 : bHeadersDone = true;
2664 : }
2665 : }
2666 597 : if (!bHeadersDone)
2667 : {
2668 390 : const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef(
2669 : papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES");
2670 : const bool bAuthorizationHeaderAllowed =
2671 390 : CPLTestBool(pszAuthorizationHeaderAllowed);
2672 :
2673 : // We accept both raw headers with \r\n as a separator, or as
2674 : // a comma separated list of foo: bar values.
2675 : const CPLStringList aosTokens(
2676 390 : strstr(pszHeaders, "\r\n")
2677 56 : ? CSLTokenizeString2(pszHeaders, "\r\n", 0)
2678 836 : : CSLTokenizeString2(pszHeaders, ",", CSLT_HONOURSTRINGS));
2679 842 : for (int i = 0; i < aosTokens.size(); ++i)
2680 : {
2681 456 : if (bAuthorizationHeaderAllowed ||
2682 4 : !STARTS_WITH_CI(aosTokens[i], "Authorization:"))
2683 : {
2684 448 : headers = curl_slist_append(headers, aosTokens[i]);
2685 : }
2686 : }
2687 : }
2688 : }
2689 :
2690 2414 : return headers;
2691 : }
2692 :
2693 : /************************************************************************/
2694 : /* CPLHTTPIgnoreSigPipe() */
2695 : /************************************************************************/
2696 :
2697 : /* If using OpenSSL with Curl, openssl can cause SIGPIPE to be triggered */
2698 : /* As we set CURLOPT_NOSIGNAL = 1, we must manually handle this situation */
2699 :
2700 2606 : void *CPLHTTPIgnoreSigPipe()
2701 : {
2702 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2703 : struct sigaction old_pipe_act;
2704 : struct sigaction action;
2705 : /* Get previous handler */
2706 2606 : memset(&old_pipe_act, 0, sizeof(struct sigaction));
2707 2606 : sigaction(SIGPIPE, nullptr, &old_pipe_act);
2708 :
2709 : /* Install new handler */
2710 2606 : action = old_pipe_act;
2711 2606 : action.sa_handler = SIG_IGN;
2712 2606 : sigaction(SIGPIPE, &action, nullptr);
2713 :
2714 2606 : void *ret = CPLMalloc(sizeof(old_pipe_act));
2715 2606 : memcpy(ret, &old_pipe_act, sizeof(old_pipe_act));
2716 2606 : return ret;
2717 : #else
2718 : return nullptr;
2719 : #endif
2720 : }
2721 :
2722 : /************************************************************************/
2723 : /* CPLHTTPRestoreSigPipeHandler() */
2724 : /************************************************************************/
2725 :
2726 2606 : void CPLHTTPRestoreSigPipeHandler(void *old_handler)
2727 : {
2728 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2729 2606 : sigaction(SIGPIPE, static_cast<struct sigaction *>(old_handler), nullptr);
2730 2606 : CPLFree(old_handler);
2731 : #else
2732 : (void)old_handler;
2733 : #endif
2734 2606 : }
2735 :
2736 : #endif // def HAVE_CURL
2737 :
2738 : /************************************************************************/
2739 : /* CPLHTTPEnabled() */
2740 : /************************************************************************/
2741 :
2742 : /**
2743 : * \brief Return if CPLHTTP services can be useful
2744 : *
2745 : * Those services depend on GDAL being build with libcurl support.
2746 : *
2747 : * @return TRUE if libcurl support is enabled
2748 : */
2749 8 : int CPLHTTPEnabled()
2750 :
2751 : {
2752 : #ifdef HAVE_CURL
2753 8 : return TRUE;
2754 : #else
2755 : return FALSE;
2756 : #endif
2757 : }
2758 :
2759 : /************************************************************************/
2760 : /* CPLHTTPCleanup() */
2761 : /************************************************************************/
2762 :
2763 : /**
2764 : * \brief Cleanup function to call at application termination
2765 : */
2766 933 : void CPLHTTPCleanup()
2767 :
2768 : {
2769 : #ifdef HAVE_CURL
2770 933 : if (!hSessionMapMutex)
2771 932 : return;
2772 :
2773 : {
2774 2 : CPLMutexHolder oHolder(&hSessionMapMutex);
2775 1 : if (poSessionMap)
2776 : {
2777 0 : for (auto &kv : *poSessionMap)
2778 : {
2779 0 : curl_easy_cleanup(kv.second);
2780 : }
2781 0 : delete poSessionMap;
2782 0 : poSessionMap = nullptr;
2783 : }
2784 1 : if (poSessionMultiMap)
2785 : {
2786 0 : for (auto &kv : *poSessionMultiMap)
2787 : {
2788 0 : VSICURLMultiCleanup(kv.second);
2789 : }
2790 0 : delete poSessionMultiMap;
2791 0 : poSessionMultiMap = nullptr;
2792 : }
2793 : }
2794 :
2795 : // Not quite a safe sequence.
2796 1 : CPLDestroyMutex(hSessionMapMutex);
2797 1 : hSessionMapMutex = nullptr;
2798 :
2799 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2800 : // This cleanup must be absolutely done before CPLOpenSSLCleanup()
2801 : // for some unknown reason, but otherwise X509_free() in
2802 : // CPLWindowsCertificateListCleanup() will crash.
2803 : CPLWindowsCertificateListCleanup();
2804 : #endif
2805 :
2806 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
2807 : CPLOpenSSLCleanup();
2808 : #endif
2809 :
2810 : #endif
2811 : }
2812 :
2813 : /************************************************************************/
2814 : /* CPLHTTPDestroyResult() */
2815 : /************************************************************************/
2816 :
2817 : /**
2818 : * \brief Clean the memory associated with the return value of CPLHTTPFetch()
2819 : *
2820 : * @param psResult pointer to the return value of CPLHTTPFetch()
2821 : */
2822 2116 : void CPLHTTPDestroyResult(CPLHTTPResult *psResult)
2823 :
2824 : {
2825 2116 : if (psResult)
2826 : {
2827 1929 : CPLFree(psResult->pabyData);
2828 1929 : CPLFree(psResult->pszErrBuf);
2829 1929 : CPLFree(psResult->pszContentType);
2830 1929 : CSLDestroy(psResult->papszHeaders);
2831 :
2832 1986 : for (int i = 0; i < psResult->nMimePartCount; i++)
2833 : {
2834 57 : CSLDestroy(psResult->pasMimePart[i].papszHeaders);
2835 : }
2836 1929 : CPLFree(psResult->pasMimePart);
2837 :
2838 1929 : CPLFree(psResult);
2839 : }
2840 2116 : }
2841 :
2842 : /************************************************************************/
2843 : /* CPLHTTPParseMultipartMime() */
2844 : /************************************************************************/
2845 :
2846 : /**
2847 : * \brief Parses a MIME multipart message.
2848 : *
2849 : * This function will iterate over each part and put it in a separate
2850 : * element of the pasMimePart array of the provided psResult structure.
2851 : *
2852 : * @param psResult pointer to the return value of CPLHTTPFetch()
2853 : * @return TRUE if the message contains MIME multipart message.
2854 : */
2855 45 : int CPLHTTPParseMultipartMime(CPLHTTPResult *psResult)
2856 :
2857 : {
2858 : /* -------------------------------------------------------------------- */
2859 : /* Is it already done? */
2860 : /* -------------------------------------------------------------------- */
2861 45 : if (psResult->nMimePartCount > 0)
2862 5 : return TRUE;
2863 :
2864 : /* -------------------------------------------------------------------- */
2865 : /* Find the boundary setting in the content type. */
2866 : /* -------------------------------------------------------------------- */
2867 40 : const char *pszBound = nullptr;
2868 :
2869 40 : if (psResult->pszContentType != nullptr)
2870 38 : pszBound = strstr(psResult->pszContentType, "boundary=");
2871 :
2872 40 : if (pszBound == nullptr)
2873 : {
2874 2 : CPLError(CE_Failure, CPLE_AppDefined,
2875 : "Unable to parse multi-part mime, no boundary setting.");
2876 2 : return FALSE;
2877 : }
2878 :
2879 76 : CPLString osBoundary;
2880 : char **papszTokens =
2881 38 : CSLTokenizeStringComplex(pszBound + 9, "\n ;", TRUE, FALSE);
2882 :
2883 38 : if (CSLCount(papszTokens) == 0 || strlen(papszTokens[0]) == 0)
2884 : {
2885 1 : CPLError(CE_Failure, CPLE_AppDefined,
2886 : "Unable to parse multi-part mime, boundary not parsable.");
2887 1 : CSLDestroy(papszTokens);
2888 1 : return FALSE;
2889 : }
2890 :
2891 37 : osBoundary = "--";
2892 37 : osBoundary += papszTokens[0];
2893 37 : CSLDestroy(papszTokens);
2894 :
2895 : /* -------------------------------------------------------------------- */
2896 : /* Find the start of the first chunk. */
2897 : /* -------------------------------------------------------------------- */
2898 37 : char *pszNext = psResult->pabyData
2899 37 : ? strstr(reinterpret_cast<char *>(psResult->pabyData),
2900 : osBoundary.c_str())
2901 37 : : nullptr;
2902 :
2903 37 : if (pszNext == nullptr)
2904 : {
2905 1 : CPLError(CE_Failure, CPLE_AppDefined, "No parts found.");
2906 1 : return FALSE;
2907 : }
2908 :
2909 36 : pszNext += osBoundary.size();
2910 146 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2911 110 : pszNext++;
2912 36 : if (*pszNext == '\r')
2913 30 : pszNext++;
2914 36 : if (*pszNext == '\n')
2915 36 : pszNext++;
2916 :
2917 : /* -------------------------------------------------------------------- */
2918 : /* Loop over parts... */
2919 : /* -------------------------------------------------------------------- */
2920 : while (true)
2921 : {
2922 57 : psResult->nMimePartCount++;
2923 57 : psResult->pasMimePart = static_cast<CPLMimePart *>(
2924 114 : CPLRealloc(psResult->pasMimePart,
2925 57 : sizeof(CPLMimePart) * psResult->nMimePartCount));
2926 :
2927 57 : CPLMimePart *psPart =
2928 57 : psResult->pasMimePart + psResult->nMimePartCount - 1;
2929 :
2930 57 : memset(psPart, 0, sizeof(CPLMimePart));
2931 :
2932 : /* --------------------------------------------------------------------
2933 : */
2934 : /* Collect headers. */
2935 : /* --------------------------------------------------------------------
2936 : */
2937 145 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2938 : {
2939 90 : if (!STARTS_WITH(pszNext, "Content-"))
2940 : {
2941 1 : break;
2942 : }
2943 89 : char *pszEOL = strstr(pszNext, "\n");
2944 :
2945 89 : if (pszEOL == nullptr)
2946 : {
2947 1 : CPLError(CE_Failure, CPLE_AppDefined,
2948 : "Error while parsing multipart content (at line %d)",
2949 : __LINE__);
2950 1 : return FALSE;
2951 : }
2952 :
2953 88 : *pszEOL = '\0';
2954 88 : bool bRestoreAntislashR = false;
2955 88 : if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r')
2956 : {
2957 80 : bRestoreAntislashR = true;
2958 80 : pszEOL[-1] = '\0';
2959 : }
2960 88 : char *pszKey = nullptr;
2961 88 : const char *pszValue = CPLParseNameValue(pszNext, &pszKey);
2962 88 : if (pszKey && pszValue)
2963 : {
2964 88 : psPart->papszHeaders =
2965 88 : CSLSetNameValue(psPart->papszHeaders, pszKey, pszValue);
2966 : }
2967 88 : CPLFree(pszKey);
2968 88 : if (bRestoreAntislashR)
2969 80 : pszEOL[-1] = '\r';
2970 88 : *pszEOL = '\n';
2971 :
2972 88 : pszNext = pszEOL + 1;
2973 : }
2974 :
2975 56 : if (*pszNext == '\r')
2976 54 : pszNext++;
2977 56 : if (*pszNext == '\n')
2978 54 : pszNext++;
2979 :
2980 : /* --------------------------------------------------------------------
2981 : */
2982 : /* Work out the data block size. */
2983 : /* --------------------------------------------------------------------
2984 : */
2985 56 : psPart->pabyData = reinterpret_cast<GByte *>(pszNext);
2986 :
2987 56 : int nBytesAvail = psResult->nDataLen -
2988 56 : static_cast<int>(pszNext - reinterpret_cast<char *>(
2989 56 : psResult->pabyData));
2990 :
2991 6579040 : while (nBytesAvail > 0 &&
2992 3289520 : (*pszNext != '-' ||
2993 109 : strncmp(pszNext, osBoundary, osBoundary.size()) != 0))
2994 : {
2995 3289470 : pszNext++;
2996 3289470 : nBytesAvail--;
2997 : }
2998 :
2999 56 : if (nBytesAvail == 0)
3000 : {
3001 2 : CPLError(CE_Failure, CPLE_AppDefined,
3002 : "Error while parsing multipart content (at line %d)",
3003 : __LINE__);
3004 2 : return FALSE;
3005 : }
3006 :
3007 54 : psPart->nDataLen = static_cast<int>(
3008 54 : pszNext - reinterpret_cast<char *>(psPart->pabyData));
3009 : // Normally the part should end with "\r\n--boundary_marker"
3010 54 : if (psPart->nDataLen >= 2 && pszNext[-2] == '\r' && pszNext[-1] == '\n')
3011 : {
3012 46 : psPart->nDataLen -= 2;
3013 : }
3014 :
3015 54 : pszNext += osBoundary.size();
3016 :
3017 54 : if (STARTS_WITH(pszNext, "--"))
3018 : {
3019 31 : break;
3020 : }
3021 :
3022 23 : if (*pszNext == '\r')
3023 19 : pszNext++;
3024 23 : if (*pszNext == '\n')
3025 21 : pszNext++;
3026 : else
3027 : {
3028 2 : CPLError(CE_Failure, CPLE_AppDefined,
3029 : "Error while parsing multipart content (at line %d)",
3030 : __LINE__);
3031 2 : return FALSE;
3032 : }
3033 21 : }
3034 :
3035 31 : return TRUE;
3036 : }
3037 :
3038 : #if defined(__GNUC__)
3039 : #pragma GCC diagnostic pop
3040 : #endif
|
