Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: libcurl based HTTP client
4 : * Purpose: libcurl based HTTP client
5 : * Author: Frank Warmerdam, warmerdam@pobox.com
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2006, Frank Warmerdam
9 : * Copyright (c) 2008-2013, Even Rouault <even dot rouault at spatialys.com>
10 : *
11 : * SPDX-License-Identifier: MIT
12 : ****************************************************************************/
13 :
14 : #include "cpl_port.h"
15 : #include "cpl_http.h"
16 :
17 : #include <cstddef>
18 : #include <cstring>
19 :
20 : #include <algorithm>
21 : #include <array>
22 : #include <map>
23 : #include <mutex>
24 : #include <string>
25 : #include <vector>
26 :
27 : #include "cpl_http.h"
28 : #include "cpl_error.h"
29 : #include "cpl_multiproc.h"
30 : #include "cpl_vsi_virtual.h"
31 : #include "cpl_vsil_curl_class.h"
32 :
33 : // gcc or clang complains about C-style cast in #define like
34 : // CURL_ZERO_TERMINATED
35 : #if defined(__GNUC__)
36 : #pragma GCC diagnostic push
37 : #pragma GCC diagnostic ignored "-Wold-style-cast"
38 : #endif
39 :
40 : #ifdef HAVE_CURL
41 :
42 : #include "cpl_curl_priv.h"
43 :
44 : #ifdef HAVE_OPENSSL_CRYPTO
45 : #include <openssl/err.h>
46 : #include <openssl/ssl.h>
47 : #include <openssl/x509v3.h>
48 :
49 : #if defined(_WIN32)
50 : #include <wincrypt.h>
51 : #endif
52 :
53 : #endif
54 :
55 : #ifdef HAVE_SIGACTION
56 : #include <signal.h>
57 : #endif
58 :
59 : #define unchecked_curl_easy_setopt(handle, opt, param) \
60 : CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param))
61 :
62 : #endif // HAVE_CURL
63 :
64 : // list of named persistent http sessions
65 :
66 : #ifdef HAVE_CURL
67 : static std::map<CPLString, CURL *> *poSessionMap = nullptr;
68 : static std::map<CPLString, CURLM *> *poSessionMultiMap = nullptr;
69 : static CPLMutex *hSessionMapMutex = nullptr;
70 : static bool bHasCheckVersion = false;
71 : static bool bSupportGZip = false;
72 : static bool bSupportHTTP2 = false;
73 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
74 : static std::vector<X509 *> *poWindowsCertificateList = nullptr;
75 :
76 : #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
77 : #define EVP_PKEY_get0_RSA(x) (x->pkey.rsa)
78 : #define EVP_PKEY_get0_DSA(x) (x->pkey.dsa)
79 : #define X509_get_extension_flags(x) (x->ex_flags)
80 : #define X509_get_key_usage(x) (x->ex_kusage)
81 : #define X509_get_extended_key_usage(x) (x->ex_xkusage)
82 : #endif
83 :
84 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
85 :
86 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
87 :
88 : // Ported from https://curl.haxx.se/libcurl/c/opensslthreadlock.html
89 : static CPLMutex **pahSSLMutex = nullptr;
90 :
91 : static void CPLOpenSSLLockingFunction(int mode, int n, const char * /*file*/,
92 : int /*line*/)
93 : {
94 : if (mode & CRYPTO_LOCK)
95 : {
96 : CPLAcquireMutex(pahSSLMutex[n], 3600.0);
97 : }
98 : else
99 : {
100 : CPLReleaseMutex(pahSSLMutex[n]);
101 : }
102 : }
103 :
104 : static unsigned long CPLOpenSSLIdCallback(void)
105 : {
106 : return static_cast<unsigned long>(CPLGetPID());
107 : }
108 :
109 : static void CPLOpenSSLInit()
110 : {
111 : if (strstr(curl_version(), "OpenSSL") &&
112 : CPLTestBool(CPLGetConfigOption("CPL_OPENSSL_INIT_ENABLED", "YES")) &&
113 : CRYPTO_get_id_callback() == nullptr)
114 : {
115 : pahSSLMutex = static_cast<CPLMutex **>(
116 : CPLMalloc(CRYPTO_num_locks() * sizeof(CPLMutex *)));
117 : for (int i = 0; i < CRYPTO_num_locks(); i++)
118 : {
119 : pahSSLMutex[i] = CPLCreateMutex();
120 : CPLReleaseMutex(pahSSLMutex[i]);
121 : }
122 : CRYPTO_set_id_callback(CPLOpenSSLIdCallback);
123 : CRYPTO_set_locking_callback(CPLOpenSSLLockingFunction);
124 : }
125 : }
126 :
127 : static void CPLOpenSSLCleanup()
128 : {
129 : if (pahSSLMutex)
130 : {
131 : for (int i = 0; i < CRYPTO_num_locks(); i++)
132 : {
133 : CPLDestroyMutex(pahSSLMutex[i]);
134 : }
135 : CPLFree(pahSSLMutex);
136 : pahSSLMutex = nullptr;
137 : CRYPTO_set_id_callback(nullptr);
138 : CRYPTO_set_locking_callback(nullptr);
139 : }
140 : }
141 :
142 : #endif
143 :
144 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
145 :
146 : /************************************************************************/
147 : /* CPLWindowsCertificateListCleanup() */
148 : /************************************************************************/
149 :
150 : static void CPLWindowsCertificateListCleanup()
151 : {
152 : if (poWindowsCertificateList)
153 : {
154 : for (auto &&pX509 : *poWindowsCertificateList)
155 : {
156 : X509_free(pX509);
157 : }
158 : delete poWindowsCertificateList;
159 : poWindowsCertificateList = nullptr;
160 : }
161 : }
162 :
163 : /************************************************************************/
164 : /* LoadCAPICertificates() */
165 : /************************************************************************/
166 :
167 : static CPLErr LoadCAPICertificates(const char *pszName,
168 : std::vector<X509 *> *poCertificateList)
169 : {
170 : CPLAssert(pszName);
171 : CPLAssert(poCertificateList);
172 :
173 : HCERTSTORE pCertStore = CertOpenSystemStore(
174 : reinterpret_cast<HCRYPTPROV_LEGACY>(nullptr), pszName);
175 : if (pCertStore == nullptr)
176 : {
177 : CPLError(CE_Failure, CPLE_AppDefined,
178 : "CPLLoadCAPICertificates(): Unable open system "
179 : "certificate store %s.",
180 : pszName);
181 : return CE_Failure;
182 : }
183 :
184 : PCCERT_CONTEXT pCertificate =
185 : CertEnumCertificatesInStore(pCertStore, nullptr);
186 : while (pCertificate != nullptr)
187 : {
188 : X509 *pX509 = d2i_X509(
189 : nullptr,
190 : const_cast<unsigned char const **>(&pCertificate->pbCertEncoded),
191 : pCertificate->cbCertEncoded);
192 : if (pX509 == nullptr)
193 : {
194 : CPLError(CE_Warning, CPLE_AppDefined,
195 : "CPLLoadCAPICertificates(): CertEnumCertificatesInStore() "
196 : "returned a null certificate, skipping.");
197 : }
198 : else
199 : {
200 : #ifdef DEBUG_VERBOSE
201 : char szSubject[256] = {0};
202 : CPLString osSubject;
203 : X509_NAME *pName = X509_get_subject_name(pX509);
204 : if (pName)
205 : {
206 : X509_NAME_oneline(pName, szSubject, sizeof(szSubject));
207 : osSubject = szSubject;
208 : }
209 : if (!osSubject.empty())
210 : CPLDebug("HTTP", "SSL Certificate: %s", osSubject.c_str());
211 : #endif
212 : poCertificateList->push_back(pX509);
213 : }
214 : pCertificate = CertEnumCertificatesInStore(pCertStore, pCertificate);
215 : }
216 : CertCloseStore(pCertStore, 0);
217 : return CE_None;
218 : }
219 :
220 : /************************************************************************/
221 : /* CPL_ssl_ctx_callback() */
222 : /************************************************************************/
223 :
224 : // Load certificates from Windows Crypto API store.
225 : static CURLcode CPL_ssl_ctx_callback(CURL *, void *pSSL, void *)
226 : {
227 : SSL_CTX *pSSL_CTX = static_cast<SSL_CTX *>(pSSL);
228 : if (pSSL_CTX == nullptr)
229 : {
230 : CPLError(CE_Failure, CPLE_AppDefined,
231 : "CPL_ssl_ctx_callback(): OpenSSL context pointer is NULL.");
232 : return CURLE_ABORTED_BY_CALLBACK;
233 : }
234 :
235 : static std::mutex goMutex;
236 : {
237 : std::lock_guard<std::mutex> oLock(goMutex);
238 : if (poWindowsCertificateList == nullptr)
239 : {
240 : poWindowsCertificateList = new std::vector<X509 *>();
241 : if (!poWindowsCertificateList)
242 : {
243 : CPLError(CE_Failure, CPLE_AppDefined,
244 : "CPL_ssl_ctx_callback(): Unable to allocate "
245 : "structure to hold certificates.");
246 : return CURLE_FAILED_INIT;
247 : }
248 :
249 : const std::array<const char *, 3> aszStores{
250 : {"CA", "AuthRoot", "ROOT"}};
251 : for (auto &&pszStore : aszStores)
252 : {
253 : if (LoadCAPICertificates(pszStore, poWindowsCertificateList) ==
254 : CE_Failure)
255 : {
256 : CPLError(
257 : CE_Failure, CPLE_AppDefined,
258 : "CPL_ssl_ctx_callback(): Unable to load certificates "
259 : "from '%s' store.",
260 : pszStore);
261 : return CURLE_FAILED_INIT;
262 : }
263 : }
264 :
265 : CPLDebug("HTTP", "Loading %d certificates from Windows store.",
266 : static_cast<int>(poWindowsCertificateList->size()));
267 : }
268 : }
269 :
270 : X509_STORE *pX509Store = SSL_CTX_get_cert_store(pSSL_CTX);
271 : for (X509 *x509 : *poWindowsCertificateList)
272 : X509_STORE_add_cert(pX509Store, x509);
273 :
274 : return CURLE_OK;
275 : }
276 :
277 : #endif // defined(_WIN32) && defined (HAVE_OPENSSL_CRYPTO)
278 :
279 : /************************************************************************/
280 : /* CheckCurlFeatures() */
281 : /************************************************************************/
282 :
283 2587 : static void CheckCurlFeatures()
284 : {
285 5174 : CPLMutexHolder oHolder(&hSessionMapMutex);
286 2587 : if (!bHasCheckVersion)
287 : {
288 6 : const char *pszVersion = curl_version();
289 6 : CPLDebug("HTTP", "%s", pszVersion);
290 6 : bSupportGZip = strstr(pszVersion, "zlib/") != nullptr;
291 6 : bSupportHTTP2 = strstr(curl_version(), "nghttp2/") != nullptr;
292 6 : bHasCheckVersion = true;
293 :
294 6 : curl_version_info_data *data = curl_version_info(CURLVERSION_NOW);
295 6 : if (data->version_num < LIBCURL_VERSION_NUM)
296 : {
297 0 : CPLError(CE_Warning, CPLE_AppDefined,
298 : "GDAL was built against curl %d.%d.%d, but is "
299 : "running against %s. Runtime failure is likely !",
300 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
301 : LIBCURL_VERSION_PATCH, data->version);
302 : }
303 6 : else if (data->version_num > LIBCURL_VERSION_NUM)
304 : {
305 0 : CPLDebug("HTTP",
306 : "GDAL was built against curl %d.%d.%d, but is "
307 : "running against %s.",
308 : LIBCURL_VERSION_MAJOR, LIBCURL_VERSION_MINOR,
309 : LIBCURL_VERSION_PATCH, data->version);
310 : }
311 :
312 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
313 : CPLOpenSSLInit();
314 : #endif
315 : }
316 2587 : }
317 :
318 : /************************************************************************/
319 : /* CPLWriteFct() */
320 : /* */
321 : /* Append incoming text to our collection buffer, reallocating */
322 : /* it larger as needed. */
323 : /************************************************************************/
324 :
325 : class CPLHTTPResultWithLimit
326 : {
327 : public:
328 : CPLHTTPResult *psResult = nullptr;
329 : int nMaxFileSize = 0;
330 : };
331 :
332 12718 : static size_t CPLWriteFct(void *buffer, size_t size, size_t nmemb,
333 : void *reqInfo)
334 :
335 : {
336 12718 : CPLHTTPResultWithLimit *psResultWithLimit =
337 : static_cast<CPLHTTPResultWithLimit *>(reqInfo);
338 12718 : CPLHTTPResult *psResult = psResultWithLimit->psResult;
339 :
340 12718 : int nBytesToWrite = static_cast<int>(nmemb) * static_cast<int>(size);
341 12718 : int nNewSize = psResult->nDataLen + nBytesToWrite + 1;
342 12718 : if (nNewSize > psResult->nDataAlloc)
343 : {
344 1637 : psResult->nDataAlloc = static_cast<int>(nNewSize * 1.25 + 100);
345 : GByte *pabyNewData = static_cast<GByte *>(
346 1637 : VSIRealloc(psResult->pabyData, psResult->nDataAlloc));
347 1637 : if (pabyNewData == nullptr)
348 : {
349 0 : VSIFree(psResult->pabyData);
350 0 : psResult->pabyData = nullptr;
351 0 : psResult->pszErrBuf = CPLStrdup(CPLString().Printf(
352 : "Out of memory allocating %d bytes for HTTP data buffer.",
353 0 : psResult->nDataAlloc));
354 0 : psResult->nDataAlloc = psResult->nDataLen = 0;
355 :
356 0 : return 0;
357 : }
358 1637 : psResult->pabyData = pabyNewData;
359 : }
360 :
361 12718 : memcpy(psResult->pabyData + psResult->nDataLen, buffer, nBytesToWrite);
362 :
363 12718 : psResult->nDataLen += nBytesToWrite;
364 12718 : psResult->pabyData[psResult->nDataLen] = 0;
365 :
366 12718 : if (psResultWithLimit->nMaxFileSize > 0 &&
367 8 : psResult->nDataLen > psResultWithLimit->nMaxFileSize)
368 : {
369 1 : CPLError(CE_Failure, CPLE_AppDefined, "Maximum file size reached");
370 1 : return 0;
371 : }
372 :
373 12717 : return nmemb;
374 : }
375 :
376 : /************************************************************************/
377 : /* CPLHdrWriteFct() */
378 : /************************************************************************/
379 7571 : static size_t CPLHdrWriteFct(void *buffer, size_t size, size_t nmemb,
380 : void *reqInfo)
381 : {
382 7571 : CPLHTTPResult *psResult = static_cast<CPLHTTPResult *>(reqInfo);
383 : // Copy the buffer to a char* and initialize with zeros (zero
384 : // terminate as well).
385 7571 : size_t nBytes = size * nmemb;
386 7571 : char *pszHdr = static_cast<char *>(CPLCalloc(1, nBytes + 1));
387 7571 : memcpy(pszHdr, buffer, nBytes);
388 7571 : size_t nIdx = nBytes - 1;
389 : // Remove end of line characters
390 21544 : while (nIdx > 0 && (pszHdr[nIdx] == '\r' || pszHdr[nIdx] == '\n'))
391 : {
392 13973 : pszHdr[nIdx] = 0;
393 13973 : nIdx--;
394 : }
395 7571 : char *pszKey = nullptr;
396 7571 : const char *pszValue = CPLParseNameValue(pszHdr, &pszKey);
397 7571 : if (pszKey && pszValue)
398 : {
399 5245 : psResult->papszHeaders =
400 5245 : CSLAddNameValue(psResult->papszHeaders, pszKey, pszValue);
401 : }
402 7571 : CPLFree(pszHdr);
403 7571 : CPLFree(pszKey);
404 7571 : return nmemb;
405 : }
406 :
407 : /************************************************************************/
408 : /* CPLHTTPReadFunction() */
409 : /************************************************************************/
410 0 : static size_t CPLHTTPReadFunction(char *buffer, size_t size, size_t nitems,
411 : void *arg)
412 : {
413 0 : return VSIFReadL(buffer, size, nitems, static_cast<VSILFILE *>(arg));
414 : }
415 :
416 : /************************************************************************/
417 : /* CPLHTTPSeekFunction() */
418 : /************************************************************************/
419 0 : static int CPLHTTPSeekFunction(void *arg, curl_off_t offset, int origin)
420 : {
421 0 : if (VSIFSeekL(static_cast<VSILFILE *>(arg), offset, origin) == 0)
422 0 : return CURL_SEEKFUNC_OK;
423 : else
424 0 : return CURL_SEEKFUNC_FAIL;
425 : }
426 :
427 : /************************************************************************/
428 : /* CPLHTTPFreeFunction() */
429 : /************************************************************************/
430 0 : static void CPLHTTPFreeFunction(void *arg)
431 : {
432 0 : VSIFCloseL(static_cast<VSILFILE *>(arg));
433 0 : }
434 :
435 : typedef struct
436 : {
437 : GDALProgressFunc pfnProgress;
438 : void *pProgressArg;
439 : } CurlProcessData, *CurlProcessDataL;
440 :
441 386 : static int NewProcessFunction(void *p, curl_off_t dltotal, curl_off_t dlnow,
442 : curl_off_t ultotal, curl_off_t ulnow)
443 : {
444 386 : CurlProcessDataL pData = static_cast<CurlProcessDataL>(p);
445 386 : if (nullptr != pData && pData->pfnProgress)
446 : {
447 386 : if (dltotal > 0)
448 : {
449 0 : const double dfDone = double(dlnow) / dltotal;
450 0 : return pData->pfnProgress(dfDone, "Downloading ...",
451 : pData->pProgressArg) == TRUE
452 0 : ? 0
453 0 : : 1;
454 : }
455 386 : else if (ultotal > 0)
456 : {
457 0 : const double dfDone = double(ulnow) / ultotal;
458 0 : return pData->pfnProgress(dfDone, "Uploading ...",
459 : pData->pProgressArg) == TRUE
460 0 : ? 0
461 0 : : 1;
462 : }
463 : }
464 386 : return 0;
465 : }
466 :
467 : #endif /* def HAVE_CURL */
468 :
469 : /************************************************************************/
470 : /* CPLHTTPSetDefaultUserAgent() */
471 : /************************************************************************/
472 :
473 : static std::string gosDefaultUserAgent;
474 :
475 : /**
476 : * \brief Set the default user agent.
477 : *
478 : * GDAL core will by default call this method with "GDAL/x.y.z" where x.y.z
479 : * is the GDAL version number (during driver initialization). Applications may
480 : * override it.
481 : *
482 : * @param pszUserAgent String (or nullptr to cancel the default user agent)
483 : *
484 : * @since GDAL 3.7
485 : */
486 1648 : void CPLHTTPSetDefaultUserAgent(const char *pszUserAgent)
487 : {
488 1648 : gosDefaultUserAgent = pszUserAgent ? pszUserAgent : "";
489 1648 : }
490 :
491 : /************************************************************************/
492 : /* CPLHTTPGetOptionsFromEnv() */
493 : /************************************************************************/
494 :
495 : typedef struct
496 : {
497 : const char *pszEnvVar;
498 : const char *pszOptionName;
499 : } TupleEnvVarOptionName;
500 :
501 : constexpr TupleEnvVarOptionName asAssocEnvVarOptionName[] = {
502 : {"GDAL_HTTP_VERSION", "HTTP_VERSION"},
503 : {"GDAL_HTTP_CONNECTTIMEOUT", "CONNECTTIMEOUT"},
504 : {"GDAL_HTTP_TIMEOUT", "TIMEOUT"},
505 : {"GDAL_HTTP_LOW_SPEED_TIME", "LOW_SPEED_TIME"},
506 : {"GDAL_HTTP_LOW_SPEED_LIMIT", "LOW_SPEED_LIMIT"},
507 : {"GDAL_HTTP_USERPWD", "USERPWD"},
508 : {"GDAL_HTTP_PROXY", "PROXY"},
509 : {"GDAL_HTTPS_PROXY", "HTTPS_PROXY"},
510 : {"GDAL_HTTP_PROXYUSERPWD", "PROXYUSERPWD"},
511 : {"GDAL_PROXY_AUTH", "PROXYAUTH"},
512 : {"GDAL_HTTP_NETRC", "NETRC"},
513 : {"GDAL_HTTP_NETRC_FILE", "NETRC_FILE"},
514 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
515 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
516 : {"GDAL_HTTP_RETRY_CODES", "RETRY_CODES"},
517 : {"GDAL_CURL_CA_BUNDLE", "CAINFO"},
518 : {"CURL_CA_BUNDLE", "CAINFO"},
519 : {"SSL_CERT_FILE", "CAINFO"},
520 : {"GDAL_HTTP_CAPATH", "CAPATH"},
521 : {"GDAL_HTTP_SSL_VERIFYSTATUS", "SSL_VERIFYSTATUS"},
522 : {"GDAL_HTTP_USE_CAPI_STORE", "USE_CAPI_STORE"},
523 : {"GDAL_HTTP_HEADERS", "HEADERS"},
524 : {"GDAL_HTTP_HEADER_FILE", "HEADER_FILE"},
525 : {"GDAL_HTTP_AUTH", "HTTPAUTH"},
526 : {"GDAL_GSSAPI_DELEGATION", "GSSAPI_DELEGATION"},
527 : {"GDAL_HTTP_BEARER", "HTTP_BEARER"},
528 : {"GDAL_HTTP_COOKIE", "COOKIE"},
529 : {"GDAL_HTTP_COOKIEFILE", "COOKIEFILE"},
530 : {"GDAL_HTTP_COOKIEJAR", "COOKIEJAR"},
531 : {"GDAL_HTTP_MAX_RETRY", "MAX_RETRY"},
532 : {"GDAL_HTTP_RETRY_DELAY", "RETRY_DELAY"},
533 : {"GDAL_HTTP_TCP_KEEPALIVE", "TCP_KEEPALIVE"},
534 : {"GDAL_HTTP_TCP_KEEPIDLE", "TCP_KEEPIDLE"},
535 : {"GDAL_HTTP_TCP_KEEPINTVL", "TCP_KEEPINTVL"},
536 : };
537 :
538 1488 : char **CPLHTTPGetOptionsFromEnv(const char *pszFilename)
539 : {
540 2976 : CPLStringList aosOptions;
541 2976 : std::string osNonStreamingFilename;
542 1488 : if (pszFilename && STARTS_WITH(pszFilename, "/vsi"))
543 : {
544 : VSIFilesystemHandler *poFSHandler =
545 1486 : VSIFileManager::GetHandler(pszFilename);
546 : osNonStreamingFilename =
547 1486 : poFSHandler->GetNonStreamingFilename(pszFilename);
548 1486 : if (osNonStreamingFilename == pszFilename)
549 : {
550 1421 : osNonStreamingFilename.clear();
551 : }
552 : else
553 : {
554 : // CPLDebug("HTTP", "Non-streaming filename for %s: %s", pszFilename, osNonStreamingFilename.c_str());
555 : }
556 : }
557 52080 : for (const auto &sTuple : asAssocEnvVarOptionName)
558 : {
559 50592 : const char *pszVal = nullptr;
560 50592 : if (pszFilename)
561 : {
562 50592 : pszVal = VSIGetPathSpecificOption(pszFilename, sTuple.pszEnvVar,
563 : nullptr);
564 50592 : if (!pszVal && !osNonStreamingFilename.empty())
565 : {
566 2198 : pszVal = VSIGetPathSpecificOption(
567 2198 : osNonStreamingFilename.c_str(), sTuple.pszEnvVar, nullptr);
568 : }
569 : }
570 50592 : if (!pszVal)
571 : {
572 50434 : pszVal = CPLGetConfigOption(sTuple.pszEnvVar, nullptr);
573 : }
574 50592 : if (pszVal)
575 : {
576 158 : aosOptions.AddNameValue(sTuple.pszOptionName, pszVal);
577 : }
578 : }
579 2976 : return aosOptions.StealList();
580 : }
581 :
582 : /************************************************************************/
583 : /* CPLHTTPGetNewRetryDelay() */
584 : /************************************************************************/
585 :
586 : /** Return the new retry delay.
587 : *
588 : * This takes into account the HTTP response code, the previous delay, the
589 : * HTTP payload error message, the Curl error message and a potential list of
590 : * retriable HTTP codes.
591 : *
592 : * @param response_code HTTP response code (e.g. 400)
593 : * @param dfOldDelay Previous delay (nominally in second)
594 : * @param pszErrBuf HTTP response body of the failed request (may be NULL)
595 : * @param pszCurlError Curl error as returned by CURLOPT_ERRORBUFFER (may be NULL)
596 : * @param pszRetriableCodes nullptr to limit to the default hard-coded scenarios,
597 : * "ALL" to ask to retry for all non-200 codes, or a comma-separated list of
598 : * HTTP codes (e.g. "400,500") that are accepted for retry.
599 : * @return the new delay, or 0 if no retry should be attempted.
600 : */
601 229 : static double CPLHTTPGetNewRetryDelay(int response_code, double dfOldDelay,
602 : const char *pszErrBuf,
603 : const char *pszCurlError,
604 : const char *pszRetriableCodes)
605 : {
606 229 : bool bRetry = false;
607 229 : if (pszRetriableCodes && pszRetriableCodes[0])
608 : {
609 5 : bRetry = EQUAL(pszRetriableCodes, "ALL") ||
610 2 : strstr(pszRetriableCodes, CPLSPrintf("%d", response_code));
611 : }
612 226 : else if (response_code == 429 || response_code == 500 ||
613 216 : (response_code >= 502 && response_code <= 504) ||
614 : // S3 sends some client timeout errors as 400 Client Error
615 6 : (response_code == 400 && pszErrBuf &&
616 204 : strstr(pszErrBuf, "RequestTimeout")) ||
617 204 : (pszCurlError &&
618 204 : (strstr(pszCurlError, "Connection timed out") ||
619 204 : strstr(pszCurlError, "Operation timed out") ||
620 204 : strstr(pszCurlError, "Connection reset by peer") ||
621 204 : strstr(pszCurlError, "Connection was reset") ||
622 200 : strstr(pszCurlError, "SSL connection timeout"))))
623 : {
624 26 : bRetry = true;
625 : }
626 229 : if (bRetry)
627 : {
628 : // 'Operation timed out': seen during some long running operation 'hang'
629 : // no error but no response from server and we are in the cURL loop
630 : // infinitely.
631 :
632 : // 'Connection was reset': was found with Azure: server resets
633 : // connection during TLS handshake (10054 error code). It seems like
634 : // the server process crashed or something forced TCP reset;
635 : // the request succeeds on retry.
636 :
637 : // Use an exponential backoff factor of 2 plus some random jitter
638 : // We don't care about cryptographic quality randomness, hence:
639 : #ifndef __COVERITY__
640 28 : return dfOldDelay * (2 + rand() * 0.5 / RAND_MAX);
641 : #else
642 : return dfOldDelay * 2;
643 : #endif
644 : }
645 : else
646 : {
647 201 : return 0;
648 : }
649 : }
650 :
651 : /*! @cond Doxygen_Suppress */
652 :
653 : /************************************************************************/
654 : /* CPLHTTPRetryParameters() */
655 : /************************************************************************/
656 :
657 : /** Constructs a CPLHTTPRetryParameters instance from configuration
658 : * options or path-specific options.
659 : *
660 : * @param aosHTTPOptions HTTP options returned by CPLHTTPGetOptionsFromEnv()
661 : */
662 1294 : CPLHTTPRetryParameters::CPLHTTPRetryParameters(
663 1294 : const CPLStringList &aosHTTPOptions)
664 1294 : : nMaxRetry(atoi(aosHTTPOptions.FetchNameValueDef(
665 : "MAX_RETRY", CPLSPrintf("%d", CPL_HTTP_MAX_RETRY)))),
666 1294 : dfInitialDelay(CPLAtof(aosHTTPOptions.FetchNameValueDef(
667 : "RETRY_DELAY", CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY)))),
668 2588 : osRetryCodes(aosHTTPOptions.FetchNameValueDef("RETRY_CODES", ""))
669 : {
670 1294 : }
671 :
672 : /************************************************************************/
673 : /* CPLHTTPRetryContext() */
674 : /************************************************************************/
675 :
676 : /** Constructor */
677 1086 : CPLHTTPRetryContext::CPLHTTPRetryContext(const CPLHTTPRetryParameters &oParams)
678 1086 : : m_oParameters(oParams), m_dfNextDelay(oParams.dfInitialDelay)
679 : {
680 1086 : }
681 :
682 : /************************************************************************/
683 : /* CPLHTTPRetryContext::CanRetry() */
684 : /************************************************************************/
685 :
686 : /** Returns whether we can attempt a new retry, based on the retry counter,
687 : * and increment that counter.
688 : */
689 0 : bool CPLHTTPRetryContext::CanRetry()
690 : {
691 0 : if (m_nRetryCount >= m_oParameters.nMaxRetry)
692 0 : return false;
693 0 : m_nRetryCount++;
694 0 : return true;
695 : }
696 :
697 : /** Returns whether we can attempt a new retry, based on the retry counter,
698 : * the response code, payload and curl error buffers.
699 : *
700 : * If successful, the retry counter is incremented, and GetCurrentDelay()
701 : * returns the delay to apply with CPLSleep().
702 : */
703 245 : bool CPLHTTPRetryContext::CanRetry(int response_code, const char *pszErrBuf,
704 : const char *pszCurlError)
705 : {
706 245 : if (m_nRetryCount >= m_oParameters.nMaxRetry)
707 224 : return false;
708 21 : m_dfCurDelay = m_dfNextDelay;
709 21 : m_dfNextDelay = CPLHTTPGetNewRetryDelay(response_code, m_dfNextDelay,
710 : pszErrBuf, pszCurlError,
711 : m_oParameters.osRetryCodes.c_str());
712 21 : if (m_dfNextDelay == 0.0)
713 2 : return false;
714 19 : m_nRetryCount++;
715 19 : return true;
716 : }
717 :
718 : /************************************************************************/
719 : /* CPLHTTPRetryContext::GetCurrentDelay() */
720 : /************************************************************************/
721 :
722 : /** Returns the delay to apply. Only valid after a successful call to CanRetry() */
723 38 : double CPLHTTPRetryContext::GetCurrentDelay() const
724 : {
725 38 : if (m_nRetryCount == 0)
726 0 : CPLDebug("CPL",
727 : "GetCurrentDelay() should only be called after CanRetry()");
728 38 : return m_dfCurDelay;
729 : }
730 :
731 : /*! @endcond Doxygen_Suppress */
732 :
733 : #ifdef HAVE_CURL
734 :
735 : /************************************************************************/
736 : /* CPLHTTPEmitFetchDebug() */
737 : /************************************************************************/
738 :
739 1173 : static void CPLHTTPEmitFetchDebug(const char *pszURL,
740 : const char *pszExtraDebug = "")
741 : {
742 1173 : const char *pszArobase = strchr(pszURL, '@');
743 1173 : const char *pszSlash = strchr(pszURL, '/');
744 1173 : const char *pszColon = (pszSlash) ? strchr(pszSlash, ':') : nullptr;
745 1173 : if (pszArobase != nullptr && pszColon != nullptr &&
746 0 : pszArobase - pszColon > 0)
747 : {
748 : /* http://user:password@www.example.com */
749 0 : char *pszSanitizedURL = CPLStrdup(pszURL);
750 0 : pszSanitizedURL[pszColon - pszURL] = 0;
751 0 : CPLDebug("HTTP", "Fetch(%s:#password#%s%s)", pszSanitizedURL,
752 : pszArobase, pszExtraDebug);
753 0 : CPLFree(pszSanitizedURL);
754 : }
755 : else
756 : {
757 1173 : CPLDebug("HTTP", "Fetch(%s%s)", pszURL, pszExtraDebug);
758 : }
759 1173 : }
760 :
761 : #endif
762 :
763 : #ifdef HAVE_CURL
764 :
765 : /************************************************************************/
766 : /* class CPLHTTPPostFields */
767 : /************************************************************************/
768 :
769 : class CPLHTTPPostFields
770 : {
771 : public:
772 1173 : CPLHTTPPostFields() = default;
773 : CPLHTTPPostFields &operator=(const CPLHTTPPostFields &) = delete;
774 : CPLHTTPPostFields(const CPLHTTPPostFields &) = delete;
775 :
776 1173 : CPLErr Fill(CURL *http_handle, CSLConstList papszOptions)
777 : {
778 : // Fill POST form if present
779 : const char *pszFormFilePath =
780 1173 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
781 : const char *pszParametersCount =
782 1173 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
783 :
784 1173 : if (pszFormFilePath != nullptr || pszParametersCount != nullptr)
785 : {
786 2 : mime = curl_mime_init(http_handle);
787 2 : curl_mimepart *mimepart = curl_mime_addpart(mime);
788 2 : if (pszFormFilePath != nullptr)
789 : {
790 : const char *pszFormFileName =
791 1 : CSLFetchNameValue(papszOptions, "FORM_FILE_NAME");
792 1 : const char *pszFilename = CPLGetFilename(pszFormFilePath);
793 1 : if (pszFormFileName == nullptr)
794 : {
795 1 : pszFormFileName = pszFilename;
796 : }
797 :
798 : VSIStatBufL sStat;
799 1 : if (VSIStatL(pszFormFilePath, &sStat) == 0)
800 : {
801 0 : VSILFILE *mime_fp = VSIFOpenL(pszFormFilePath, "rb");
802 0 : if (mime_fp != nullptr)
803 : {
804 0 : curl_mime_name(mimepart, pszFormFileName);
805 0 : CPL_IGNORE_RET_VAL(
806 0 : curl_mime_filename(mimepart, pszFilename));
807 0 : curl_mime_data_cb(
808 : mimepart, sStat.st_size, CPLHTTPReadFunction,
809 : CPLHTTPSeekFunction, CPLHTTPFreeFunction, mime_fp);
810 : }
811 : else
812 : {
813 : osErrMsg = CPLSPrintf("Failed to open file %s",
814 0 : pszFormFilePath);
815 1 : return CE_Failure;
816 : }
817 :
818 0 : CPLDebug("HTTP", "Send file: %s, COPYNAME: %s",
819 : pszFormFilePath, pszFormFileName);
820 : }
821 : else
822 : {
823 : osErrMsg =
824 1 : CPLSPrintf("File '%s' not found", pszFormFilePath);
825 1 : return CE_Failure;
826 : }
827 : }
828 :
829 1 : int nParametersCount = 0;
830 1 : if (pszParametersCount != nullptr)
831 : {
832 1 : nParametersCount = atoi(pszParametersCount);
833 : }
834 :
835 2 : for (int i = 0; i < nParametersCount; ++i)
836 : {
837 2 : const char *pszKey = CSLFetchNameValue(
838 : papszOptions, CPLSPrintf("FORM_KEY_%d", i));
839 2 : const char *pszValue = CSLFetchNameValue(
840 : papszOptions, CPLSPrintf("FORM_VALUE_%d", i));
841 :
842 2 : if (nullptr == pszKey)
843 : {
844 : osErrMsg = CPLSPrintf("Key #%d is not exists. Maybe wrong "
845 : "count of form items",
846 1 : i);
847 1 : return CE_Failure;
848 : }
849 :
850 1 : if (nullptr == pszValue)
851 : {
852 : osErrMsg = CPLSPrintf("Value #%d is not exists. Maybe "
853 : "wrong count of form items",
854 0 : i);
855 0 : return CE_Failure;
856 : }
857 :
858 1 : mimepart = curl_mime_addpart(mime);
859 1 : curl_mime_name(mimepart, pszKey);
860 1 : CPL_IGNORE_RET_VAL(
861 1 : curl_mime_data(mimepart, pszValue, CURL_ZERO_TERMINATED));
862 :
863 1 : CPLDebug("HTTP", "COPYNAME: %s, COPYCONTENTS: %s", pszKey,
864 : pszValue);
865 : }
866 :
867 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MIMEPOST, mime);
868 : }
869 1171 : return CE_None;
870 : }
871 :
872 1173 : ~CPLHTTPPostFields()
873 1173 : {
874 1173 : if (mime != nullptr)
875 : {
876 2 : curl_mime_free(mime);
877 : }
878 1173 : }
879 :
880 2 : std::string GetErrorMessage() const
881 : {
882 2 : return osErrMsg;
883 : }
884 :
885 : private:
886 : curl_mime *mime = nullptr;
887 : std::string osErrMsg{};
888 : };
889 :
890 : /************************************************************************/
891 : /* CPLHTTPFetchCleanup() */
892 : /************************************************************************/
893 :
894 1173 : static void CPLHTTPFetchCleanup(CURL *http_handle, struct curl_slist *headers,
895 : const char *pszPersistent,
896 : CSLConstList papszOptions)
897 : {
898 1173 : if (CSLFetchNameValue(papszOptions, "POSTFIELDS"))
899 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 0);
900 1173 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, nullptr);
901 :
902 1173 : if (!pszPersistent)
903 671 : curl_easy_cleanup(http_handle);
904 :
905 1173 : curl_slist_free_all(headers);
906 1173 : }
907 : #endif // HAVE_CURL
908 :
909 : struct CPLHTTPFetchContext
910 : {
911 : std::vector<std::pair<CPLHTTPFetchCallbackFunc, void *>> stack{};
912 : };
913 :
914 : /************************************************************************/
915 : /* GetHTTPFetchContext() */
916 : /************************************************************************/
917 :
918 1366 : static CPLHTTPFetchContext *GetHTTPFetchContext(bool bAlloc)
919 : {
920 1366 : int bError = FALSE;
921 : CPLHTTPFetchContext *psCtx = static_cast<CPLHTTPFetchContext *>(
922 1366 : CPLGetTLSEx(CTLS_HTTPFETCHCALLBACK, &bError));
923 1366 : if (bError)
924 0 : return nullptr;
925 :
926 1366 : if (psCtx == nullptr && bAlloc)
927 : {
928 1 : const auto FreeFunc = [](void *pData)
929 1 : { delete static_cast<CPLHTTPFetchContext *>(pData); };
930 1 : psCtx = new CPLHTTPFetchContext();
931 1 : CPLSetTLSWithFreeFuncEx(CTLS_HTTPFETCHCALLBACK, psCtx, FreeFunc,
932 : &bError);
933 1 : if (bError)
934 : {
935 0 : delete psCtx;
936 0 : psCtx = nullptr;
937 : }
938 : }
939 1366 : return psCtx;
940 : }
941 :
942 : /************************************************************************/
943 : /* CPLHTTPSetFetchCallback() */
944 : /************************************************************************/
945 :
946 : static CPLHTTPFetchCallbackFunc gpsHTTPFetchCallbackFunc = nullptr;
947 : static void *gpHTTPFetchCallbackUserData = nullptr;
948 :
949 : /** Installs an alternate callback to the default implementation of
950 : * CPLHTTPFetchEx().
951 : *
952 : * This callback will be used by all threads, unless contextual callbacks are
953 : * installed with CPLHTTPPushFetchCallback().
954 : *
955 : * It is the responsibility of the caller to make sure this function is not
956 : * called concurrently, or during CPLHTTPFetchEx() execution.
957 : *
958 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called
959 : * (or NULL to restore default handler)
960 : * @param pUserData Last argument to provide to the pFunc callback.
961 : *
962 : * @since GDAL 3.2
963 : */
964 2 : void CPLHTTPSetFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
965 : {
966 2 : gpsHTTPFetchCallbackFunc = pFunc;
967 2 : gpHTTPFetchCallbackUserData = pUserData;
968 2 : }
969 :
970 : /************************************************************************/
971 : /* CPLHTTPPushFetchCallback() */
972 : /************************************************************************/
973 :
974 : /** Installs an alternate callback to the default implementation of
975 : * CPLHTTPFetchEx().
976 : *
977 : * This callback will only be used in the thread where this function has been
978 : * called. It must be un-installed by CPLHTTPPopFetchCallback(), which must also
979 : * be called from the same thread.
980 : *
981 : * @param pFunc Callback function to be called with CPLHTTPFetchEx() is called.
982 : * @param pUserData Last argument to provide to the pFunc callback.
983 : * @return TRUE in case of success.
984 : *
985 : * @since GDAL 3.2
986 : */
987 1 : int CPLHTTPPushFetchCallback(CPLHTTPFetchCallbackFunc pFunc, void *pUserData)
988 : {
989 1 : auto psCtx = GetHTTPFetchContext(true);
990 1 : if (psCtx == nullptr)
991 0 : return false;
992 : psCtx->stack.emplace_back(
993 1 : std::pair<CPLHTTPFetchCallbackFunc, void *>(pFunc, pUserData));
994 1 : return true;
995 : }
996 :
997 : /************************************************************************/
998 : /* CPLHTTPPopFetchCallback() */
999 : /************************************************************************/
1000 :
1001 : /** Uninstalls a callback set by CPLHTTPPushFetchCallback().
1002 : *
1003 : * @see CPLHTTPPushFetchCallback()
1004 : * @return TRUE in case of success.
1005 : * @since GDAL 3.2
1006 : */
1007 2 : int CPLHTTPPopFetchCallback(void)
1008 : {
1009 2 : auto psCtx = GetHTTPFetchContext(false);
1010 2 : if (psCtx == nullptr || psCtx->stack.empty())
1011 : {
1012 1 : CPLError(
1013 : CE_Failure, CPLE_AppDefined,
1014 : "CPLHTTPPushFetchCallback / CPLHTTPPopFetchCallback not balanced");
1015 1 : return false;
1016 : }
1017 : else
1018 : {
1019 1 : psCtx->stack.pop_back();
1020 1 : return true;
1021 : }
1022 : }
1023 :
1024 : /************************************************************************/
1025 : /* CPLHTTPFetch() */
1026 : /************************************************************************/
1027 :
1028 : // NOTE: when adding an option below, add it in asAssocEnvVarOptionName[]
1029 :
1030 : // clang-format off
1031 : /**
1032 : * \brief Fetch a document from an url and return in a string.
1033 : *
1034 : * Different options may be passed through the papszOptions function parameter,
1035 : * or for most of them through a configuration option:
1036 : * <ul>
1037 : * <li>CONNECTTIMEOUT=val, where
1038 : * val is in seconds (possibly with decimals). This is the maximum delay for the
1039 : * connection to be established before being aborted (GDAL >= 2.2).
1040 : * Corresponding configuration option: GDAL_HTTP_CONNECTTIMEOUT.
1041 : * </li>
1042 : * <li>TIMEOUT=val, where val is in seconds. This is the maximum delay for the
1043 : * whole request to complete before being aborted.
1044 : * Corresponding configuration option: GDAL_HTTP_TIMEOUT.
1045 : * </li>
1046 : * <li>LOW_SPEED_TIME=val,
1047 : * where val is in seconds. This is the maximum time where the transfer speed
1048 : * should be below the LOW_SPEED_LIMIT (if not specified 1b/s), before the
1049 : * transfer to be considered too slow and aborted. (GDAL >= 2.1).
1050 : * Corresponding configuration option: GDAL_HTTP_LOW_SPEED_TIME.
1051 : * </li>
1052 : * <li>LOW_SPEED_LIMIT=val, where val is in bytes/second. See LOW_SPEED_TIME.
1053 : * Has only effect if LOW_SPEED_TIME is specified too. (GDAL >= 2.1).
1054 : * Corresponding configuration option: GDAL_HTTP_LOW_SPEED_LIMIT.
1055 : * </li>
1056 : * <li>HEADERS=val, where val is an extra header to use when getting a web page.
1057 : * For example "Accept: application/x-ogcwkt"
1058 : * Corresponding configuration option: GDAL_HTTP_HEADERS.
1059 : * Starting with GDAL 3.6, the GDAL_HTTP_HEADERS configuration option can also
1060 : * be used to specify a comma separated list of key: value pairs. This is an
1061 : * alternative to the GDAL_HTTP_HEADER_FILE mechanism. If a comma or a
1062 : * double-quote character is needed in the value, then the key: value pair must
1063 : * be enclosed in double-quote characters. In that situation, backslash and
1064 : * double quote character must be backslash-escaped. e.g GDAL_HTTP_HEADERS=Foo:
1065 : * Bar,"Baz: escaped backslash \\, escaped double-quote \", end of
1066 : * value",Another: Header
1067 : * </li>
1068 : * <li>HEADER_FILE=filename: filename of a text file with "key: value" headers.
1069 : * The content of the file is not cached, and thus it is read again before
1070 : * issuing each HTTP request. (GDAL >= 2.2)
1071 : * Corresponding configuration option: GDAL_HTTP_HEADER_FILE.
1072 : * </li>
1073 : * <li>HTTPAUTH=[BASIC/NTLM/NEGOTIATE/ANY/ANYSAFE/BEARER] to specify an
1074 : * authentication scheme to use.
1075 : * Corresponding configuration option: GDAL_HTTP_AUTH.
1076 : * </li>
1077 : * <li>USERPWD=userid:password to specify a user and password for
1078 : * authentication.
1079 : * Corresponding configuration option: GDAL_HTTP_USERPWD.
1080 : * </li>
1081 : * <li>GSSAPI_DELEGATION=[NONE/POLICY/ALWAYS] set allowed
1082 : * GSS-API delegation. Relevant only with HTTPAUTH=NEGOTIATE (GDAL >= 3.3).
1083 : * Corresponding configuration option: GDAL_GSSAPI_DELEGATION (note: no "HTTP_" in the name)
1084 : * </li>
1085 : * <li>HTTP_BEARER=val set OAuth 2.0 Bearer Access Token.
1086 : * Relevant only with HTTPAUTH=BEARER (GDAL >= 3.9).
1087 : * Corresponding configuration option: GDAL_HTTP_BEARER
1088 : * </li>
1089 : * <li>POSTFIELDS=val, where val is a nul-terminated string to be passed to the
1090 : * server with a POST request.
1091 : * No Corresponding configuration option.
1092 : * </li>
1093 : * <li>PROXY=val, to make requests go through a
1094 : * proxy server, where val is of the form proxy.server.com:port_number. This
1095 : * option affects both HTTP and HTTPS URLs.
1096 : * Corresponding configuration option: GDAL_HTTP_PROXY.
1097 : * </li>
1098 : * <li>HTTPS_PROXY=val (GDAL
1099 : * >= 2.4), the same meaning as PROXY, but this option is taken into account
1100 : * only for HTTPS URLs.
1101 : * Corresponding configuration option: GDAL_HTTPS_PROXY.
1102 : * </li>
1103 : * <li>PROXYUSERPWD=val, where val is of the form username:password.
1104 : * Corresponding configuration option: GDAL_HTTP_PROXYUSERPWD
1105 : * </li>
1106 : * <li>PROXYAUTH=[BASIC/NTLM/DIGEST/NEGOTIATE/ANY/ANYSAFE] to
1107 : * specify an proxy authentication scheme to use..
1108 : * Corresponding configuration option: GDAL_PROXYAUTH (note: no "HTTP_" in the name)
1109 : * </li>
1110 : * <li>NETRC=[YES/NO] to
1111 : * enable or disable use of $HOME/.netrc (or NETRC_FILE), default YES.
1112 : * Corresponding configuration option: GDAL_HTTP_NETRC.
1113 : * </li>
1114 : * <li>NETRC_FILE=file name to read .netrc info from (GDAL >= 3.7).
1115 : * Corresponding configuration option: GDAL_HTTP_NETRC_FILE.
1116 : * </li>
1117 : * <li>CUSTOMREQUEST=val, where val is GET, PUT, POST, DELETE, etc...
1118 : * No corresponding configuration option.
1119 : * </li>
1120 : * <li>FORM_FILE_NAME=val, where val is upload file name. If this
1121 : * option and FORM_FILE_PATH present, request type will set to POST.
1122 : * No corresponding configuration option.
1123 : * </li>
1124 : * <li>FORM_FILE_PATH=val, where val is upload file path.
1125 : * No corresponding configuration option.
1126 : * </li>
1127 : * <li>FORM_KEY_0=val...FORM_KEY_N, where val is name of form item.
1128 : * No corresponding configuration option.
1129 : * </li>
1130 : * <li>FORM_VALUE_0=val...FORM_VALUE_N, where val is value of the form
1131 : * item.
1132 : * No corresponding configuration option.
1133 : * </li>
1134 : * <li>FORM_ITEM_COUNT=val, where val is count of form items.
1135 : * No corresponding configuration option.
1136 : * </li>
1137 : * <li>COOKIE=val, where val is formatted as COOKIE1=VALUE1; COOKIE2=VALUE2;...
1138 : * Corresponding configuration option: GDAL_HTTP_COOKIE.</li>
1139 : * <li>COOKIEFILE=val, where val is file name to read cookies from
1140 : * (GDAL >= 2.4).
1141 : * Corresponding configuration option: GDAL_HTTP_COOKIEFILE.</li>
1142 : * <li>COOKIEJAR=val, where val is file name to store cookies to (GDAL >= 2.4).
1143 : * Corresponding configuration option: GDAL_HTTP_COOKIEJAR.</li>
1144 : * <li>MAX_RETRY=val, where val is the maximum number of
1145 : * retry attempts, when a retry is allowed (cf RETRY_CODES option).
1146 : * Default is 0, meaning no retry.
1147 : * Corresponding configuration option: GDAL_HTTP_MAX_RETRY.
1148 : * </li>
1149 : * <li>RETRY_DELAY=val, where val is the number of seconds
1150 : * between retry attempts. Default is 30.
1151 : * Corresponding configuration option: GDAL_HTTP_RETRY_DELAY.
1152 : * <li>RETRY_CODES=val, where val is "ALL" or a comma-separated list of HTTP
1153 : * codes that are considered for retry. By default, 429, 500, 502, 503 or 504
1154 : * HTTP errors are considered, as well as other situations with a particular
1155 : * HTTP or Curl error message. (GDAL >= 3.10).
1156 : * Corresponding configuration option: GDAL_HTTP_RETRY_CODES.
1157 : * </li>
1158 : * <li>MAX_FILE_SIZE=val, where val is a number of bytes (GDAL >= 2.2)
1159 : * No corresponding configuration option.
1160 : * </li>
1161 : * <li>CAINFO=/path/to/bundle.crt. This is path to Certificate Authority (CA)
1162 : * bundle file. By default, it will be looked for in a system location. If
1163 : * the CAINFO option is not defined, GDAL will also look in the
1164 : * CURL_CA_BUNDLE and SSL_CERT_FILE environment variables respectively
1165 : * and use the first one found as the CAINFO value (GDAL >= 2.1.3). The
1166 : * GDAL_CURL_CA_BUNDLE environment variable may also be used to set the
1167 : * CAINFO value in GDAL >= 3.2.</li>
1168 : * <li>HTTP_VERSION=1.0/1.1/2/2TLS (GDAL >= 2.3)/2PRIOR_KNOWLEDGE (GDAL >= 3.10).
1169 : * Specify HTTP version to use.
1170 : * Will default to 1.1 generally (except on some controlled environments,
1171 : * like Google Compute Engine VMs, where 2TLS will be the default).
1172 : * Support for HTTP/2 requires curl 7.33 or later, built against nghttp2.
1173 : * "2TLS" means that HTTP/2 will be attempted for HTTPS connections only.
1174 : * Whereas "2" means that HTTP/2 will be attempted for HTTP or HTTPS.
1175 : * "2PRIOR_KNOWLEDGE" means that the server will be assumed to support
1176 : * HTTP/2.
1177 : * Corresponding configuration option: GDAL_HTTP_VERSION.
1178 : * </li>
1179 : * <li>SSL_VERIFYSTATUS=YES/NO (GDAL >= 2.3, and curl >= 7.41): determines
1180 : * whether the status of the server cert using the "Certificate Status Request"
1181 : * TLS extension (aka. OCSP stapling) should be checked. If this option is
1182 : * enabled but the server does not support the TLS extension, the verification
1183 : * will fail. Default to NO.
1184 : * Corresponding configuration option: GDAL_HTTP_SSL_VERIFYSTATUS.
1185 : * </li>
1186 : * <li>USE_CAPI_STORE=YES/NO (GDAL >= 2.3,
1187 : * Windows only): whether CA certificates from the Windows certificate store.
1188 : * Defaults to NO.
1189 : * Corresponding configuration option: GDAL_HTTP_USE_CAPI_STORE.
1190 : * </li>
1191 : * <li>TCP_KEEPALIVE=YES/NO (GDAL >= 3.6): whether to
1192 : * enable TCP keep-alive. Defaults to NO.
1193 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPALIVE.
1194 : * </li>
1195 : * <li>TCP_KEEPIDLE=integer, in
1196 : * seconds (GDAL >= 3.6): keep-alive idle time. Defaults to 60. Only taken into
1197 : * account if TCP_KEEPALIVE=YES.
1198 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPIDLE.
1199 : * </li>
1200 : * <li>TCP_KEEPINTVL=integer, in seconds
1201 : * (GDAL >= 3.6): interval time between keep-alive probes. Defaults to 60. Only
1202 : * taken into account if TCP_KEEPALIVE=YES.
1203 : * Corresponding configuration option: GDAL_HTTP_TCP_KEEPINTVL.
1204 : * </li>
1205 : * <li>USERAGENT=string: value of User-Agent header. Starting with GDAL 3.7,
1206 : * GDAL core sets it by default (during driver initialization) to GDAL/x.y.z
1207 : * where x.y.z is the GDAL version number. Applications may override it with the
1208 : * CPLHTTPSetDefaultUserAgent() function.
1209 : * Corresponding configuration option: GDAL_HTTP_USERAGENT.
1210 : * </li>
1211 : * <li>SSLCERT=filename (GDAL >= 3.7): Filename of the the SSL client certificate.
1212 : * Cf https://curl.se/libcurl/c/CURLOPT_SSLCERT.html.
1213 : * Corresponding configuration option: GDAL_HTTP_SSLCERT.
1214 : * </li>
1215 : * <li>SSLCERTTYPE=string (GDAL >= 3.7): Format of the SSL certificate: "PEM"
1216 : * or "DER". Cf https://curl.se/libcurl/c/CURLOPT_SSLCERTTYPE.html.
1217 : * Corresponding configuration option: GDAL_HTTP_SSLCERTTYPE.
1218 : * </li>
1219 : * <li>SSLKEY=filename (GDAL >= 3.7): Private key file for TLS and SSL client
1220 : * certificate. Cf https://curl.se/libcurl/c/CURLOPT_SSLKEY.html.
1221 : * Corresponding configuration option: GDAL_HTTP_SSLKEY.
1222 : * </li>
1223 : * <li>KEYPASSWD=string (GDAL >= 3.7): Passphrase to private key.
1224 : * Cf https://curl.se/libcurl/c/CURLOPT_KEYPASSWD.html.
1225 : * Corresponding configuration option: GDAL_HTTP_KEYPASSWD.
1226 : * </ul>
1227 : *
1228 : * If an option is specified through papszOptions and as a configuration option,
1229 : * the former takes precedence over the later.
1230 : *
1231 : * Starting with GDAL 3.7, the above configuration options can also be specified
1232 : * as path-specific options with VSISetPathSpecificOption().
1233 : *
1234 : * @param pszURL valid URL recognized by underlying download library (libcurl)
1235 : * @param papszOptions option list as a NULL-terminated array of strings. May be
1236 : * NULL.
1237 : *
1238 : * @return a CPLHTTPResult* structure that must be freed by
1239 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled
1240 : */
1241 : // clang-format on
1242 2127 : CPLHTTPResult *CPLHTTPFetch(const char *pszURL, CSLConstList papszOptions)
1243 : {
1244 2127 : return CPLHTTPFetchEx(pszURL, papszOptions, nullptr, nullptr, nullptr,
1245 2127 : nullptr);
1246 : }
1247 :
1248 : /**
1249 : * Fetch a document from an url and return in a string.
1250 : * @param pszURL Url to fetch document from web.
1251 : * @param papszOptions Option list as a NULL-terminated array of strings.
1252 : * Available keys see in CPLHTTPFetch.
1253 : * @param pfnProgress Callback for reporting algorithm progress matching the
1254 : * GDALProgressFunc() semantics. May be NULL.
1255 : * @param pProgressArg Callback argument passed to pfnProgress.
1256 : * @param pfnWrite Write function pointer matching the CPLHTTPWriteFunc()
1257 : * semantics. May be NULL.
1258 : * @param pWriteArg Argument which will pass to a write function.
1259 : * @return A CPLHTTPResult* structure that must be freed by
1260 : * CPLHTTPDestroyResult(), or NULL if libcurl support is disabled.
1261 : */
1262 2157 : CPLHTTPResult *CPLHTTPFetchEx(const char *pszURL, CSLConstList papszOptions,
1263 : GDALProgressFunc pfnProgress, void *pProgressArg,
1264 : CPLHTTPFetchWriteFunc pfnWrite, void *pWriteArg)
1265 :
1266 : {
1267 2976 : if (STARTS_WITH(pszURL, "/vsimem/") &&
1268 : // Disabled by default for potential security issues.
1269 819 : CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")))
1270 : {
1271 794 : CPLString osURL(pszURL);
1272 : const char *pszCustomRequest =
1273 794 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
1274 794 : if (pszCustomRequest != nullptr)
1275 : {
1276 5 : osURL += "&CUSTOMREQUEST=";
1277 5 : osURL += pszCustomRequest;
1278 : }
1279 794 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
1280 794 : if (pszUserPwd != nullptr)
1281 : {
1282 1 : osURL += "&USERPWD=";
1283 1 : osURL += pszUserPwd;
1284 : }
1285 794 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
1286 794 : if (pszPost != nullptr) // Hack: We append post content to filename.
1287 : {
1288 281 : osURL += "&POSTFIELDS=";
1289 281 : osURL += pszPost;
1290 : }
1291 794 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1292 926 : if (pszHeaders != nullptr &&
1293 132 : CPLTestBool(
1294 : CPLGetConfigOption("CPL_CURL_VSIMEM_PRINT_HEADERS", "FALSE")))
1295 : {
1296 5 : osURL += "&HEADERS=";
1297 5 : osURL += pszHeaders;
1298 : }
1299 794 : vsi_l_offset nLength = 0;
1300 : CPLHTTPResult *psResult =
1301 794 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1302 794 : GByte *pabyData = VSIGetMemFileBuffer(osURL, &nLength, FALSE);
1303 794 : if (pabyData == nullptr)
1304 : {
1305 132 : CPLDebug("HTTP", "Cannot find %s", osURL.c_str());
1306 132 : psResult->nStatus = 1;
1307 132 : psResult->pszErrBuf =
1308 132 : CPLStrdup(CPLSPrintf("HTTP error code : %d", 404));
1309 132 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1310 : }
1311 662 : else if (nLength != 0)
1312 : {
1313 619 : psResult->nDataLen = static_cast<int>(nLength);
1314 619 : psResult->pabyData = static_cast<GByte *>(
1315 619 : CPLMalloc(static_cast<size_t>(nLength) + 1));
1316 619 : memcpy(psResult->pabyData, pabyData, static_cast<size_t>(nLength));
1317 619 : psResult->pabyData[static_cast<size_t>(nLength)] = 0;
1318 : }
1319 :
1320 794 : if (psResult->pabyData != nullptr &&
1321 619 : STARTS_WITH(reinterpret_cast<char *>(psResult->pabyData),
1322 : "Content-Type: "))
1323 : {
1324 7 : const char *pszContentType =
1325 7 : reinterpret_cast<char *>(psResult->pabyData) +
1326 : strlen("Content-type: ");
1327 7 : const char *pszEOL = strchr(pszContentType, '\r');
1328 7 : if (pszEOL)
1329 7 : pszEOL = strchr(pszContentType, '\n');
1330 7 : if (pszEOL)
1331 : {
1332 7 : size_t nContentLength = pszEOL - pszContentType;
1333 7 : psResult->pszContentType =
1334 7 : static_cast<char *>(CPLMalloc(nContentLength + 1));
1335 7 : memcpy(psResult->pszContentType, pszContentType,
1336 : nContentLength);
1337 7 : psResult->pszContentType[nContentLength] = 0;
1338 : }
1339 : }
1340 :
1341 794 : return psResult;
1342 : }
1343 :
1344 : // Try to user alternate network layer if set.
1345 1363 : auto pCtx = GetHTTPFetchContext(false);
1346 1363 : if (pCtx)
1347 : {
1348 3 : for (size_t i = pCtx->stack.size(); i > 0;)
1349 : {
1350 1 : --i;
1351 1 : const auto &cbk = pCtx->stack[i];
1352 1 : auto cbkFunc = cbk.first;
1353 1 : auto pUserData = cbk.second;
1354 1 : auto res = cbkFunc(pszURL, papszOptions, pfnProgress, pProgressArg,
1355 : pfnWrite, pWriteArg, pUserData);
1356 1 : if (res)
1357 : {
1358 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1359 : {
1360 0 : CPLHTTPDestroyResult(res);
1361 0 : res = nullptr;
1362 : }
1363 1 : return res;
1364 : }
1365 : }
1366 : }
1367 :
1368 1362 : if (gpsHTTPFetchCallbackFunc)
1369 : {
1370 1 : auto res = gpsHTTPFetchCallbackFunc(pszURL, papszOptions, pfnProgress,
1371 : pProgressArg, pfnWrite, pWriteArg,
1372 : gpHTTPFetchCallbackUserData);
1373 1 : if (res)
1374 : {
1375 1 : if (CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT"))
1376 : {
1377 0 : CPLHTTPDestroyResult(res);
1378 0 : res = nullptr;
1379 : }
1380 1 : return res;
1381 : }
1382 : }
1383 :
1384 : #ifndef HAVE_CURL
1385 : CPLError(CE_Failure, CPLE_NotSupported,
1386 : "GDAL/OGR not compiled with libcurl support, "
1387 : "remote requests not supported.");
1388 : return nullptr;
1389 : #else
1390 :
1391 : /* -------------------------------------------------------------------- */
1392 : /* Are we using a persistent named session? If so, search for */
1393 : /* or create it. */
1394 : /* */
1395 : /* Currently this code does not attempt to protect against */
1396 : /* multiple threads asking for the same named session. If that */
1397 : /* occurs it will be in use in multiple threads at once, which */
1398 : /* will lead to potential crashes in libcurl. */
1399 : /* -------------------------------------------------------------------- */
1400 1361 : CURL *http_handle = nullptr;
1401 :
1402 1361 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1403 : const char *pszClosePersistent =
1404 1361 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1405 1361 : if (pszPersistent)
1406 : {
1407 1004 : CPLString osSessionName = pszPersistent;
1408 502 : CPLMutexHolder oHolder(&hSessionMapMutex);
1409 :
1410 502 : if (poSessionMap == nullptr)
1411 136 : poSessionMap = new std::map<CPLString, CURL *>;
1412 502 : if (poSessionMap->count(osSessionName) == 0)
1413 : {
1414 158 : (*poSessionMap)[osSessionName] = curl_easy_init();
1415 158 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1416 : osSessionName.c_str());
1417 : }
1418 :
1419 502 : http_handle = (*poSessionMap)[osSessionName];
1420 : }
1421 : /* -------------------------------------------------------------------- */
1422 : /* Are we requested to close a persistent named session? */
1423 : /* -------------------------------------------------------------------- */
1424 859 : else if (pszClosePersistent)
1425 : {
1426 376 : CPLString osSessionName = pszClosePersistent;
1427 188 : CPLMutexHolder oHolder(&hSessionMapMutex);
1428 :
1429 188 : if (poSessionMap)
1430 : {
1431 : std::map<CPLString, CURL *>::iterator oIter =
1432 158 : poSessionMap->find(osSessionName);
1433 158 : if (oIter != poSessionMap->end())
1434 : {
1435 158 : curl_easy_cleanup(oIter->second);
1436 158 : poSessionMap->erase(oIter);
1437 158 : if (poSessionMap->empty())
1438 : {
1439 136 : delete poSessionMap;
1440 136 : poSessionMap = nullptr;
1441 : }
1442 158 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1443 : osSessionName.c_str());
1444 : }
1445 : else
1446 : {
1447 0 : CPLDebug("HTTP",
1448 : "Could not find persistent session named '%s'.",
1449 : osSessionName.c_str());
1450 : }
1451 : }
1452 :
1453 188 : return nullptr;
1454 : }
1455 : else
1456 671 : http_handle = curl_easy_init();
1457 :
1458 : /* -------------------------------------------------------------------- */
1459 : /* Setup the request. */
1460 : /* -------------------------------------------------------------------- */
1461 1173 : char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
1462 :
1463 1173 : CPLHTTPEmitFetchDebug(pszURL);
1464 :
1465 : CPLHTTPResult *psResult =
1466 1173 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1467 :
1468 : struct curl_slist *headers = reinterpret_cast<struct curl_slist *>(
1469 1173 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1470 1173 : if (headers != nullptr)
1471 581 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER, headers);
1472 :
1473 : // Are we making a head request.
1474 1173 : const char *pszNoBody = nullptr;
1475 1173 : if ((pszNoBody = CSLFetchNameValue(papszOptions, "NO_BODY")) != nullptr)
1476 : {
1477 0 : if (CPLTestBool(pszNoBody))
1478 : {
1479 0 : CPLDebug("HTTP", "HEAD Request: %s", pszURL);
1480 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOBODY, 1L);
1481 : }
1482 : }
1483 :
1484 : // Capture response headers.
1485 1173 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA, psResult);
1486 1173 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1487 : CPLHdrWriteFct);
1488 :
1489 1173 : CPLHTTPResultWithLimit sResultWithLimit;
1490 1173 : if (nullptr == pfnWrite)
1491 : {
1492 1172 : pfnWrite = CPLWriteFct;
1493 :
1494 1172 : sResultWithLimit.psResult = psResult;
1495 1172 : sResultWithLimit.nMaxFileSize = 0;
1496 : const char *pszMaxFileSize =
1497 1172 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1498 1172 : if (pszMaxFileSize != nullptr)
1499 : {
1500 10 : sResultWithLimit.nMaxFileSize = atoi(pszMaxFileSize);
1501 : // Only useful if size is returned by server before actual download.
1502 10 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1503 : sResultWithLimit.nMaxFileSize);
1504 : }
1505 1172 : pWriteArg = &sResultWithLimit;
1506 : }
1507 :
1508 1173 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA, pWriteArg);
1509 1173 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION, pfnWrite);
1510 :
1511 1173 : CurlProcessData stProcessData = {pfnProgress, pProgressArg};
1512 1173 : if (nullptr != pfnProgress)
1513 : {
1514 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFOFUNCTION,
1515 : NewProcessFunction);
1516 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XFERINFODATA,
1517 : &stProcessData);
1518 24 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOPROGRESS, 0L);
1519 : }
1520 :
1521 1173 : szCurlErrBuf[0] = '\0';
1522 :
1523 1173 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER, szCurlErrBuf);
1524 :
1525 1173 : bool bGZipRequested = false;
1526 1173 : if (bSupportGZip && CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1527 : {
1528 1173 : bGZipRequested = true;
1529 1173 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1530 : }
1531 :
1532 2346 : CPLHTTPPostFields oPostFields;
1533 1173 : if (oPostFields.Fill(http_handle, papszOptions) != CE_None)
1534 : {
1535 2 : psResult->nStatus = 34; // CURLE_HTTP_POST_ERROR
1536 2 : psResult->pszErrBuf = CPLStrdup(oPostFields.GetErrorMessage().c_str());
1537 2 : CPLError(CE_Failure, CPLE_AppDefined, "%s", psResult->pszErrBuf);
1538 2 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1539 2 : return psResult;
1540 : }
1541 :
1542 : /* -------------------------------------------------------------------- */
1543 : /* Depending on status code, retry this HTTP call until max */
1544 : /* retry has been reached */
1545 : /* -------------------------------------------------------------------- */
1546 1171 : const char *pszRetryDelay = CSLFetchNameValue(papszOptions, "RETRY_DELAY");
1547 1171 : if (pszRetryDelay == nullptr)
1548 1171 : pszRetryDelay = CPLGetConfigOption(
1549 : "GDAL_HTTP_RETRY_DELAY", CPLSPrintf("%f", CPL_HTTP_RETRY_DELAY));
1550 1171 : const char *pszMaxRetries = CSLFetchNameValue(papszOptions, "MAX_RETRY");
1551 1171 : if (pszMaxRetries == nullptr)
1552 1107 : pszMaxRetries = CPLGetConfigOption(
1553 : "GDAL_HTTP_MAX_RETRY", CPLSPrintf("%d", CPL_HTTP_MAX_RETRY));
1554 : // coverity[tainted_data]
1555 1171 : double dfRetryDelaySecs = CPLAtof(pszRetryDelay);
1556 1171 : int nMaxRetries = atoi(pszMaxRetries);
1557 1171 : const char *pszRetryCodes = CSLFetchNameValue(papszOptions, "RETRY_CODES");
1558 1171 : if (!pszRetryCodes)
1559 1171 : pszRetryCodes = CPLGetConfigOption("GDAL_HTTP_RETRY_CODES", nullptr);
1560 1171 : int nRetryCount = 0;
1561 :
1562 : while (true)
1563 : {
1564 : /* --------------------------------------------------------------------
1565 : */
1566 : /* Execute the request, waiting for results. */
1567 : /* --------------------------------------------------------------------
1568 : */
1569 1171 : void *old_handler = CPLHTTPIgnoreSigPipe();
1570 1171 : psResult->nStatus = static_cast<int>(curl_easy_perform(http_handle));
1571 1171 : CPLHTTPRestoreSigPipeHandler(old_handler);
1572 :
1573 : /* --------------------------------------------------------------------
1574 : */
1575 : /* Fetch content-type if possible. */
1576 : /* --------------------------------------------------------------------
1577 : */
1578 1171 : psResult->pszContentType = nullptr;
1579 1171 : curl_easy_getinfo(http_handle, CURLINFO_CONTENT_TYPE,
1580 : &(psResult->pszContentType));
1581 1171 : if (psResult->pszContentType != nullptr)
1582 530 : psResult->pszContentType = CPLStrdup(psResult->pszContentType);
1583 :
1584 1171 : long response_code = 0;
1585 1171 : curl_easy_getinfo(http_handle, CURLINFO_RESPONSE_CODE, &response_code);
1586 1171 : if (response_code != 200)
1587 : {
1588 416 : const double dfNewRetryDelay = CPLHTTPGetNewRetryDelay(
1589 : static_cast<int>(response_code), dfRetryDelaySecs,
1590 208 : reinterpret_cast<const char *>(psResult->pabyData),
1591 : szCurlErrBuf, pszRetryCodes);
1592 208 : if (dfNewRetryDelay > 0 && nRetryCount < nMaxRetries)
1593 : {
1594 0 : CPLError(CE_Warning, CPLE_AppDefined,
1595 : "HTTP error code: %d - %s. "
1596 : "Retrying again in %.1f secs",
1597 : static_cast<int>(response_code), pszURL,
1598 : dfRetryDelaySecs);
1599 0 : CPLSleep(dfRetryDelaySecs);
1600 0 : dfRetryDelaySecs = dfNewRetryDelay;
1601 0 : nRetryCount++;
1602 :
1603 0 : CPLFree(psResult->pszContentType);
1604 0 : psResult->pszContentType = nullptr;
1605 0 : CSLDestroy(psResult->papszHeaders);
1606 0 : psResult->papszHeaders = nullptr;
1607 0 : CPLFree(psResult->pabyData);
1608 0 : psResult->pabyData = nullptr;
1609 0 : psResult->nDataLen = 0;
1610 0 : psResult->nDataAlloc = 0;
1611 :
1612 0 : continue;
1613 : }
1614 : }
1615 :
1616 : /* --------------------------------------------------------------------
1617 : */
1618 : /* Have we encountered some sort of error? */
1619 : /* --------------------------------------------------------------------
1620 : */
1621 1171 : if (strlen(szCurlErrBuf) > 0)
1622 : {
1623 8 : bool bSkipError = false;
1624 : const char *pszContentLength =
1625 8 : CSLFetchNameValue(psResult->papszHeaders, "Content-Length");
1626 : // Some servers such as
1627 : // http://115.113.193.14/cgi-bin/world/qgis_mapserv.fcgi?VERSION=1.1.1&SERVICE=WMS&REQUEST=GetCapabilities
1628 : // invalidly return Content-Length as the uncompressed size, with
1629 : // makes curl to wait for more data and time-out finally. If we got
1630 : // the expected data size, then we don't emit an error but turn off
1631 : // GZip requests.
1632 8 : if (bGZipRequested &&
1633 8 : strstr(szCurlErrBuf, "transfer closed with") &&
1634 0 : strstr(szCurlErrBuf, "bytes remaining to read"))
1635 : {
1636 0 : if (pszContentLength && psResult->nDataLen != 0 &&
1637 0 : atoi(pszContentLength) == psResult->nDataLen)
1638 : {
1639 : const char *pszCurlGZIPOption =
1640 0 : CPLGetConfigOption("CPL_CURL_GZIP", nullptr);
1641 0 : if (pszCurlGZIPOption == nullptr)
1642 : {
1643 0 : CPLSetConfigOption("CPL_CURL_GZIP", "NO");
1644 0 : CPLDebug("HTTP",
1645 : "Disabling CPL_CURL_GZIP, "
1646 : "because %s doesn't support it properly",
1647 : pszURL);
1648 : }
1649 0 : psResult->nStatus = 0;
1650 0 : bSkipError = true;
1651 0 : }
1652 : }
1653 :
1654 : // Ignore SSL errors about non-properly terminated connection,
1655 : // often due to HTTP proxies
1656 8 : else if (pszContentLength == nullptr &&
1657 : // Cf https://github.com/curl/curl/pull/3148
1658 8 : (strstr(szCurlErrBuf,
1659 : "GnuTLS recv error (-110): The TLS connection was "
1660 8 : "non-properly terminated") != nullptr ||
1661 : // Cf https://github.com/curl/curl/issues/9024
1662 8 : strstr(szCurlErrBuf,
1663 : "SSL_read: error:0A000126:SSL "
1664 : "routines::unexpected eof while reading") !=
1665 : nullptr))
1666 : {
1667 0 : psResult->nStatus = 0;
1668 0 : bSkipError = true;
1669 : }
1670 8 : else if (CPLTestBool(
1671 : CPLGetConfigOption("CPL_CURL_IGNORE_ERROR", "NO")))
1672 : {
1673 0 : psResult->nStatus = 0;
1674 0 : bSkipError = true;
1675 : }
1676 :
1677 8 : if (!bSkipError)
1678 : {
1679 8 : psResult->pszErrBuf = CPLStrdup(szCurlErrBuf);
1680 8 : if (psResult->nDataLen > 0)
1681 : {
1682 1 : CPLError(CE_Failure, CPLE_AppDefined,
1683 : "%s. You may set the CPL_CURL_IGNORE_ERROR "
1684 : "configuration option to YES to try to ignore it.",
1685 : szCurlErrBuf);
1686 : }
1687 : else
1688 : {
1689 7 : CPLError(CE_Failure, CPLE_AppDefined, "%s", szCurlErrBuf);
1690 : }
1691 : }
1692 : }
1693 : else
1694 : {
1695 1163 : if (response_code >= 400 && response_code < 600)
1696 : {
1697 189 : psResult->pszErrBuf = CPLStrdup(CPLSPrintf(
1698 : "HTTP error code : %d", static_cast<int>(response_code)));
1699 189 : CPLError(CE_Failure, CPLE_AppDefined, "%s",
1700 : psResult->pszErrBuf);
1701 : }
1702 : }
1703 1171 : break;
1704 0 : }
1705 :
1706 1171 : CPLHTTPFetchCleanup(http_handle, headers, pszPersistent, papszOptions);
1707 :
1708 1171 : return psResult;
1709 : #endif /* def HAVE_CURL */
1710 : }
1711 :
1712 : #ifdef HAVE_CURL
1713 : /************************************************************************/
1714 : /* CPLMultiPerformWait() */
1715 : /************************************************************************/
1716 :
1717 3276 : bool CPLMultiPerformWait(void *hCurlMultiHandleIn, int & /*repeats*/)
1718 : {
1719 3276 : CURLM *hCurlMultiHandle = static_cast<CURLM *>(hCurlMultiHandleIn);
1720 :
1721 : // Wait for events on the sockets
1722 :
1723 : // Using curl_multi_poll() is preferred to avoid hitting the 1024 file
1724 : // descriptor limit
1725 :
1726 3276 : int numfds = 0;
1727 3276 : if (curl_multi_poll(hCurlMultiHandle, nullptr, 0, 1000, &numfds) !=
1728 : CURLM_OK)
1729 : {
1730 0 : CPLError(CE_Failure, CPLE_AppDefined, "curl_multi_poll() failed");
1731 0 : return false;
1732 : }
1733 3276 : return true;
1734 : }
1735 :
1736 : class CPLHTTPErrorBuffer
1737 : {
1738 : public:
1739 : char szBuffer[CURL_ERROR_SIZE + 1];
1740 :
1741 0 : CPLHTTPErrorBuffer()
1742 0 : {
1743 0 : szBuffer[0] = '\0';
1744 0 : }
1745 : };
1746 :
1747 : #endif // HAVE_CURL
1748 :
1749 : /************************************************************************/
1750 : /* CPLHTTPMultiFetch() */
1751 : /************************************************************************/
1752 :
1753 : /**
1754 : * \brief Fetch several documents at once
1755 : *
1756 : * @param papszURL array of valid URLs recognized by underlying download library
1757 : * (libcurl)
1758 : * @param nURLCount number of URLs of papszURL
1759 : * @param nMaxSimultaneous maximum number of downloads to issue simultaneously.
1760 : * Any negative or zer value means unlimited.
1761 : * @param papszOptions option list as a NULL-terminated array of strings. May be
1762 : * NULL. Refer to CPLHTTPFetch() for valid options.
1763 : * @return an array of CPLHTTPResult* structures that must be freed by
1764 : * CPLHTTPDestroyMultiResult() or NULL if libcurl support is disabled
1765 : *
1766 : * @since GDAL 2.3
1767 : */
1768 0 : CPLHTTPResult **CPLHTTPMultiFetch(const char *const *papszURL, int nURLCount,
1769 : int nMaxSimultaneous,
1770 : CSLConstList papszOptions)
1771 : {
1772 : #ifndef HAVE_CURL
1773 : (void)papszURL;
1774 : (void)nURLCount;
1775 : (void)nMaxSimultaneous;
1776 : (void)papszOptions;
1777 :
1778 : CPLError(CE_Failure, CPLE_NotSupported,
1779 : "GDAL/OGR not compiled with libcurl support, "
1780 : "remote requests not supported.");
1781 : return nullptr;
1782 : #else /* def HAVE_CURL */
1783 :
1784 : /* -------------------------------------------------------------------- */
1785 : /* Are we using a persistent named session? If so, search for */
1786 : /* or create it. */
1787 : /* */
1788 : /* Currently this code does not attempt to protect against */
1789 : /* multiple threads asking for the same named session. If that */
1790 : /* occurs it will be in use in multiple threads at once, which */
1791 : /* will lead to potential crashes in libcurl. */
1792 : /* -------------------------------------------------------------------- */
1793 0 : CURLM *hCurlMultiHandle = nullptr;
1794 :
1795 0 : const char *pszPersistent = CSLFetchNameValue(papszOptions, "PERSISTENT");
1796 : const char *pszClosePersistent =
1797 0 : CSLFetchNameValue(papszOptions, "CLOSE_PERSISTENT");
1798 0 : if (pszPersistent)
1799 : {
1800 0 : CPLString osSessionName = pszPersistent;
1801 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1802 :
1803 0 : if (poSessionMultiMap == nullptr)
1804 0 : poSessionMultiMap = new std::map<CPLString, CURLM *>;
1805 0 : if (poSessionMultiMap->count(osSessionName) == 0)
1806 : {
1807 0 : (*poSessionMultiMap)[osSessionName] = curl_multi_init();
1808 0 : CPLDebug("HTTP", "Establish persistent session named '%s'.",
1809 : osSessionName.c_str());
1810 : }
1811 :
1812 0 : hCurlMultiHandle = (*poSessionMultiMap)[osSessionName];
1813 : }
1814 : /* -------------------------------------------------------------------- */
1815 : /* Are we requested to close a persistent named session? */
1816 : /* -------------------------------------------------------------------- */
1817 0 : else if (pszClosePersistent)
1818 : {
1819 0 : CPLString osSessionName = pszClosePersistent;
1820 0 : CPLMutexHolder oHolder(&hSessionMapMutex);
1821 :
1822 0 : if (poSessionMultiMap)
1823 : {
1824 0 : auto oIter = poSessionMultiMap->find(osSessionName);
1825 0 : if (oIter != poSessionMultiMap->end())
1826 : {
1827 0 : VSICURLMultiCleanup(oIter->second);
1828 0 : poSessionMultiMap->erase(oIter);
1829 0 : if (poSessionMultiMap->empty())
1830 : {
1831 0 : delete poSessionMultiMap;
1832 0 : poSessionMultiMap = nullptr;
1833 : }
1834 0 : CPLDebug("HTTP", "Ended persistent session named '%s'.",
1835 : osSessionName.c_str());
1836 : }
1837 : else
1838 : {
1839 0 : CPLDebug("HTTP",
1840 : "Could not find persistent session named '%s'.",
1841 : osSessionName.c_str());
1842 : }
1843 : }
1844 :
1845 0 : return nullptr;
1846 : }
1847 : else
1848 : {
1849 0 : hCurlMultiHandle = curl_multi_init();
1850 : }
1851 :
1852 : CPLHTTPResult **papsResults = static_cast<CPLHTTPResult **>(
1853 0 : CPLCalloc(nURLCount, sizeof(CPLHTTPResult *)));
1854 0 : std::vector<CURL *> asHandles;
1855 0 : std::vector<CPLHTTPResultWithLimit> asResults;
1856 0 : asResults.resize(nURLCount);
1857 0 : std::vector<struct curl_slist *> aHeaders;
1858 0 : aHeaders.resize(nURLCount);
1859 0 : std::vector<CPLHTTPErrorBuffer> asErrorBuffers;
1860 0 : asErrorBuffers.resize(nURLCount);
1861 :
1862 0 : for (int i = 0; i < nURLCount; i++)
1863 : {
1864 0 : papsResults[i] =
1865 0 : static_cast<CPLHTTPResult *>(CPLCalloc(1, sizeof(CPLHTTPResult)));
1866 :
1867 0 : const char *pszURL = papszURL[i];
1868 0 : CURL *http_handle = curl_easy_init();
1869 :
1870 0 : aHeaders[i] = reinterpret_cast<struct curl_slist *>(
1871 0 : CPLHTTPSetOptions(http_handle, pszURL, papszOptions));
1872 :
1873 : // Set Headers.
1874 0 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
1875 0 : if (pszHeaders != nullptr)
1876 : {
1877 : char **papszTokensHeaders =
1878 0 : CSLTokenizeString2(pszHeaders, "\r\n", 0);
1879 0 : for (int j = 0; papszTokensHeaders[j] != nullptr; ++j)
1880 0 : aHeaders[i] =
1881 0 : curl_slist_append(aHeaders[i], papszTokensHeaders[j]);
1882 0 : CSLDestroy(papszTokensHeaders);
1883 : }
1884 :
1885 0 : if (aHeaders[i] != nullptr)
1886 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPHEADER,
1887 : aHeaders[i]);
1888 :
1889 : // Capture response headers.
1890 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERDATA,
1891 : papsResults[i]);
1892 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HEADERFUNCTION,
1893 : CPLHdrWriteFct);
1894 :
1895 0 : asResults[i].psResult = papsResults[i];
1896 : const char *pszMaxFileSize =
1897 0 : CSLFetchNameValue(papszOptions, "MAX_FILE_SIZE");
1898 0 : if (pszMaxFileSize != nullptr)
1899 : {
1900 0 : asResults[i].nMaxFileSize = atoi(pszMaxFileSize);
1901 : // Only useful if size is returned by server before actual download.
1902 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXFILESIZE,
1903 : asResults[i].nMaxFileSize);
1904 : }
1905 :
1906 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEDATA,
1907 : &asResults[i]);
1908 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_WRITEFUNCTION,
1909 : CPLWriteFct);
1910 :
1911 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ERRORBUFFER,
1912 : asErrorBuffers[i].szBuffer);
1913 :
1914 0 : if (bSupportGZip &&
1915 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_GZIP", "YES")))
1916 : {
1917 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_ENCODING, "gzip");
1918 : }
1919 :
1920 0 : asHandles.push_back(http_handle);
1921 : }
1922 :
1923 0 : int iCurRequest = 0;
1924 0 : for (;
1925 0 : iCurRequest <
1926 0 : std::min(nURLCount, nMaxSimultaneous > 0 ? nMaxSimultaneous : INT_MAX);
1927 : iCurRequest++)
1928 : {
1929 0 : CPLHTTPEmitFetchDebug(papszURL[iCurRequest],
1930 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1931 0 : curl_multi_add_handle(hCurlMultiHandle, asHandles[iCurRequest]);
1932 : }
1933 :
1934 0 : int repeats = 0;
1935 0 : void *old_handler = CPLHTTPIgnoreSigPipe();
1936 : while (true)
1937 : {
1938 0 : int still_running = 0;
1939 0 : while (curl_multi_perform(hCurlMultiHandle, &still_running) ==
1940 : CURLM_CALL_MULTI_PERFORM)
1941 : {
1942 : // loop
1943 : }
1944 0 : if (!still_running && iCurRequest == nURLCount)
1945 : {
1946 0 : break;
1947 : }
1948 :
1949 0 : bool bRequestsAdded = false;
1950 : CURLMsg *msg;
1951 0 : do
1952 : {
1953 0 : int msgq = 0;
1954 0 : msg = curl_multi_info_read(hCurlMultiHandle, &msgq);
1955 0 : if (msg && (msg->msg == CURLMSG_DONE))
1956 : {
1957 0 : if (iCurRequest < nURLCount)
1958 : {
1959 0 : CPLHTTPEmitFetchDebug(
1960 0 : papszURL[iCurRequest],
1961 : CPLSPrintf(" %d/%d", iCurRequest + 1, nURLCount));
1962 0 : curl_multi_add_handle(hCurlMultiHandle,
1963 0 : asHandles[iCurRequest]);
1964 0 : iCurRequest++;
1965 0 : bRequestsAdded = true;
1966 : }
1967 : }
1968 0 : } while (msg);
1969 :
1970 0 : if (!bRequestsAdded)
1971 0 : CPLMultiPerformWait(hCurlMultiHandle, repeats);
1972 0 : }
1973 0 : CPLHTTPRestoreSigPipeHandler(old_handler);
1974 :
1975 0 : for (int i = 0; i < nURLCount; i++)
1976 : {
1977 0 : if (asErrorBuffers[i].szBuffer[0] != '\0')
1978 0 : papsResults[i]->pszErrBuf = CPLStrdup(asErrorBuffers[i].szBuffer);
1979 : else
1980 : {
1981 0 : long response_code = 0;
1982 0 : curl_easy_getinfo(asHandles[i], CURLINFO_RESPONSE_CODE,
1983 : &response_code);
1984 :
1985 0 : if (response_code >= 400 && response_code < 600)
1986 : {
1987 0 : papsResults[i]->pszErrBuf = CPLStrdup(CPLSPrintf(
1988 : "HTTP error code : %d", static_cast<int>(response_code)));
1989 : }
1990 : }
1991 :
1992 0 : curl_easy_getinfo(asHandles[i], CURLINFO_CONTENT_TYPE,
1993 : &(papsResults[i]->pszContentType));
1994 0 : if (papsResults[i]->pszContentType != nullptr)
1995 0 : papsResults[i]->pszContentType =
1996 0 : CPLStrdup(papsResults[i]->pszContentType);
1997 :
1998 0 : curl_multi_remove_handle(hCurlMultiHandle, asHandles[i]);
1999 0 : curl_easy_cleanup(asHandles[i]);
2000 : }
2001 :
2002 0 : if (!pszPersistent)
2003 0 : VSICURLMultiCleanup(hCurlMultiHandle);
2004 :
2005 0 : for (size_t i = 0; i < aHeaders.size(); i++)
2006 0 : curl_slist_free_all(aHeaders[i]);
2007 :
2008 0 : return papsResults;
2009 : #endif /* def HAVE_CURL */
2010 : }
2011 :
2012 : /************************************************************************/
2013 : /* CPLHTTPDestroyMultiResult() */
2014 : /************************************************************************/
2015 : /**
2016 : * \brief Clean the memory associated with the return value of
2017 : * CPLHTTPMultiFetch()
2018 : *
2019 : * @param papsResults pointer to the return value of CPLHTTPMultiFetch()
2020 : * @param nCount value of the nURLCount parameter passed to CPLHTTPMultiFetch()
2021 : * @since GDAL 2.3
2022 : */
2023 0 : void CPLHTTPDestroyMultiResult(CPLHTTPResult **papsResults, int nCount)
2024 : {
2025 0 : if (papsResults)
2026 : {
2027 0 : for (int i = 0; i < nCount; i++)
2028 : {
2029 0 : CPLHTTPDestroyResult(papsResults[i]);
2030 : }
2031 0 : CPLFree(papsResults);
2032 : }
2033 0 : }
2034 :
2035 : #ifdef HAVE_CURL
2036 :
2037 : #ifdef _WIN32
2038 :
2039 : #include <windows.h>
2040 :
2041 : /************************************************************************/
2042 : /* CPLFindWin32CurlCaBundleCrt() */
2043 : /************************************************************************/
2044 :
2045 : static const char *CPLFindWin32CurlCaBundleCrt()
2046 : {
2047 : DWORD nResLen;
2048 : const DWORD nBufSize = MAX_PATH + 1;
2049 : char *pszFilePart = nullptr;
2050 :
2051 : char *pszPath = static_cast<char *>(CPLCalloc(1, nBufSize));
2052 :
2053 : nResLen = SearchPathA(nullptr, "curl-ca-bundle.crt", nullptr, nBufSize,
2054 : pszPath, &pszFilePart);
2055 : if (nResLen > 0)
2056 : {
2057 : const char *pszRet = CPLSPrintf("%s", pszPath);
2058 : CPLFree(pszPath);
2059 : return pszRet;
2060 : }
2061 : CPLFree(pszPath);
2062 : return nullptr;
2063 : }
2064 :
2065 : #endif // WIN32
2066 :
2067 : /************************************************************************/
2068 : /* CPLHTTPCurlDebugFunction() */
2069 : /************************************************************************/
2070 :
2071 16 : static int CPLHTTPCurlDebugFunction(CURL *handle, curl_infotype type,
2072 : char *data, size_t size, void *userp)
2073 : {
2074 : (void)handle;
2075 : (void)userp;
2076 :
2077 16 : const char *pszDebugKey = nullptr;
2078 16 : if (type == CURLINFO_TEXT)
2079 : {
2080 10 : pszDebugKey = "CURL_INFO_TEXT";
2081 : }
2082 6 : else if (type == CURLINFO_HEADER_OUT)
2083 : {
2084 1 : pszDebugKey = "CURL_INFO_HEADER_OUT";
2085 : }
2086 5 : else if (type == CURLINFO_HEADER_IN)
2087 : {
2088 5 : pszDebugKey = "CURL_INFO_HEADER_IN";
2089 : }
2090 0 : else if (type == CURLINFO_DATA_IN &&
2091 0 : CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE_DATA_IN", "NO")))
2092 : {
2093 0 : pszDebugKey = "CURL_INFO_DATA_IN";
2094 : }
2095 :
2096 16 : if (pszDebugKey)
2097 : {
2098 32 : std::string osMsg(data, size);
2099 16 : if (!osMsg.empty() && osMsg.back() == '\n')
2100 16 : osMsg.pop_back();
2101 16 : CPLDebug(pszDebugKey, "%s", osMsg.c_str());
2102 : }
2103 16 : return 0;
2104 : }
2105 :
2106 : /************************************************************************/
2107 : /* CPLHTTPSetOptions() */
2108 : /************************************************************************/
2109 :
2110 : // Note: papszOptions must be kept alive until curl_easy/multi_perform()
2111 : // has completed, and we must be careful not to set short lived strings
2112 : // with unchecked_curl_easy_setopt(), as long as we need to support curl < 7.17
2113 : // see https://curl.haxx.se/libcurl/c/unchecked_curl_easy_setopt.html
2114 : // caution: if we remove that assumption, we'll needto use
2115 : // CURLOPT_COPYPOSTFIELDS
2116 :
2117 2587 : void *CPLHTTPSetOptions(void *pcurl, const char *pszURL,
2118 : const char *const *papszOptions)
2119 : {
2120 2587 : CheckCurlFeatures();
2121 :
2122 2587 : CURL *http_handle = reinterpret_cast<CURL *>(pcurl);
2123 :
2124 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_URL, pszURL);
2125 :
2126 2587 : if (CPLTestBool(CPLGetConfigOption("CPL_CURL_VERBOSE", "NO")))
2127 : {
2128 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_VERBOSE, 1);
2129 :
2130 1 : if (CPLIsDebugEnabled())
2131 : {
2132 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_DEBUGFUNCTION,
2133 : CPLHTTPCurlDebugFunction);
2134 : }
2135 : }
2136 :
2137 : const char *pszHttpVersion =
2138 2587 : CSLFetchNameValue(papszOptions, "HTTP_VERSION");
2139 2587 : if (pszHttpVersion == nullptr)
2140 2587 : pszHttpVersion = CPLGetConfigOption("GDAL_HTTP_VERSION", nullptr);
2141 2587 : if (pszHttpVersion && strcmp(pszHttpVersion, "1.0") == 0)
2142 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2143 : CURL_HTTP_VERSION_1_0);
2144 2587 : else if (pszHttpVersion && strcmp(pszHttpVersion, "1.1") == 0)
2145 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2146 : CURL_HTTP_VERSION_1_1);
2147 2587 : else if (pszHttpVersion && (strcmp(pszHttpVersion, "2") == 0 ||
2148 0 : strcmp(pszHttpVersion, "2.0") == 0))
2149 : {
2150 0 : if (bSupportHTTP2)
2151 : {
2152 : // Try HTTP/2 both for HTTP and HTTPS. With fallback to HTTP/1.1
2153 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2154 : CURL_HTTP_VERSION_2_0);
2155 : }
2156 : }
2157 2587 : else if (pszHttpVersion && strcmp(pszHttpVersion, "2PRIOR_KNOWLEDGE") == 0)
2158 : {
2159 0 : if (bSupportHTTP2)
2160 : {
2161 : // Assume HTTP/2 is supported by the server. The cURL docs indicate
2162 : // that it makes no difference for HTTPS, but it does seem to work
2163 : // in practice.
2164 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2165 : CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE);
2166 : }
2167 : }
2168 2587 : else if (pszHttpVersion == nullptr || strcmp(pszHttpVersion, "2TLS") == 0)
2169 : {
2170 2587 : if (bSupportHTTP2)
2171 : {
2172 : // Only enable this mode if explicitly required, or if the
2173 : // machine is a GCE instance. On other networks, requesting a
2174 : // file in HTTP/2 is found to be significantly slower than HTTP/1.1
2175 : // for unknown reasons.
2176 2587 : if (pszHttpVersion != nullptr || CPLIsMachineForSureGCEInstance())
2177 : {
2178 : static bool bDebugEmitted = false;
2179 0 : if (!bDebugEmitted)
2180 : {
2181 0 : CPLDebug("HTTP", "Using HTTP/2 for HTTPS when possible");
2182 0 : bDebugEmitted = true;
2183 : }
2184 :
2185 : // CURL_HTTP_VERSION_2TLS means for HTTPS connection, try to
2186 : // negotiate HTTP/2 with the server (and fallback to HTTP/1.1
2187 : // otherwise), and for HTTP connection do HTTP/1
2188 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTP_VERSION,
2189 : CURL_HTTP_VERSION_2TLS);
2190 : }
2191 2587 : }
2192 : }
2193 : else
2194 : {
2195 0 : CPLError(CE_Warning, CPLE_NotSupported, "HTTP_VERSION=%s not supported",
2196 : pszHttpVersion);
2197 : }
2198 :
2199 : // Default value is 1 since curl 7.50.2. But worth applying it on
2200 : // previous versions as well.
2201 : const char *pszTCPNoDelay =
2202 2587 : CSLFetchNameValueDef(papszOptions, "TCP_NODELAY", "1");
2203 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_NODELAY,
2204 : atoi(pszTCPNoDelay));
2205 :
2206 : /* Support control over HTTPAUTH */
2207 2587 : const char *pszHttpAuth = CSLFetchNameValue(papszOptions, "HTTPAUTH");
2208 2587 : if (pszHttpAuth == nullptr)
2209 2577 : pszHttpAuth = CPLGetConfigOption("GDAL_HTTP_AUTH", nullptr);
2210 2587 : if (pszHttpAuth == nullptr)
2211 : {
2212 : /* do nothing */;
2213 : }
2214 15 : else if (EQUAL(pszHttpAuth, "BASIC"))
2215 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2216 : CURLAUTH_BASIC);
2217 15 : else if (EQUAL(pszHttpAuth, "NTLM"))
2218 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2219 : CURLAUTH_NTLM);
2220 15 : else if (EQUAL(pszHttpAuth, "ANY"))
2221 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
2222 15 : else if (EQUAL(pszHttpAuth, "ANYSAFE"))
2223 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2224 : CURLAUTH_ANYSAFE);
2225 15 : else if (EQUAL(pszHttpAuth, "BEARER"))
2226 : {
2227 15 : const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef(
2228 : papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES");
2229 : const bool bAuthorizationHeaderAllowed =
2230 15 : CPLTestBool(pszAuthorizationHeaderAllowed);
2231 15 : if (bAuthorizationHeaderAllowed)
2232 : {
2233 : const char *pszBearer =
2234 12 : CSLFetchNameValue(papszOptions, "HTTP_BEARER");
2235 12 : if (pszBearer == nullptr)
2236 5 : pszBearer = CPLGetConfigOption("GDAL_HTTP_BEARER", nullptr);
2237 12 : if (pszBearer != nullptr)
2238 12 : unchecked_curl_easy_setopt(http_handle, CURLOPT_XOAUTH2_BEARER,
2239 : pszBearer);
2240 12 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2241 : CURLAUTH_BEARER);
2242 : }
2243 : }
2244 0 : else if (EQUAL(pszHttpAuth, "NEGOTIATE"))
2245 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_HTTPAUTH,
2246 : CURLAUTH_NEGOTIATE);
2247 : else
2248 : {
2249 0 : CPLError(CE_Warning, CPLE_AppDefined,
2250 : "Unsupported HTTPAUTH value '%s', ignored.", pszHttpAuth);
2251 : }
2252 :
2253 : const char *pszGssDelegation =
2254 2587 : CSLFetchNameValue(papszOptions, "GSSAPI_DELEGATION");
2255 2587 : if (pszGssDelegation == nullptr)
2256 : pszGssDelegation =
2257 2587 : CPLGetConfigOption("GDAL_GSSAPI_DELEGATION", nullptr);
2258 2587 : if (pszGssDelegation == nullptr)
2259 : {
2260 : }
2261 0 : else if (EQUAL(pszGssDelegation, "NONE"))
2262 : {
2263 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2264 : CURLGSSAPI_DELEGATION_NONE);
2265 : }
2266 0 : else if (EQUAL(pszGssDelegation, "POLICY"))
2267 : {
2268 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2269 : CURLGSSAPI_DELEGATION_POLICY_FLAG);
2270 : }
2271 0 : else if (EQUAL(pszGssDelegation, "ALWAYS"))
2272 : {
2273 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_GSSAPI_DELEGATION,
2274 : CURLGSSAPI_DELEGATION_FLAG);
2275 : }
2276 : else
2277 : {
2278 0 : CPLError(CE_Warning, CPLE_AppDefined,
2279 : "Unsupported GSSAPI_DELEGATION value '%s', ignored.",
2280 : pszGssDelegation);
2281 : }
2282 :
2283 : // Support use of .netrc - default enabled.
2284 2587 : const char *pszHttpNetrc = CSLFetchNameValue(papszOptions, "NETRC");
2285 2587 : if (pszHttpNetrc == nullptr)
2286 2587 : pszHttpNetrc = CPLGetConfigOption("GDAL_HTTP_NETRC", "YES");
2287 2587 : if (pszHttpNetrc == nullptr || CPLTestBool(pszHttpNetrc))
2288 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC, 1L);
2289 :
2290 : // Custom .netrc file location
2291 : const char *pszHttpNetrcFile =
2292 2587 : CSLFetchNameValue(papszOptions, "NETRC_FILE");
2293 2587 : if (pszHttpNetrcFile == nullptr)
2294 2587 : pszHttpNetrcFile = CPLGetConfigOption("GDAL_HTTP_NETRC_FILE", nullptr);
2295 2587 : if (pszHttpNetrcFile)
2296 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NETRC_FILE,
2297 : pszHttpNetrcFile);
2298 :
2299 : // Support setting userid:password.
2300 2587 : const char *pszUserPwd = CSLFetchNameValue(papszOptions, "USERPWD");
2301 2587 : if (pszUserPwd == nullptr)
2302 2549 : pszUserPwd = CPLGetConfigOption("GDAL_HTTP_USERPWD", nullptr);
2303 2587 : if (pszUserPwd != nullptr)
2304 38 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERPWD, pszUserPwd);
2305 :
2306 : // Set Proxy parameters.
2307 2587 : const char *pszProxy = CSLFetchNameValue(papszOptions, "PROXY");
2308 2587 : if (pszProxy == nullptr)
2309 2587 : pszProxy = CPLGetConfigOption("GDAL_HTTP_PROXY", nullptr);
2310 2587 : if (pszProxy)
2311 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszProxy);
2312 :
2313 2587 : const char *pszHttpsProxy = CSLFetchNameValue(papszOptions, "HTTPS_PROXY");
2314 2587 : if (pszHttpsProxy == nullptr)
2315 2587 : pszHttpsProxy = CPLGetConfigOption("GDAL_HTTPS_PROXY", nullptr);
2316 2587 : if (pszHttpsProxy && (STARTS_WITH(pszURL, "https")))
2317 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXY, pszHttpsProxy);
2318 :
2319 : const char *pszProxyUserPwd =
2320 2587 : CSLFetchNameValue(papszOptions, "PROXYUSERPWD");
2321 2587 : if (pszProxyUserPwd == nullptr)
2322 2587 : pszProxyUserPwd = CPLGetConfigOption("GDAL_HTTP_PROXYUSERPWD", nullptr);
2323 2587 : if (pszProxyUserPwd)
2324 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYUSERPWD,
2325 : pszProxyUserPwd);
2326 :
2327 : // Support control over PROXYAUTH.
2328 2587 : const char *pszProxyAuth = CSLFetchNameValue(papszOptions, "PROXYAUTH");
2329 2587 : if (pszProxyAuth == nullptr)
2330 2587 : pszProxyAuth = CPLGetConfigOption("GDAL_PROXY_AUTH", nullptr);
2331 2587 : if (pszProxyAuth == nullptr)
2332 : {
2333 : // Do nothing.
2334 : }
2335 0 : else if (EQUAL(pszProxyAuth, "BASIC"))
2336 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2337 : CURLAUTH_BASIC);
2338 0 : else if (EQUAL(pszProxyAuth, "NTLM"))
2339 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2340 : CURLAUTH_NTLM);
2341 0 : else if (EQUAL(pszProxyAuth, "DIGEST"))
2342 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2343 : CURLAUTH_DIGEST);
2344 0 : else if (EQUAL(pszProxyAuth, "ANY"))
2345 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2346 : CURLAUTH_ANY);
2347 0 : else if (EQUAL(pszProxyAuth, "ANYSAFE"))
2348 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2349 : CURLAUTH_ANYSAFE);
2350 0 : else if (EQUAL(pszProxyAuth, "NEGOTIATE"))
2351 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_PROXYAUTH,
2352 : CURLAUTH_NEGOTIATE);
2353 : else
2354 : {
2355 0 : CPLError(CE_Warning, CPLE_AppDefined,
2356 : "Unsupported PROXYAUTH value '%s', ignored.", pszProxyAuth);
2357 : }
2358 :
2359 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SUPPRESS_CONNECT_HEADERS,
2360 : 1L);
2361 :
2362 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_FOLLOWLOCATION, 1);
2363 2587 : const char *pszUnrestrictedAuth = CPLGetConfigOption(
2364 : "CPL_VSIL_CURL_AUTHORIZATION_HEADER_ALLOWED_IF_REDIRECT",
2365 : "IF_SAME_HOST");
2366 2605 : if (!EQUAL(pszUnrestrictedAuth, "IF_SAME_HOST") &&
2367 18 : CPLTestBool(pszUnrestrictedAuth))
2368 : {
2369 9 : unchecked_curl_easy_setopt(http_handle, CURLOPT_UNRESTRICTED_AUTH, 1);
2370 : }
2371 :
2372 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_MAXREDIRS, 10);
2373 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTREDIR,
2374 : CURL_REDIR_POST_ALL);
2375 :
2376 : // Set connect timeout.
2377 : const char *pszConnectTimeout =
2378 2587 : CSLFetchNameValue(papszOptions, "CONNECTTIMEOUT");
2379 2587 : if (pszConnectTimeout == nullptr)
2380 : pszConnectTimeout =
2381 2587 : CPLGetConfigOption("GDAL_HTTP_CONNECTTIMEOUT", nullptr);
2382 2587 : if (pszConnectTimeout != nullptr)
2383 : {
2384 : // coverity[tainted_data]
2385 1 : unchecked_curl_easy_setopt(
2386 : http_handle, CURLOPT_CONNECTTIMEOUT_MS,
2387 : static_cast<int>(1000 * CPLAtof(pszConnectTimeout)));
2388 : }
2389 :
2390 : // Set timeout.
2391 2587 : const char *pszTimeout = CSLFetchNameValue(papszOptions, "TIMEOUT");
2392 2587 : if (pszTimeout == nullptr)
2393 2293 : pszTimeout = CPLGetConfigOption("GDAL_HTTP_TIMEOUT", nullptr);
2394 2587 : if (pszTimeout != nullptr)
2395 : {
2396 : // coverity[tainted_data]
2397 303 : unchecked_curl_easy_setopt(
2398 : http_handle, CURLOPT_TIMEOUT_MS,
2399 : static_cast<int>(1000 * CPLAtof(pszTimeout)));
2400 : }
2401 :
2402 : // Set low speed time and limit.
2403 : const char *pszLowSpeedTime =
2404 2587 : CSLFetchNameValue(papszOptions, "LOW_SPEED_TIME");
2405 2587 : if (pszLowSpeedTime == nullptr)
2406 : pszLowSpeedTime =
2407 2587 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_TIME", nullptr);
2408 2587 : if (pszLowSpeedTime != nullptr)
2409 : {
2410 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_TIME,
2411 : atoi(pszLowSpeedTime));
2412 :
2413 : const char *pszLowSpeedLimit =
2414 0 : CSLFetchNameValue(papszOptions, "LOW_SPEED_LIMIT");
2415 0 : if (pszLowSpeedLimit == nullptr)
2416 : pszLowSpeedLimit =
2417 0 : CPLGetConfigOption("GDAL_HTTP_LOW_SPEED_LIMIT", "1");
2418 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_LOW_SPEED_LIMIT,
2419 : atoi(pszLowSpeedLimit));
2420 : }
2421 :
2422 : /* Disable some SSL verification */
2423 2587 : const char *pszUnsafeSSL = CSLFetchNameValue(papszOptions, "UNSAFESSL");
2424 2587 : if (pszUnsafeSSL == nullptr)
2425 2564 : pszUnsafeSSL = CPLGetConfigOption("GDAL_HTTP_UNSAFESSL", nullptr);
2426 2587 : if (pszUnsafeSSL != nullptr && CPLTestBool(pszUnsafeSSL))
2427 : {
2428 23 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYPEER, 0L);
2429 23 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYHOST, 0L);
2430 : }
2431 :
2432 : const char *pszUseCAPIStore =
2433 2587 : CSLFetchNameValue(papszOptions, "USE_CAPI_STORE");
2434 2587 : if (pszUseCAPIStore == nullptr)
2435 2587 : pszUseCAPIStore = CPLGetConfigOption("GDAL_HTTP_USE_CAPI_STORE", "NO");
2436 2587 : if (CPLTestBool(pszUseCAPIStore))
2437 : {
2438 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2439 : // Use certificates from Windows certificate store; requires
2440 : // crypt32.lib, OpenSSL crypto and ssl libraries.
2441 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_CTX_FUNCTION,
2442 : *CPL_ssl_ctx_callback);
2443 : #else // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2444 2 : CPLError(CE_Warning, CPLE_NotSupported,
2445 : "GDAL_HTTP_USE_CAPI_STORE requested, but libcurl too old, "
2446 : "non-Windows platform or OpenSSL missing.");
2447 : #endif // defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2448 : }
2449 :
2450 : // Enable OCSP stapling if requested.
2451 : const char *pszSSLVerifyStatus =
2452 2587 : CSLFetchNameValue(papszOptions, "SSL_VERIFYSTATUS");
2453 2587 : if (pszSSLVerifyStatus == nullptr)
2454 : pszSSLVerifyStatus =
2455 2587 : CPLGetConfigOption("GDAL_HTTP_SSL_VERIFYSTATUS", "NO");
2456 2587 : if (CPLTestBool(pszSSLVerifyStatus))
2457 : {
2458 1 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSL_VERIFYSTATUS, 1L);
2459 : }
2460 :
2461 : // Custom path to SSL certificates.
2462 2587 : const char *pszCAInfo = CSLFetchNameValue(papszOptions, "CAINFO");
2463 2587 : if (pszCAInfo == nullptr)
2464 : // Name of GDAL environment variable for the CA Bundle path
2465 2587 : pszCAInfo = CPLGetConfigOption("GDAL_CURL_CA_BUNDLE", nullptr);
2466 2587 : if (pszCAInfo == nullptr)
2467 : // Name of environment variable used by the curl binary
2468 2587 : pszCAInfo = CPLGetConfigOption("CURL_CA_BUNDLE", nullptr);
2469 2587 : if (pszCAInfo == nullptr)
2470 : // Name of environment variable used by the curl binary (tested
2471 : // after CURL_CA_BUNDLE
2472 2587 : pszCAInfo = CPLGetConfigOption("SSL_CERT_FILE", nullptr);
2473 : #ifdef _WIN32
2474 : if (pszCAInfo == nullptr)
2475 : {
2476 : pszCAInfo = CPLFindWin32CurlCaBundleCrt();
2477 : }
2478 : #endif
2479 2587 : if (pszCAInfo != nullptr)
2480 : {
2481 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAINFO, pszCAInfo);
2482 : }
2483 :
2484 2587 : const char *pszCAPath = CSLFetchNameValue(papszOptions, "CAPATH");
2485 2587 : if (pszCAPath != nullptr)
2486 : {
2487 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CAPATH, pszCAPath);
2488 : }
2489 :
2490 : // Support for SSL client certificates
2491 :
2492 : // Filename of the the client certificate
2493 2587 : const char *pszSSLCert = CSLFetchNameValue(papszOptions, "SSLCERT");
2494 2587 : if (!pszSSLCert)
2495 2587 : pszSSLCert = CPLGetConfigOption("GDAL_HTTP_SSLCERT", nullptr);
2496 2587 : if (pszSSLCert)
2497 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERT, pszSSLCert);
2498 :
2499 : // private key file for TLS and SSL client cert
2500 2587 : const char *pszSSLKey = CSLFetchNameValue(papszOptions, "SSLKEY");
2501 2587 : if (!pszSSLKey)
2502 2587 : pszSSLKey = CPLGetConfigOption("GDAL_HTTP_SSLKEY", nullptr);
2503 2587 : if (pszSSLKey)
2504 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLKEY, pszSSLKey);
2505 :
2506 : // type of client SSL certificate ("PEM", "DER", ...)
2507 2587 : const char *pszSSLCertType = CSLFetchNameValue(papszOptions, "SSLCERTTYPE");
2508 2587 : if (!pszSSLCertType)
2509 2587 : pszSSLCertType = CPLGetConfigOption("GDAL_HTTP_SSLCERTTYPE", nullptr);
2510 2587 : if (pszSSLCertType)
2511 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_SSLCERTTYPE,
2512 : pszSSLCertType);
2513 :
2514 : // passphrase to private key
2515 2587 : const char *pszKeyPasswd = CSLFetchNameValue(papszOptions, "KEYPASSWD");
2516 2587 : if (!pszKeyPasswd)
2517 2587 : pszKeyPasswd = CPLGetConfigOption("GDAL_HTTP_KEYPASSWD", nullptr);
2518 2587 : if (pszKeyPasswd)
2519 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_KEYPASSWD,
2520 : pszKeyPasswd);
2521 :
2522 : /* Set Referer */
2523 2587 : const char *pszReferer = CSLFetchNameValue(papszOptions, "REFERER");
2524 2587 : if (pszReferer != nullptr)
2525 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_REFERER, pszReferer);
2526 :
2527 : /* Set User-Agent */
2528 2587 : const char *pszUserAgent = CSLFetchNameValue(papszOptions, "USERAGENT");
2529 2587 : if (pszUserAgent == nullptr)
2530 2499 : pszUserAgent = CPLGetConfigOption("GDAL_HTTP_USERAGENT",
2531 : gosDefaultUserAgent.c_str());
2532 2587 : if (pszUserAgent != nullptr && !EQUAL(pszUserAgent, ""))
2533 : {
2534 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_USERAGENT,
2535 : pszUserAgent);
2536 : }
2537 :
2538 : /* NOSIGNAL should be set to true for timeout to work in multithread
2539 : * environments on Unix, requires libcurl 7.10 or more recent.
2540 : * (this force avoiding the use of signal handlers)
2541 : */
2542 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_NOSIGNAL, 1);
2543 :
2544 : const char *pszFormFilePath =
2545 2587 : CSLFetchNameValue(papszOptions, "FORM_FILE_PATH");
2546 : const char *pszParametersCount =
2547 2587 : CSLFetchNameValue(papszOptions, "FORM_ITEM_COUNT");
2548 2587 : if (pszFormFilePath == nullptr && pszParametersCount == nullptr)
2549 : {
2550 : /* Set POST mode */
2551 2585 : const char *pszPost = CSLFetchNameValue(papszOptions, "POSTFIELDS");
2552 2585 : if (pszPost != nullptr)
2553 : {
2554 159 : CPLDebug("HTTP", "These POSTFIELDS were sent:%.4000s", pszPost);
2555 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POST, 1);
2556 159 : unchecked_curl_easy_setopt(http_handle, CURLOPT_POSTFIELDS,
2557 : pszPost);
2558 : }
2559 : }
2560 :
2561 : const char *pszCustomRequest =
2562 2587 : CSLFetchNameValue(papszOptions, "CUSTOMREQUEST");
2563 2587 : if (pszCustomRequest != nullptr)
2564 : {
2565 138 : unchecked_curl_easy_setopt(http_handle, CURLOPT_CUSTOMREQUEST,
2566 : pszCustomRequest);
2567 : }
2568 :
2569 2587 : const char *pszCookie = CSLFetchNameValue(papszOptions, "COOKIE");
2570 2587 : if (pszCookie == nullptr)
2571 2587 : pszCookie = CPLGetConfigOption("GDAL_HTTP_COOKIE", nullptr);
2572 2587 : if (pszCookie != nullptr)
2573 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIE, pszCookie);
2574 :
2575 2587 : const char *pszCookieFile = CSLFetchNameValue(papszOptions, "COOKIEFILE");
2576 2587 : if (pszCookieFile == nullptr)
2577 2587 : pszCookieFile = CPLGetConfigOption("GDAL_HTTP_COOKIEFILE", nullptr);
2578 2587 : if (pszCookieFile != nullptr)
2579 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEFILE,
2580 : pszCookieFile);
2581 :
2582 2587 : const char *pszCookieJar = CSLFetchNameValue(papszOptions, "COOKIEJAR");
2583 2587 : if (pszCookieJar == nullptr)
2584 2587 : pszCookieJar = CPLGetConfigOption("GDAL_HTTP_COOKIEJAR", nullptr);
2585 2587 : if (pszCookieJar != nullptr)
2586 0 : unchecked_curl_easy_setopt(http_handle, CURLOPT_COOKIEJAR,
2587 : pszCookieJar);
2588 :
2589 : // TCP keep-alive
2590 : const char *pszTCPKeepAlive =
2591 2587 : CSLFetchNameValue(papszOptions, "TCP_KEEPALIVE");
2592 2587 : if (pszTCPKeepAlive == nullptr)
2593 2587 : pszTCPKeepAlive = CPLGetConfigOption("GDAL_HTTP_TCP_KEEPALIVE", "YES");
2594 2587 : if (pszTCPKeepAlive != nullptr && CPLTestBool(pszTCPKeepAlive))
2595 : {
2596 : // Set keep-alive interval.
2597 2587 : int nKeepAliveInterval = 60;
2598 : const char *pszKeepAliveInterval =
2599 2587 : CSLFetchNameValue(papszOptions, "TCP_KEEPINTVL");
2600 2587 : if (pszKeepAliveInterval == nullptr)
2601 : pszKeepAliveInterval =
2602 2587 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPINTVL", nullptr);
2603 2587 : if (pszKeepAliveInterval != nullptr)
2604 1 : nKeepAliveInterval = atoi(pszKeepAliveInterval);
2605 :
2606 : // Set keep-alive idle wait time.
2607 2587 : int nKeepAliveIdle = 60;
2608 : const char *pszKeepAliveIdle =
2609 2587 : CSLFetchNameValue(papszOptions, "TCP_KEEPIDLE");
2610 2587 : if (pszKeepAliveIdle == nullptr)
2611 : pszKeepAliveIdle =
2612 2587 : CPLGetConfigOption("GDAL_HTTP_TCP_KEEPIDLE", nullptr);
2613 2587 : if (pszKeepAliveIdle != nullptr)
2614 1 : nKeepAliveIdle = atoi(pszKeepAliveIdle);
2615 :
2616 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPALIVE, 1L);
2617 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPINTVL,
2618 : nKeepAliveInterval);
2619 2587 : unchecked_curl_easy_setopt(http_handle, CURLOPT_TCP_KEEPIDLE,
2620 : nKeepAliveIdle);
2621 : }
2622 :
2623 2587 : struct curl_slist *headers = nullptr;
2624 2587 : const char *pszAccept = CSLFetchNameValue(papszOptions, "ACCEPT");
2625 2587 : if (pszAccept)
2626 : {
2627 4 : headers =
2628 4 : curl_slist_append(headers, CPLSPrintf("Accept: %s", pszAccept));
2629 : }
2630 :
2631 1692 : const auto AddHeader = [&headers, pszAccept](const char *pszHeader)
2632 : {
2633 675 : if (STARTS_WITH_CI(pszHeader, "Accept:") && pszAccept)
2634 : {
2635 1 : const char *pszVal = pszHeader + strlen("Accept:");
2636 2 : while (*pszVal == ' ')
2637 1 : ++pszVal;
2638 1 : if (!EQUAL(pszVal, pszAccept))
2639 : {
2640 : // Cf https://github.com/OSGeo/gdal/issues/7691#issuecomment-2873711603
2641 1 : CPLDebug(
2642 : "HTTP",
2643 : "Ignoring '%s' since ACCEPT option = '%s' is specified",
2644 : pszHeader, pszAccept);
2645 1 : }
2646 : }
2647 : else
2648 : {
2649 674 : headers = curl_slist_append(headers, pszHeader);
2650 : }
2651 3262 : };
2652 :
2653 2587 : const char *pszHeaderFile = CSLFetchNameValue(papszOptions, "HEADER_FILE");
2654 2587 : if (pszHeaderFile == nullptr)
2655 2585 : pszHeaderFile = CPLGetConfigOption("GDAL_HTTP_HEADER_FILE", nullptr);
2656 2587 : if (pszHeaderFile != nullptr)
2657 : {
2658 2 : VSILFILE *fp = nullptr;
2659 : // Do not allow /vsicurl/ access from /vsicurl because of
2660 : // GetCurlHandleFor() e.g. "/vsicurl/,HEADER_FILE=/vsicurl/,url= " would
2661 : // cause use of memory after free
2662 2 : if (!STARTS_WITH(pszHeaderFile, "/vsi") ||
2663 2 : STARTS_WITH(pszHeaderFile, "/vsimem/"))
2664 : {
2665 2 : fp = VSIFOpenL(pszHeaderFile, "rb");
2666 : }
2667 2 : if (fp == nullptr)
2668 : {
2669 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot read %s", pszHeaderFile);
2670 : }
2671 : else
2672 : {
2673 2 : const char *pszLine = nullptr;
2674 4 : while ((pszLine = CPLReadLineL(fp)) != nullptr)
2675 : {
2676 2 : AddHeader(pszLine);
2677 : }
2678 2 : VSIFCloseL(fp);
2679 : }
2680 : }
2681 :
2682 2587 : const char *pszHeaders = CSLFetchNameValue(papszOptions, "HEADERS");
2683 2587 : if (pszHeaders == nullptr)
2684 1986 : pszHeaders = CPLGetConfigOption("GDAL_HTTP_HEADERS", nullptr);
2685 2587 : if (pszHeaders)
2686 : {
2687 610 : bool bHeadersDone = false;
2688 : // Compatibility hack for "HEADERS=Accept: text/plain, application/json"
2689 610 : if (strstr(pszHeaders, "\r\n") == nullptr)
2690 : {
2691 551 : const char *pszComma = strchr(pszHeaders, ',');
2692 551 : if (pszComma != nullptr && strchr(pszComma, ':') == nullptr)
2693 : {
2694 211 : AddHeader(pszHeaders);
2695 211 : bHeadersDone = true;
2696 : }
2697 : }
2698 610 : if (!bHeadersDone)
2699 : {
2700 399 : const char *pszAuthorizationHeaderAllowed = CSLFetchNameValueDef(
2701 : papszOptions, "AUTHORIZATION_HEADER_ALLOWED", "YES");
2702 : const bool bAuthorizationHeaderAllowed =
2703 399 : CPLTestBool(pszAuthorizationHeaderAllowed);
2704 :
2705 : // We accept both raw headers with \r\n as a separator, or as
2706 : // a comma separated list of foo: bar values.
2707 : const CPLStringList aosTokens(
2708 399 : strstr(pszHeaders, "\r\n")
2709 59 : ? CSLTokenizeString2(pszHeaders, "\r\n", 0)
2710 857 : : CSLTokenizeString2(pszHeaders, ",", CSLT_HONOURSTRINGS));
2711 865 : for (int i = 0; i < aosTokens.size(); ++i)
2712 : {
2713 470 : if (bAuthorizationHeaderAllowed ||
2714 4 : !STARTS_WITH_CI(aosTokens[i], "Authorization:"))
2715 : {
2716 462 : AddHeader(aosTokens[i]);
2717 : }
2718 : }
2719 : }
2720 : }
2721 :
2722 2587 : return headers;
2723 : }
2724 :
2725 : /************************************************************************/
2726 : /* CPLHTTPIgnoreSigPipe() */
2727 : /************************************************************************/
2728 :
2729 : /* If using OpenSSL with Curl, openssl can cause SIGPIPE to be triggered */
2730 : /* As we set CURLOPT_NOSIGNAL = 1, we must manually handle this situation */
2731 :
2732 2799 : void *CPLHTTPIgnoreSigPipe()
2733 : {
2734 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2735 : struct sigaction old_pipe_act;
2736 : struct sigaction action;
2737 : /* Get previous handler */
2738 2799 : memset(&old_pipe_act, 0, sizeof(struct sigaction));
2739 2799 : sigaction(SIGPIPE, nullptr, &old_pipe_act);
2740 :
2741 : /* Install new handler */
2742 2799 : action = old_pipe_act;
2743 2799 : action.sa_handler = SIG_IGN;
2744 2799 : sigaction(SIGPIPE, &action, nullptr);
2745 :
2746 2799 : void *ret = CPLMalloc(sizeof(old_pipe_act));
2747 2799 : memcpy(ret, &old_pipe_act, sizeof(old_pipe_act));
2748 2799 : return ret;
2749 : #else
2750 : return nullptr;
2751 : #endif
2752 : }
2753 :
2754 : /************************************************************************/
2755 : /* CPLHTTPRestoreSigPipeHandler() */
2756 : /************************************************************************/
2757 :
2758 2799 : void CPLHTTPRestoreSigPipeHandler(void *old_handler)
2759 : {
2760 : #if defined(SIGPIPE) && defined(HAVE_SIGACTION)
2761 2799 : sigaction(SIGPIPE, static_cast<struct sigaction *>(old_handler), nullptr);
2762 2799 : CPLFree(old_handler);
2763 : #else
2764 : (void)old_handler;
2765 : #endif
2766 2799 : }
2767 :
2768 : #endif // def HAVE_CURL
2769 :
2770 : /************************************************************************/
2771 : /* CPLHTTPEnabled() */
2772 : /************************************************************************/
2773 :
2774 : /**
2775 : * \brief Return if CPLHTTP services can be useful
2776 : *
2777 : * Those services depend on GDAL being build with libcurl support.
2778 : *
2779 : * @return TRUE if libcurl support is enabled
2780 : */
2781 9 : int CPLHTTPEnabled()
2782 :
2783 : {
2784 : #ifdef HAVE_CURL
2785 9 : return TRUE;
2786 : #else
2787 : return FALSE;
2788 : #endif
2789 : }
2790 :
2791 : /************************************************************************/
2792 : /* CPLHTTPCleanup() */
2793 : /************************************************************************/
2794 :
2795 : /**
2796 : * \brief Cleanup function to call at application termination
2797 : */
2798 1114 : void CPLHTTPCleanup()
2799 :
2800 : {
2801 : #ifdef HAVE_CURL
2802 1114 : if (!hSessionMapMutex)
2803 1113 : return;
2804 :
2805 : {
2806 2 : CPLMutexHolder oHolder(&hSessionMapMutex);
2807 1 : if (poSessionMap)
2808 : {
2809 0 : for (auto &kv : *poSessionMap)
2810 : {
2811 0 : curl_easy_cleanup(kv.second);
2812 : }
2813 0 : delete poSessionMap;
2814 0 : poSessionMap = nullptr;
2815 : }
2816 1 : if (poSessionMultiMap)
2817 : {
2818 0 : for (auto &kv : *poSessionMultiMap)
2819 : {
2820 0 : VSICURLMultiCleanup(kv.second);
2821 : }
2822 0 : delete poSessionMultiMap;
2823 0 : poSessionMultiMap = nullptr;
2824 : }
2825 : }
2826 :
2827 : // Not quite a safe sequence.
2828 1 : CPLDestroyMutex(hSessionMapMutex);
2829 1 : hSessionMapMutex = nullptr;
2830 :
2831 : #if defined(_WIN32) && defined(HAVE_OPENSSL_CRYPTO)
2832 : // This cleanup must be absolutely done before CPLOpenSSLCleanup()
2833 : // for some unknown reason, but otherwise X509_free() in
2834 : // CPLWindowsCertificateListCleanup() will crash.
2835 : CPLWindowsCertificateListCleanup();
2836 : #endif
2837 :
2838 : #if defined(HAVE_OPENSSL_CRYPTO) && OPENSSL_VERSION_NUMBER < 0x10100000
2839 : CPLOpenSSLCleanup();
2840 : #endif
2841 :
2842 : #endif
2843 : }
2844 :
2845 : /************************************************************************/
2846 : /* CPLHTTPDestroyResult() */
2847 : /************************************************************************/
2848 :
2849 : /**
2850 : * \brief Clean the memory associated with the return value of CPLHTTPFetch()
2851 : *
2852 : * @param psResult pointer to the return value of CPLHTTPFetch()
2853 : */
2854 2199 : void CPLHTTPDestroyResult(CPLHTTPResult *psResult)
2855 :
2856 : {
2857 2199 : if (psResult)
2858 : {
2859 2011 : CPLFree(psResult->pabyData);
2860 2011 : CPLFree(psResult->pszErrBuf);
2861 2011 : CPLFree(psResult->pszContentType);
2862 2011 : CSLDestroy(psResult->papszHeaders);
2863 :
2864 2068 : for (int i = 0; i < psResult->nMimePartCount; i++)
2865 : {
2866 57 : CSLDestroy(psResult->pasMimePart[i].papszHeaders);
2867 : }
2868 2011 : CPLFree(psResult->pasMimePart);
2869 :
2870 2011 : CPLFree(psResult);
2871 : }
2872 2199 : }
2873 :
2874 : /************************************************************************/
2875 : /* CPLHTTPParseMultipartMime() */
2876 : /************************************************************************/
2877 :
2878 : /**
2879 : * \brief Parses a MIME multipart message.
2880 : *
2881 : * This function will iterate over each part and put it in a separate
2882 : * element of the pasMimePart array of the provided psResult structure.
2883 : *
2884 : * @param psResult pointer to the return value of CPLHTTPFetch()
2885 : * @return TRUE if the message contains MIME multipart message.
2886 : */
2887 45 : int CPLHTTPParseMultipartMime(CPLHTTPResult *psResult)
2888 :
2889 : {
2890 : /* -------------------------------------------------------------------- */
2891 : /* Is it already done? */
2892 : /* -------------------------------------------------------------------- */
2893 45 : if (psResult->nMimePartCount > 0)
2894 5 : return TRUE;
2895 :
2896 : /* -------------------------------------------------------------------- */
2897 : /* Find the boundary setting in the content type. */
2898 : /* -------------------------------------------------------------------- */
2899 40 : const char *pszBound = nullptr;
2900 :
2901 40 : if (psResult->pszContentType != nullptr)
2902 38 : pszBound = strstr(psResult->pszContentType, "boundary=");
2903 :
2904 40 : if (pszBound == nullptr)
2905 : {
2906 2 : CPLError(CE_Failure, CPLE_AppDefined,
2907 : "Unable to parse multi-part mime, no boundary setting.");
2908 2 : return FALSE;
2909 : }
2910 :
2911 76 : CPLString osBoundary;
2912 : char **papszTokens =
2913 38 : CSLTokenizeStringComplex(pszBound + 9, "\n ;", TRUE, FALSE);
2914 :
2915 38 : if (CSLCount(papszTokens) == 0 || strlen(papszTokens[0]) == 0)
2916 : {
2917 1 : CPLError(CE_Failure, CPLE_AppDefined,
2918 : "Unable to parse multi-part mime, boundary not parsable.");
2919 1 : CSLDestroy(papszTokens);
2920 1 : return FALSE;
2921 : }
2922 :
2923 37 : osBoundary = "--";
2924 37 : osBoundary += papszTokens[0];
2925 37 : CSLDestroy(papszTokens);
2926 :
2927 : /* -------------------------------------------------------------------- */
2928 : /* Find the start of the first chunk. */
2929 : /* -------------------------------------------------------------------- */
2930 37 : char *pszNext = psResult->pabyData
2931 37 : ? strstr(reinterpret_cast<char *>(psResult->pabyData),
2932 : osBoundary.c_str())
2933 37 : : nullptr;
2934 :
2935 37 : if (pszNext == nullptr)
2936 : {
2937 1 : CPLError(CE_Failure, CPLE_AppDefined, "No parts found.");
2938 1 : return FALSE;
2939 : }
2940 :
2941 36 : pszNext += osBoundary.size();
2942 146 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2943 110 : pszNext++;
2944 36 : if (*pszNext == '\r')
2945 30 : pszNext++;
2946 36 : if (*pszNext == '\n')
2947 36 : pszNext++;
2948 :
2949 : /* -------------------------------------------------------------------- */
2950 : /* Loop over parts... */
2951 : /* -------------------------------------------------------------------- */
2952 : while (true)
2953 : {
2954 57 : psResult->nMimePartCount++;
2955 57 : psResult->pasMimePart = static_cast<CPLMimePart *>(
2956 114 : CPLRealloc(psResult->pasMimePart,
2957 57 : sizeof(CPLMimePart) * psResult->nMimePartCount));
2958 :
2959 57 : CPLMimePart *psPart =
2960 57 : psResult->pasMimePart + psResult->nMimePartCount - 1;
2961 :
2962 57 : memset(psPart, 0, sizeof(CPLMimePart));
2963 :
2964 : /* --------------------------------------------------------------------
2965 : */
2966 : /* Collect headers. */
2967 : /* --------------------------------------------------------------------
2968 : */
2969 145 : while (*pszNext != '\n' && *pszNext != '\r' && *pszNext != '\0')
2970 : {
2971 90 : if (!STARTS_WITH(pszNext, "Content-"))
2972 : {
2973 1 : break;
2974 : }
2975 89 : char *pszEOL = strstr(pszNext, "\n");
2976 :
2977 89 : if (pszEOL == nullptr)
2978 : {
2979 1 : CPLError(CE_Failure, CPLE_AppDefined,
2980 : "Error while parsing multipart content (at line %d)",
2981 : __LINE__);
2982 1 : return FALSE;
2983 : }
2984 :
2985 88 : *pszEOL = '\0';
2986 88 : bool bRestoreAntislashR = false;
2987 88 : if (pszEOL - pszNext > 1 && pszEOL[-1] == '\r')
2988 : {
2989 80 : bRestoreAntislashR = true;
2990 80 : pszEOL[-1] = '\0';
2991 : }
2992 88 : char *pszKey = nullptr;
2993 88 : const char *pszValue = CPLParseNameValue(pszNext, &pszKey);
2994 88 : if (pszKey && pszValue)
2995 : {
2996 88 : psPart->papszHeaders =
2997 88 : CSLSetNameValue(psPart->papszHeaders, pszKey, pszValue);
2998 : }
2999 88 : CPLFree(pszKey);
3000 88 : if (bRestoreAntislashR)
3001 80 : pszEOL[-1] = '\r';
3002 88 : *pszEOL = '\n';
3003 :
3004 88 : pszNext = pszEOL + 1;
3005 : }
3006 :
3007 56 : if (*pszNext == '\r')
3008 54 : pszNext++;
3009 56 : if (*pszNext == '\n')
3010 54 : pszNext++;
3011 :
3012 : /* --------------------------------------------------------------------
3013 : */
3014 : /* Work out the data block size. */
3015 : /* --------------------------------------------------------------------
3016 : */
3017 56 : psPart->pabyData = reinterpret_cast<GByte *>(pszNext);
3018 :
3019 56 : int nBytesAvail = psResult->nDataLen -
3020 56 : static_cast<int>(pszNext - reinterpret_cast<char *>(
3021 56 : psResult->pabyData));
3022 :
3023 6579040 : while (nBytesAvail > 0 &&
3024 3289520 : (*pszNext != '-' ||
3025 109 : strncmp(pszNext, osBoundary, osBoundary.size()) != 0))
3026 : {
3027 3289470 : pszNext++;
3028 3289470 : nBytesAvail--;
3029 : }
3030 :
3031 56 : if (nBytesAvail == 0)
3032 : {
3033 2 : CPLError(CE_Failure, CPLE_AppDefined,
3034 : "Error while parsing multipart content (at line %d)",
3035 : __LINE__);
3036 2 : return FALSE;
3037 : }
3038 :
3039 54 : psPart->nDataLen = static_cast<int>(
3040 54 : pszNext - reinterpret_cast<char *>(psPart->pabyData));
3041 : // Normally the part should end with "\r\n--boundary_marker"
3042 54 : if (psPart->nDataLen >= 2 && pszNext[-2] == '\r' && pszNext[-1] == '\n')
3043 : {
3044 46 : psPart->nDataLen -= 2;
3045 : }
3046 :
3047 54 : pszNext += osBoundary.size();
3048 :
3049 54 : if (STARTS_WITH(pszNext, "--"))
3050 : {
3051 31 : break;
3052 : }
3053 :
3054 23 : if (*pszNext == '\r')
3055 19 : pszNext++;
3056 23 : if (*pszNext == '\n')
3057 21 : pszNext++;
3058 : else
3059 : {
3060 2 : CPLError(CE_Failure, CPLE_AppDefined,
3061 : "Error while parsing multipart content (at line %d)",
3062 : __LINE__);
3063 2 : return FALSE;
3064 : }
3065 21 : }
3066 :
3067 31 : return TRUE;
3068 : }
3069 :
3070 : #if defined(__GNUC__)
3071 : #pragma GCC diagnostic pop
3072 : #endif
|
