Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  WMS Client Driver
       4             :  * Purpose:  Implementation of Dataset and RasterBand classes for WMS
       5             :  *           and other similar services.
       6             :  * Author:   Adam Nowacki, nowak@xpam.de
       7             :  *
       8             :  ******************************************************************************
       9             :  * Copyright (c) 2007, Adam Nowacki
      10             :  * Copyright (c) 2007-2013, Even Rouault <even dot rouault at spatialys.com>
      11             :  * Copyright (c) 2016, Lucian Plesea
      12             :  *
      13             :  * SPDX-License-Identifier: MIT
      14             :  ****************************************************************************/
      15             : 
      16             : #include "wmsdriver.h"
      17             : #include <algorithm>
      18             : 
      19          78 : static size_t WriteFunc(void *buffer, size_t count, size_t nmemb, void *req)
      20             : {
      21          78 :     WMSHTTPRequest *psRequest = reinterpret_cast<WMSHTTPRequest *>(req);
      22          78 :     size_t size = count * nmemb;
      23             : 
      24          78 :     if (size == 0)
      25           0 :         return 0;
      26             : 
      27          78 :     const size_t required_size = psRequest->nDataLen + size + 1;
      28          78 :     if (required_size > psRequest->nDataAlloc)
      29             :     {
      30          47 :         size_t new_size = required_size * 2;
      31          47 :         if (new_size < 512)
      32           0 :             new_size = 512;
      33          47 :         psRequest->nDataAlloc = new_size;
      34             :         GByte *pabyNewData = reinterpret_cast<GByte *>(
      35          47 :             VSIRealloc(psRequest->pabyData, new_size));
      36          47 :         if (pabyNewData == nullptr)
      37             :         {
      38           0 :             VSIFree(psRequest->pabyData);
      39           0 :             psRequest->pabyData = nullptr;
      40             :             psRequest->Error.Printf(
      41             :                 "Out of memory allocating %u bytes for HTTP data buffer.",
      42           0 :                 static_cast<unsigned int>(new_size));
      43           0 :             psRequest->nDataAlloc = 0;
      44           0 :             psRequest->nDataLen = 0;
      45           0 :             return 0;
      46             :         }
      47          47 :         psRequest->pabyData = pabyNewData;
      48             :     }
      49          78 :     memcpy(psRequest->pabyData + psRequest->nDataLen, buffer, size);
      50          78 :     psRequest->nDataLen += size;
      51          78 :     psRequest->pabyData[psRequest->nDataLen] = 0;
      52          78 :     return nmemb;
      53             : }
      54             : 
      55             : // Process curl errors
      56          37 : static void ProcessCurlErrors(CURLMsg *msg, WMSHTTPRequest *pasRequest,
      57             :                               int nRequestCount)
      58             : {
      59          37 :     CPLAssert(msg != nullptr);
      60          37 :     CPLAssert(msg->msg == CURLMSG_DONE);
      61             : 
      62             :     // in case of local file error: update status code
      63          37 :     if (msg->data.result == CURLE_FILE_COULDNT_READ_FILE)
      64             :     {
      65             :         // identify current request
      66           1 :         for (int current_req_i = 0; current_req_i < nRequestCount;
      67             :              ++current_req_i)
      68             :         {
      69           1 :             WMSHTTPRequest *const psRequest = &pasRequest[current_req_i];
      70           1 :             if (psRequest->m_curl_handle != msg->easy_handle)
      71           0 :                 continue;
      72             : 
      73             :             // sanity check for local files
      74           1 :             if (STARTS_WITH(psRequest->URL.c_str(), "file://"))
      75             :             {
      76           1 :                 psRequest->nStatus = 404;
      77           1 :                 break;
      78             :             }
      79             :         }
      80             :     }
      81          37 : }
      82             : 
      83             : // Builds a curl request
      84          47 : void WMSHTTPInitializeRequest(WMSHTTPRequest *psRequest)
      85             : {
      86          47 :     psRequest->nStatus = 0;
      87          47 :     psRequest->pabyData = nullptr;
      88          47 :     psRequest->nDataLen = 0;
      89          47 :     psRequest->nDataAlloc = 0;
      90             : 
      91          47 :     psRequest->m_curl_handle = curl_easy_init();
      92          47 :     if (psRequest->m_curl_handle == nullptr)
      93             :     {
      94           0 :         CPLError(CE_Failure, CPLE_AppDefined,
      95             :                  "CPLHTTPInitializeRequest(): Unable to create CURL handle.");
      96           0 :         return;
      97             :     }
      98             : 
      99          47 :     if (!psRequest->Range.empty())
     100             :     {
     101           0 :         CPL_IGNORE_RET_VAL(curl_easy_setopt(
     102             :             psRequest->m_curl_handle, CURLOPT_RANGE, psRequest->Range.c_str()));
     103             :     }
     104             : 
     105          47 :     CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     106             :                                         CURLOPT_WRITEDATA, psRequest));
     107          47 :     CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     108             :                                         CURLOPT_WRITEFUNCTION, WriteFunc));
     109             : 
     110          47 :     psRequest->m_curl_error.resize(CURL_ERROR_SIZE + 1);
     111          47 :     CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     112             :                                         CURLOPT_ERRORBUFFER,
     113             :                                         &psRequest->m_curl_error[0]));
     114             : 
     115          47 :     psRequest->m_headers = static_cast<struct curl_slist *>(CPLHTTPSetOptions(
     116          94 :         psRequest->m_curl_handle, psRequest->URL.URLEncode().c_str(),
     117             :         psRequest->options));
     118          47 :     if (psRequest->m_headers != nullptr)
     119             :     {
     120           2 :         CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     121             :                                             CURLOPT_HTTPHEADER,
     122             :                                             psRequest->m_headers));
     123             :     }
     124             : }
     125             : 
     126          59 : WMSHTTPRequest::~WMSHTTPRequest()
     127             : {
     128          59 :     if (m_curl_handle != nullptr)
     129          47 :         curl_easy_cleanup(m_curl_handle);
     130          59 :     if (m_headers != nullptr)
     131           2 :         curl_slist_free_all(m_headers);
     132          59 :     if (pabyData != nullptr)
     133          41 :         CPLFree(pabyData);
     134          59 : }
     135             : 
     136             : //
     137             : // Like CPLHTTPFetch, but multiple requests in parallel
     138             : // By default it uses 5 connections
     139             : //
     140          39 : CPLErr WMSHTTPFetchMulti(WMSHTTPRequest *pasRequest, int nRequestCount)
     141             : {
     142          39 :     CPLErr ret = CE_None;
     143          39 :     CURLM *curl_multi = nullptr;
     144             :     int max_conn;
     145             :     int i, conn_i;
     146             : 
     147          39 :     CPLAssert(nRequestCount >= 0);
     148          39 :     if (nRequestCount == 0)
     149           5 :         return CE_None;
     150             : 
     151             :     const char *max_conn_opt =
     152          34 :         CSLFetchNameValue(const_cast<char **>(pasRequest->options), "MAXCONN");
     153          34 :     max_conn =
     154          34 :         (max_conn_opt == nullptr) ? 5 : MAX(1, MIN(atoi(max_conn_opt), 1000));
     155             : 
     156             :     // If the first url starts with vsimem, assume all do and defer to
     157             :     // CPLHTTPFetch
     158          44 :     if (STARTS_WITH(pasRequest[0].URL.c_str(), "/vsimem/") &&
     159             :         /* Disabled by default for potential security issues */
     160          10 :         CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")))
     161             :     {
     162          20 :         for (i = 0; i < nRequestCount; i++)
     163             :         {
     164             :             CPLHTTPResult *psResult =
     165          10 :                 CPLHTTPFetch(pasRequest[i].URL.c_str(),
     166          10 :                              const_cast<char **>(pasRequest[i].options));
     167          10 :             pasRequest[i].pabyData = psResult->pabyData;
     168          10 :             pasRequest[i].nDataLen = psResult->nDataLen;
     169          10 :             pasRequest[i].Error =
     170          10 :                 psResult->pszErrBuf ? psResult->pszErrBuf : "";
     171             :             // Conventions are different between this module and cpl_http...
     172          10 :             if (psResult->pszErrBuf != nullptr &&
     173           1 :                 strcmp(psResult->pszErrBuf, "HTTP error code : 404") == 0)
     174           1 :                 pasRequest[i].nStatus = 404;
     175             :             else
     176           9 :                 pasRequest[i].nStatus = 200;
     177          10 :             pasRequest[i].ContentType =
     178          10 :                 psResult->pszContentType ? psResult->pszContentType : "";
     179             :             // took ownership of content, we're done with the rest
     180          10 :             psResult->pabyData = nullptr;
     181          10 :             psResult->nDataLen = 0;
     182          10 :             CPLHTTPDestroyResult(psResult);
     183             :         }
     184          10 :         return CE_None;
     185             :     }
     186             : 
     187          24 :     curl_multi = curl_multi_init();
     188          24 :     if (curl_multi == nullptr)
     189             :     {
     190           0 :         CPLError(CE_Fatal, CPLE_AppDefined,
     191             :                  "CPLHTTPFetchMulti(): Unable to create CURL multi-handle.");
     192             :     }
     193             : 
     194             :     // add at most max_conn requests
     195          24 :     int torun = std::min(nRequestCount, max_conn);
     196          57 :     for (conn_i = 0; conn_i < torun; ++conn_i)
     197             :     {
     198          33 :         WMSHTTPRequest *const psRequest = &pasRequest[conn_i];
     199          33 :         CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1, nRequestCount,
     200          33 :                  pasRequest[conn_i].URL.c_str());
     201          33 :         curl_multi_add_handle(curl_multi, psRequest->m_curl_handle);
     202             :     }
     203             : 
     204          24 :     void *old_handler = CPLHTTPIgnoreSigPipe();
     205             :     int still_running;
     206         445 :     do
     207             :     {
     208             :         CURLMcode mc;
     209           0 :         do
     210             :         {
     211         469 :             mc = curl_multi_perform(curl_multi, &still_running);
     212         469 :         } while (CURLM_CALL_MULTI_PERFORM == mc);
     213             : 
     214             :         // Pick up messages, clean up the completed ones, add more
     215         469 :         int msgs_in_queue = 0;
     216           0 :         do
     217             :         {
     218         469 :             CURLMsg *m = curl_multi_info_read(curl_multi, &msgs_in_queue);
     219         469 :             if (m && (m->msg == CURLMSG_DONE))
     220             :             {
     221          37 :                 ProcessCurlErrors(m, pasRequest, nRequestCount);
     222             : 
     223          37 :                 curl_multi_remove_handle(curl_multi, m->easy_handle);
     224          37 :                 if (conn_i < nRequestCount)
     225             :                 {
     226           4 :                     auto psRequest = &pasRequest[conn_i];
     227           4 :                     CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1,
     228           4 :                              nRequestCount, pasRequest[conn_i].URL.c_str());
     229           4 :                     curl_multi_add_handle(curl_multi, psRequest->m_curl_handle);
     230           4 :                     ++conn_i;
     231           4 :                     still_running = 1;  // Still have request pending
     232             :                 }
     233             :             }
     234         469 :         } while (msgs_in_queue);
     235             : 
     236         469 :         if (CURLM_OK == mc)
     237             :         {
     238             :             int numfds;
     239         469 :             curl_multi_wait(curl_multi, nullptr, 0, 100, &numfds);
     240             :         }
     241         469 :     } while (still_running || conn_i != nRequestCount);
     242             : 
     243             :     // process any message still in queue
     244             :     CURLMsg *msg;
     245             :     int msgs_in_queue;
     246           0 :     do
     247             :     {
     248          24 :         msg = curl_multi_info_read(curl_multi, &msgs_in_queue);
     249          24 :         if (msg != nullptr)
     250             :         {
     251           0 :             if (msg->msg == CURLMSG_DONE)
     252             :             {
     253           0 :                 ProcessCurlErrors(msg, pasRequest, nRequestCount);
     254             :             }
     255             :         }
     256          24 :     } while (msg != nullptr);
     257             : 
     258          24 :     CPLHTTPRestoreSigPipeHandler(old_handler);
     259             : 
     260          24 :     if (conn_i != nRequestCount)
     261             :     {  // something gone really really wrong
     262             :         // oddly built libcurl or perhaps absence of network interface
     263           0 :         CPLError(CE_Failure, CPLE_AppDefined,
     264             :                  "WMSHTTPFetchMulti(): conn_i != nRequestCount, this should "
     265             :                  "never happen ...");
     266           0 :         nRequestCount = conn_i;
     267           0 :         ret = CE_Failure;
     268             :     }
     269             : 
     270          61 :     for (i = 0; i < nRequestCount; ++i)
     271             :     {
     272          37 :         WMSHTTPRequest *const psRequest = &pasRequest[i];
     273             : 
     274             :         long response_code;
     275          37 :         curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_RESPONSE_CODE,
     276             :                           &response_code);
     277             :         // for local files, don't update the status code if one is already set
     278          38 :         if (!(psRequest->nStatus != 0 &&
     279           1 :               STARTS_WITH(psRequest->URL.c_str(), "file://")))
     280          36 :             psRequest->nStatus = static_cast<int>(response_code);
     281             : 
     282          37 :         char *content_type = nullptr;
     283          37 :         curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_CONTENT_TYPE,
     284             :                           &content_type);
     285          37 :         psRequest->ContentType = content_type ? content_type : "";
     286             : 
     287          37 :         if (psRequest->Error.empty())
     288          37 :             psRequest->Error = &psRequest->m_curl_error[0];
     289             : 
     290             :         /* In the case of a file:// URL, curl will return a status == 0, so if
     291             :          * there's no */
     292             :         /* error returned, patch the status code to be 200, as it would be for
     293             :          * http:// */
     294          37 :         if (psRequest->nStatus == 0 && psRequest->Error.empty() &&
     295           0 :             STARTS_WITH(psRequest->URL.c_str(), "file://"))
     296           0 :             psRequest->nStatus = 200;
     297             : 
     298             :         // If there is an error with no error message, use the content if it is
     299             :         // text
     300          72 :         if (psRequest->Error.empty() && psRequest->nStatus != 0 &&
     301          35 :             psRequest->nStatus != 200 &&
     302          72 :             strstr(psRequest->ContentType, "text") &&
     303           0 :             psRequest->pabyData != nullptr)
     304             :             psRequest->Error =
     305           0 :                 reinterpret_cast<const char *>(psRequest->pabyData);
     306             : 
     307          72 :         CPLDebug(
     308             :             "HTTP", "Request [%d] %s : status = %d, type = %s, error = %s", i,
     309             :             psRequest->URL.c_str(), psRequest->nStatus,
     310          37 :             !psRequest->ContentType.empty() ? psRequest->ContentType.c_str()
     311             :                                             : "(null)",
     312          37 :             !psRequest->Error.empty() ? psRequest->Error.c_str() : "(null)");
     313             : 
     314          37 :         curl_multi_remove_handle(curl_multi, pasRequest->m_curl_handle);
     315             :     }
     316             : 
     317          24 :     curl_multi_cleanup(curl_multi);
     318             : 
     319          24 :     return ret;
     320             : }