Line data    Source code

       1             : /******************************************************************************
       2             :  *
       3             :  * Project:  WMS Client Driver
       4             :  * Purpose:  Implementation of Dataset and RasterBand classes for WMS
       5             :  *           and other similar services.
       6             :  * Author:   Adam Nowacki, nowak@xpam.de
       7             :  *
       8             :  ******************************************************************************
       9             :  * Copyright (c) 2007, Adam Nowacki
      10             :  * Copyright (c) 2007-2013, Even Rouault <even dot rouault at spatialys.com>
      11             :  * Copyright (c) 2016, Lucian Plesea
      12             :  *
      13             :  * SPDX-License-Identifier: MIT
      14             :  ****************************************************************************/
      15             : 
      16             : #include "wmsdriver.h"
      17             : #include <algorithm>
      18             : 
      19          78 : static size_t WriteFunc(void *buffer, size_t count, size_t nmemb, void *req)
      20             : {
      21          78 :     WMSHTTPRequest *psRequest = reinterpret_cast<WMSHTTPRequest *>(req);
      22          78 :     size_t size = count * nmemb;
      23             : 
      24          78 :     if (size == 0)
      25           0 :         return 0;
      26             : 
      27          78 :     const size_t required_size = psRequest->nDataLen + size + 1;
      28          78 :     if (required_size > psRequest->nDataAlloc)
      29             :     {
      30          48 :         size_t new_size = required_size * 2;
      31          48 :         if (new_size < 512)
      32           0 :             new_size = 512;
      33          48 :         psRequest->nDataAlloc = new_size;
      34             :         GByte *pabyNewData = reinterpret_cast<GByte *>(
      35          48 :             VSIRealloc(psRequest->pabyData, new_size));
      36          48 :         if (pabyNewData == nullptr)
      37             :         {
      38           0 :             VSIFree(psRequest->pabyData);
      39           0 :             psRequest->pabyData = nullptr;
      40             :             psRequest->Error.Printf(
      41             :                 "Out of memory allocating %u bytes for HTTP data buffer.",
      42           0 :                 static_cast<unsigned int>(new_size));
      43           0 :             psRequest->nDataAlloc = 0;
      44           0 :             psRequest->nDataLen = 0;
      45           0 :             return 0;
      46             :         }
      47          48 :         psRequest->pabyData = pabyNewData;
      48             :     }
      49          78 :     memcpy(psRequest->pabyData + psRequest->nDataLen, buffer, size);
      50          78 :     psRequest->nDataLen += size;
      51          78 :     psRequest->pabyData[psRequest->nDataLen] = 0;
      52          78 :     return nmemb;
      53             : }
      54             : 
      55             : // Process curl errors
      56          37 : static void ProcessCurlErrors(CURLMsg *msg, WMSHTTPRequest *pasRequest,
      57             :                               int nRequestCount)
      58             : {
      59          37 :     CPLAssert(msg != nullptr);
      60          37 :     CPLAssert(msg->msg == CURLMSG_DONE);
      61             : 
      62             :     // in case of local file error: update status code
      63          37 :     if (msg->data.result == CURLE_FILE_COULDNT_READ_FILE)
      64             :     {
      65             :         // identify current request
      66           1 :         for (int current_req_i = 0; current_req_i < nRequestCount;
      67             :              ++current_req_i)
      68             :         {
      69           1 :             WMSHTTPRequest *const psRequest = &pasRequest[current_req_i];
      70           1 :             if (psRequest->m_curl_handle != msg->easy_handle)
      71           0 :                 continue;
      72             : 
      73             :             // sanity check for local files
      74           1 :             if (STARTS_WITH(psRequest->URL.c_str(), "file://"))
      75             :             {
      76           1 :                 psRequest->nStatus = 404;
      77           1 :                 break;
      78             :             }
      79             :         }
      80             :     }
      81          37 : }
      82             : 
      83             : // Builds a curl request
      84          47 : void WMSHTTPInitializeRequest(WMSHTTPRequest *psRequest)
      85             : {
      86          47 :     psRequest->nStatus = 0;
      87          47 :     psRequest->pabyData = nullptr;
      88          47 :     psRequest->nDataLen = 0;
      89          47 :     psRequest->nDataAlloc = 0;
      90             : 
      91          47 :     psRequest->m_curl_handle = curl_easy_init();
      92          47 :     if (psRequest->m_curl_handle == nullptr)
      93             :     {
      94           0 :         CPLError(CE_Failure, CPLE_AppDefined,
      95             :                  "CPLHTTPInitializeRequest(): Unable to create CURL handle.");
      96           0 :         return;
      97             :     }
      98             : 
      99          47 :     if (!psRequest->Range.empty())
     100             :     {
     101           0 :         CPL_IGNORE_RET_VAL(curl_easy_setopt(
     102             :             psRequest->m_curl_handle, CURLOPT_RANGE, psRequest->Range.c_str()));
     103             :     }
     104             : 
     105          47 :     CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     106             :                                         CURLOPT_WRITEDATA, psRequest));
     107          47 :     CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     108             :                                         CURLOPT_WRITEFUNCTION, WriteFunc));
     109             : 
     110          47 :     psRequest->m_curl_error.resize(CURL_ERROR_SIZE + 1);
     111          47 :     CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     112             :                                         CURLOPT_ERRORBUFFER,
     113             :                                         &psRequest->m_curl_error[0]));
     114             : 
     115          47 :     psRequest->m_headers = static_cast<struct curl_slist *>(CPLHTTPSetOptions(
     116             :         psRequest->m_curl_handle, psRequest->URL.c_str(), psRequest->options));
     117          47 :     if (psRequest->m_headers != nullptr)
     118             :     {
     119           2 :         CPL_IGNORE_RET_VAL(curl_easy_setopt(psRequest->m_curl_handle,
     120             :                                             CURLOPT_HTTPHEADER,
     121             :                                             psRequest->m_headers));
     122             :     }
     123             : }
     124             : 
     125          59 : WMSHTTPRequest::~WMSHTTPRequest()
     126             : {
     127          59 :     if (m_curl_handle != nullptr)
     128          47 :         curl_easy_cleanup(m_curl_handle);
     129          59 :     if (m_headers != nullptr)
     130           2 :         curl_slist_free_all(m_headers);
     131          59 :     if (pabyData != nullptr)
     132          41 :         CPLFree(pabyData);
     133          59 : }
     134             : 
     135             : //
     136             : // Like CPLHTTPFetch, but multiple requests in parallel
     137             : // By default it uses 5 connections
     138             : //
     139          39 : CPLErr WMSHTTPFetchMulti(WMSHTTPRequest *pasRequest, int nRequestCount)
     140             : {
     141          39 :     CPLErr ret = CE_None;
     142          39 :     CURLM *curl_multi = nullptr;
     143             :     int max_conn;
     144             :     int i, conn_i;
     145             : 
     146          39 :     CPLAssert(nRequestCount >= 0);
     147          39 :     if (nRequestCount == 0)
     148           5 :         return CE_None;
     149             : 
     150             :     const char *max_conn_opt =
     151          34 :         CSLFetchNameValue(const_cast<char **>(pasRequest->options), "MAXCONN");
     152          34 :     max_conn =
     153          34 :         (max_conn_opt == nullptr) ? 5 : MAX(1, MIN(atoi(max_conn_opt), 1000));
     154             : 
     155             :     // If the first url starts with vsimem, assume all do and defer to
     156             :     // CPLHTTPFetch
     157          44 :     if (STARTS_WITH(pasRequest[0].URL.c_str(), "/vsimem/") &&
     158             :         /* Disabled by default for potential security issues */
     159          10 :         CPLTestBool(CPLGetConfigOption("CPL_CURL_ENABLE_VSIMEM", "FALSE")))
     160             :     {
     161          20 :         for (i = 0; i < nRequestCount; i++)
     162             :         {
     163             :             CPLHTTPResult *psResult =
     164          10 :                 CPLHTTPFetch(pasRequest[i].URL.c_str(),
     165          10 :                              const_cast<char **>(pasRequest[i].options));
     166          10 :             pasRequest[i].pabyData = psResult->pabyData;
     167          10 :             pasRequest[i].nDataLen = psResult->nDataLen;
     168          10 :             pasRequest[i].Error =
     169          10 :                 psResult->pszErrBuf ? psResult->pszErrBuf : "";
     170             :             // Conventions are different between this module and cpl_http...
     171          10 :             if (psResult->pszErrBuf != nullptr &&
     172           1 :                 strcmp(psResult->pszErrBuf, "HTTP error code : 404") == 0)
     173           1 :                 pasRequest[i].nStatus = 404;
     174             :             else
     175           9 :                 pasRequest[i].nStatus = 200;
     176          10 :             pasRequest[i].ContentType =
     177          10 :                 psResult->pszContentType ? psResult->pszContentType : "";
     178             :             // took ownership of content, we're done with the rest
     179          10 :             psResult->pabyData = nullptr;
     180          10 :             psResult->nDataLen = 0;
     181          10 :             CPLHTTPDestroyResult(psResult);
     182             :         }
     183          10 :         return CE_None;
     184             :     }
     185             : 
     186          24 :     curl_multi = curl_multi_init();
     187          24 :     if (curl_multi == nullptr)
     188             :     {
     189           0 :         CPLError(CE_Fatal, CPLE_AppDefined,
     190             :                  "CPLHTTPFetchMulti(): Unable to create CURL multi-handle.");
     191             :     }
     192             : 
     193             :     // add at most max_conn requests
     194          24 :     int torun = std::min(nRequestCount, max_conn);
     195          57 :     for (conn_i = 0; conn_i < torun; ++conn_i)
     196             :     {
     197          33 :         WMSHTTPRequest *const psRequest = &pasRequest[conn_i];
     198          33 :         CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1, nRequestCount,
     199          33 :                  pasRequest[conn_i].URL.c_str());
     200          33 :         curl_multi_add_handle(curl_multi, psRequest->m_curl_handle);
     201             :     }
     202             : 
     203          24 :     void *old_handler = CPLHTTPIgnoreSigPipe();
     204             :     int still_running;
     205         638 :     do
     206             :     {
     207             :         CURLMcode mc;
     208           0 :         do
     209             :         {
     210         662 :             mc = curl_multi_perform(curl_multi, &still_running);
     211         662 :         } while (CURLM_CALL_MULTI_PERFORM == mc);
     212             : 
     213             :         // Pick up messages, clean up the completed ones, add more
     214         662 :         int msgs_in_queue = 0;
     215           2 :         do
     216             :         {
     217         664 :             CURLMsg *m = curl_multi_info_read(curl_multi, &msgs_in_queue);
     218         664 :             if (m && (m->msg == CURLMSG_DONE))
     219             :             {
     220          37 :                 ProcessCurlErrors(m, pasRequest, nRequestCount);
     221             : 
     222          37 :                 curl_multi_remove_handle(curl_multi, m->easy_handle);
     223          37 :                 if (conn_i < nRequestCount)
     224             :                 {
     225           4 :                     auto psRequest = &pasRequest[conn_i];
     226           4 :                     CPLDebug("HTTP", "Requesting [%d/%d] %s", conn_i + 1,
     227           4 :                              nRequestCount, pasRequest[conn_i].URL.c_str());
     228           4 :                     curl_multi_add_handle(curl_multi, psRequest->m_curl_handle);
     229           4 :                     ++conn_i;
     230           4 :                     still_running = 1;  // Still have request pending
     231             :                 }
     232             :             }
     233         664 :         } while (msgs_in_queue);
     234             : 
     235         662 :         if (CURLM_OK == mc)
     236             :         {
     237             :             int numfds;
     238         662 :             curl_multi_wait(curl_multi, nullptr, 0, 100, &numfds);
     239             :         }
     240         662 :     } while (still_running || conn_i != nRequestCount);
     241             : 
     242             :     // process any message still in queue
     243             :     CURLMsg *msg;
     244             :     int msgs_in_queue;
     245           0 :     do
     246             :     {
     247          24 :         msg = curl_multi_info_read(curl_multi, &msgs_in_queue);
     248          24 :         if (msg != nullptr)
     249             :         {
     250           0 :             if (msg->msg == CURLMSG_DONE)
     251             :             {
     252           0 :                 ProcessCurlErrors(msg, pasRequest, nRequestCount);
     253             :             }
     254             :         }
     255          24 :     } while (msg != nullptr);
     256             : 
     257          24 :     CPLHTTPRestoreSigPipeHandler(old_handler);
     258             : 
     259          24 :     if (conn_i != nRequestCount)
     260             :     {  // something gone really really wrong
     261             :         // oddly built libcurl or perhaps absence of network interface
     262           0 :         CPLError(CE_Failure, CPLE_AppDefined,
     263             :                  "WMSHTTPFetchMulti(): conn_i != nRequestCount, this should "
     264             :                  "never happen ...");
     265           0 :         nRequestCount = conn_i;
     266           0 :         ret = CE_Failure;
     267             :     }
     268             : 
     269          61 :     for (i = 0; i < nRequestCount; ++i)
     270             :     {
     271          37 :         WMSHTTPRequest *const psRequest = &pasRequest[i];
     272             : 
     273             :         long response_code;
     274          37 :         curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_RESPONSE_CODE,
     275             :                           &response_code);
     276             :         // for local files, don't update the status code if one is already set
     277          38 :         if (!(psRequest->nStatus != 0 &&
     278           1 :               STARTS_WITH(psRequest->URL.c_str(), "file://")))
     279          36 :             psRequest->nStatus = static_cast<int>(response_code);
     280             : 
     281          37 :         char *content_type = nullptr;
     282          37 :         curl_easy_getinfo(psRequest->m_curl_handle, CURLINFO_CONTENT_TYPE,
     283             :                           &content_type);
     284          37 :         psRequest->ContentType = content_type ? content_type : "";
     285             : 
     286          37 :         if (psRequest->Error.empty())
     287          37 :             psRequest->Error = &psRequest->m_curl_error[0];
     288             : 
     289             :         /* In the case of a file:// URL, curl will return a status == 0, so if
     290             :          * there's no */
     291             :         /* error returned, patch the status code to be 200, as it would be for
     292             :          * http:// */
     293          37 :         if (psRequest->nStatus == 0 && psRequest->Error.empty() &&
     294           0 :             STARTS_WITH(psRequest->URL.c_str(), "file://"))
     295           0 :             psRequest->nStatus = 200;
     296             : 
     297             :         // If there is an error with no error message, use the content if it is
     298             :         // text
     299          72 :         if (psRequest->Error.empty() && psRequest->nStatus != 0 &&
     300          35 :             psRequest->nStatus != 200 &&
     301          72 :             strstr(psRequest->ContentType, "text") &&
     302           0 :             psRequest->pabyData != nullptr)
     303             :             psRequest->Error =
     304           0 :                 reinterpret_cast<const char *>(psRequest->pabyData);
     305             : 
     306          72 :         CPLDebug(
     307             :             "HTTP", "Request [%d] %s : status = %d, type = %s, error = %s", i,
     308             :             psRequest->URL.c_str(), psRequest->nStatus,
     309          37 :             !psRequest->ContentType.empty() ? psRequest->ContentType.c_str()
     310             :                                             : "(null)",
     311          37 :             !psRequest->Error.empty() ? psRequest->Error.c_str() : "(null)");
     312             : 
     313          37 :         curl_multi_remove_handle(curl_multi, pasRequest->m_curl_handle);
     314             :     }
     315             : 
     316          24 :     curl_multi_cleanup(curl_multi);
     317             : 
     318          24 :     return ret;
     319             : }