Line data Source code
1 : /**********************************************************************
2 : * Project: CPL - Common Portability Library
3 : * Purpose: Microsoft Azure Storage Blob routines
4 : * Author: Even Rouault <even.rouault at spatialys.com>
5 : *
6 : **********************************************************************
7 : * Copyright (c) 2017, Even Rouault <even.rouault at spatialys.com>
8 : *
9 : * Permission is hereby granted, free of charge, to any person obtaining a
10 : * copy of this software and associated documentation files (the "Software"),
11 : * to deal in the Software without restriction, including without limitation
12 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 : * and/or sell copies of the Software, and to permit persons to whom the
14 : * Software is furnished to do so, subject to the following conditions:
15 : *
16 : * The above copyright notice and this permission notice shall be included
17 : * in all copies or substantial portions of the Software.
18 : *
19 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
22 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
24 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25 : * DEALINAzureBlob IN THE SOFTWARE.
26 : ****************************************************************************/
27 :
28 : #include "cpl_azure.h"
29 : #include "cpl_json.h"
30 : #include "cpl_vsi_error.h"
31 : #include "cpl_sha256.h"
32 : #include "cpl_time.h"
33 : #include "cpl_http.h"
34 : #include "cpl_multiproc.h"
35 : #include "cpl_vsi_virtual.h"
36 :
37 : #include <mutex>
38 :
39 : //! @cond Doxygen_Suppress
40 :
41 : #ifdef HAVE_CURL
42 :
43 : /************************************************************************/
44 : /* RemoveTrailingSlash() */
45 : /************************************************************************/
46 :
47 569 : static std::string RemoveTrailingSlash(const std::string &osStr)
48 : {
49 569 : std::string osRet(osStr);
50 569 : if (!osRet.empty() && osRet.back() == '/')
51 1 : osRet.pop_back();
52 569 : return osRet;
53 : }
54 :
55 : /************************************************************************/
56 : /* CPLAzureGetSignature() */
57 : /************************************************************************/
58 :
59 210 : static std::string CPLAzureGetSignature(const std::string &osStringToSign,
60 : const std::string &osStorageKeyB64)
61 : {
62 :
63 : /* -------------------------------------------------------------------- */
64 : /* Compute signature. */
65 : /* -------------------------------------------------------------------- */
66 :
67 420 : std::string osStorageKeyUnbase64(osStorageKeyB64);
68 420 : int nB64Length = CPLBase64DecodeInPlace(
69 210 : reinterpret_cast<GByte *>(&osStorageKeyUnbase64[0]));
70 210 : osStorageKeyUnbase64.resize(nB64Length);
71 : #ifdef DEBUG_VERBOSE
72 : CPLDebug("AZURE", "signing key size: %d", nB64Length);
73 : #endif
74 :
75 210 : GByte abySignature[CPL_SHA256_HASH_SIZE] = {};
76 420 : CPL_HMAC_SHA256(osStorageKeyUnbase64.c_str(), nB64Length,
77 210 : osStringToSign.c_str(), osStringToSign.size(),
78 : abySignature);
79 :
80 210 : char *pszB64Signature = CPLBase64Encode(CPL_SHA256_HASH_SIZE, abySignature);
81 210 : std::string osSignature(pszB64Signature);
82 210 : CPLFree(pszB64Signature);
83 420 : return osSignature;
84 : }
85 :
86 : /************************************************************************/
87 : /* GetAzureBlobHeaders() */
88 : /************************************************************************/
89 :
90 214 : static struct curl_slist *GetAzureBlobHeaders(
91 : const std::string &osVerb, const struct curl_slist *psExistingHeaders,
92 : const std::string &osResource,
93 : const std::map<std::string, std::string> &oMapQueryParameters,
94 : const std::string &osStorageAccount, const std::string &osStorageKeyB64,
95 : bool bIncludeMSVersion)
96 : {
97 : /* See
98 : * https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
99 : */
100 :
101 428 : std::string osDate = CPLGetConfigOption("CPL_AZURE_TIMESTAMP", "");
102 214 : if (osDate.empty())
103 : {
104 7 : osDate = IVSIS3LikeHandleHelper::GetRFC822DateTime();
105 : }
106 214 : if (osStorageKeyB64.empty())
107 : {
108 8 : struct curl_slist *headers = nullptr;
109 8 : headers = curl_slist_append(
110 : headers, CPLSPrintf("x-ms-date: %s", osDate.c_str()));
111 8 : return headers;
112 : }
113 :
114 412 : std::string osMsVersion("2019-12-12");
115 412 : std::map<std::string, std::string> oSortedMapMSHeaders;
116 206 : if (bIncludeMSVersion)
117 199 : oSortedMapMSHeaders["x-ms-version"] = osMsVersion;
118 206 : oSortedMapMSHeaders["x-ms-date"] = osDate;
119 : std::string osCanonicalizedHeaders(
120 : IVSIS3LikeHandleHelper::BuildCanonicalizedHeaders(
121 412 : oSortedMapMSHeaders, psExistingHeaders, "x-ms-"));
122 :
123 412 : std::string osCanonicalizedResource;
124 206 : osCanonicalizedResource += "/" + osStorageAccount;
125 206 : osCanonicalizedResource += osResource;
126 :
127 : // We assume query parameters are in lower case and they are not repeated
128 : std::map<std::string, std::string>::const_iterator oIter =
129 206 : oMapQueryParameters.begin();
130 468 : for (; oIter != oMapQueryParameters.end(); ++oIter)
131 : {
132 262 : osCanonicalizedResource += "\n";
133 262 : osCanonicalizedResource += oIter->first;
134 262 : osCanonicalizedResource += ":";
135 262 : osCanonicalizedResource += oIter->second;
136 : }
137 :
138 412 : std::string osStringToSign;
139 206 : osStringToSign += osVerb + "\n";
140 : osStringToSign +=
141 206 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-Encoding") + "\n";
142 : osStringToSign +=
143 206 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-Language") + "\n";
144 : std::string osContentLength(
145 412 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-Length"));
146 206 : if (osContentLength == "0")
147 38 : osContentLength.clear(); // since x-ms-version 2015-02-21
148 206 : osStringToSign += osContentLength + "\n";
149 : osStringToSign +=
150 206 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-MD5") + "\n";
151 : osStringToSign +=
152 206 : CPLAWSGetHeaderVal(psExistingHeaders, "Content-Type") + "\n";
153 206 : osStringToSign += CPLAWSGetHeaderVal(psExistingHeaders, "Date") + "\n";
154 : osStringToSign +=
155 206 : CPLAWSGetHeaderVal(psExistingHeaders, "If-Modified-Since") + "\n";
156 206 : osStringToSign += CPLAWSGetHeaderVal(psExistingHeaders, "If-Match") + "\n";
157 : osStringToSign +=
158 206 : CPLAWSGetHeaderVal(psExistingHeaders, "If-None-Match") + "\n";
159 : osStringToSign +=
160 206 : CPLAWSGetHeaderVal(psExistingHeaders, "If-Unmodified-Since") + "\n";
161 206 : osStringToSign += CPLAWSGetHeaderVal(psExistingHeaders, "Range") + "\n";
162 206 : osStringToSign += osCanonicalizedHeaders;
163 206 : osStringToSign += osCanonicalizedResource;
164 :
165 : #ifdef DEBUG_VERBOSE
166 : CPLDebug("AZURE", "osStringToSign = '%s'", osStringToSign.c_str());
167 : #endif
168 :
169 : /* -------------------------------------------------------------------- */
170 : /* Compute signature. */
171 : /* -------------------------------------------------------------------- */
172 :
173 : std::string osAuthorization(
174 412 : "SharedKey " + osStorageAccount + ":" +
175 412 : CPLAzureGetSignature(osStringToSign, osStorageKeyB64));
176 :
177 206 : struct curl_slist *headers = nullptr;
178 : headers =
179 206 : curl_slist_append(headers, CPLSPrintf("x-ms-date: %s", osDate.c_str()));
180 206 : if (bIncludeMSVersion)
181 : {
182 199 : headers = curl_slist_append(
183 : headers, CPLSPrintf("x-ms-version: %s", osMsVersion.c_str()));
184 : }
185 206 : headers = curl_slist_append(
186 : headers, CPLSPrintf("Authorization: %s", osAuthorization.c_str()));
187 206 : return headers;
188 : }
189 :
190 : /************************************************************************/
191 : /* VSIAzureBlobHandleHelper() */
192 : /************************************************************************/
193 295 : VSIAzureBlobHandleHelper::VSIAzureBlobHandleHelper(
194 : const std::string &osPathForOption, const std::string &osEndpoint,
195 : const std::string &osBucket, const std::string &osObjectKey,
196 : const std::string &osStorageAccount, const std::string &osStorageKey,
197 : const std::string &osSAS, const std::string &osAccessToken,
198 295 : bool bFromManagedIdentities)
199 : : m_osPathForOption(osPathForOption),
200 : m_osURL(BuildURL(osEndpoint, osBucket, osObjectKey, osSAS)),
201 : m_osEndpoint(osEndpoint), m_osBucket(osBucket),
202 : m_osObjectKey(osObjectKey), m_osStorageAccount(osStorageAccount),
203 : m_osStorageKey(osStorageKey), m_osSAS(osSAS),
204 : m_osAccessToken(osAccessToken),
205 295 : m_bFromManagedIdentities(bFromManagedIdentities)
206 : {
207 295 : }
208 :
209 : /************************************************************************/
210 : /* ~VSIAzureBlobHandleHelper() */
211 : /************************************************************************/
212 :
213 590 : VSIAzureBlobHandleHelper::~VSIAzureBlobHandleHelper()
214 : {
215 590 : }
216 :
217 : /************************************************************************/
218 : /* AzureCSGetParameter() */
219 : /************************************************************************/
220 :
221 1067 : static std::string AzureCSGetParameter(const std::string &osStr,
222 : const char *pszKey, bool bErrorIfMissing)
223 : {
224 3201 : std::string osKey(pszKey + std::string("="));
225 1067 : size_t nPos = osStr.find(osKey);
226 1067 : if (nPos == std::string::npos)
227 : {
228 : const char *pszMsg =
229 11 : CPLSPrintf("%s missing in AZURE_STORAGE_CONNECTION_STRING", pszKey);
230 11 : if (bErrorIfMissing)
231 : {
232 0 : CPLDebug("AZURE", "%s", pszMsg);
233 0 : VSIError(VSIE_AWSInvalidCredentials, "%s", pszMsg);
234 : }
235 11 : return std::string();
236 : }
237 1056 : size_t nPos2 = osStr.find(";", nPos);
238 1056 : return osStr.substr(nPos + osKey.size(), nPos2 == std::string::npos
239 : ? nPos2
240 2112 : : nPos2 - nPos - osKey.size());
241 : }
242 :
243 : /************************************************************************/
244 : /* CPLAzureCachedToken */
245 : /************************************************************************/
246 :
247 : std::mutex gMutex;
248 :
249 : struct CPLAzureCachedToken
250 : {
251 : std::string osAccessToken{};
252 : GIntBig nExpiresOn = 0;
253 : };
254 :
255 : static std::map<std::string, CPLAzureCachedToken> goMapIMDSURLToCachedToken;
256 :
257 : /************************************************************************/
258 : /* GetConfigurationFromIMDSCredentials() */
259 : /************************************************************************/
260 :
261 : static bool
262 13 : GetConfigurationFromIMDSCredentials(const std::string &osPathForOption,
263 : std::string &osAccessToken)
264 : {
265 : // coverity[tainted_data]
266 : const std::string osRootURL(CPLGetConfigOption("CPL_AZURE_VM_API_ROOT_URL",
267 26 : "http://169.254.169.254"));
268 13 : if (osRootURL == "disabled")
269 1 : return false;
270 :
271 : std::string osURLResource("/metadata/identity/oauth2/"
272 : "token?api-version=2018-02-01&resource=https%"
273 24 : "3A%2F%2Fstorage.azure.com%2F");
274 12 : const char *pszObjectId = VSIGetPathSpecificOption(
275 : osPathForOption.c_str(), "AZURE_IMDS_OBJECT_ID", nullptr);
276 12 : if (pszObjectId)
277 6 : osURLResource += "&object_id=" + CPLAWSURLEncode(pszObjectId, false);
278 12 : const char *pszClientId = VSIGetPathSpecificOption(
279 : osPathForOption.c_str(), "AZURE_IMDS_CLIENT_ID", nullptr);
280 12 : if (pszClientId)
281 6 : osURLResource += "&client_id=" + CPLAWSURLEncode(pszClientId, false);
282 12 : const char *pszMsiResId = VSIGetPathSpecificOption(
283 : osPathForOption.c_str(), "AZURE_IMDS_MSI_RES_ID", nullptr);
284 12 : if (pszMsiResId)
285 6 : osURLResource += "&msi_res_id=" + CPLAWSURLEncode(pszMsiResId, false);
286 :
287 24 : std::lock_guard<std::mutex> guard(gMutex);
288 :
289 : // Look for cached token corresponding to this IMDS request URL
290 12 : auto oIter = goMapIMDSURLToCachedToken.find(osURLResource);
291 12 : if (oIter != goMapIMDSURLToCachedToken.end())
292 : {
293 8 : const auto &oCachedToken = oIter->second;
294 : time_t nCurTime;
295 8 : time(&nCurTime);
296 : // Try to reuse credentials if they are still valid, but
297 : // keep one minute of margin...
298 8 : if (nCurTime < oCachedToken.nExpiresOn - 60)
299 : {
300 7 : osAccessToken = oCachedToken.osAccessToken;
301 7 : return true;
302 : }
303 : }
304 :
305 : // Fetch credentials
306 5 : CPLStringList oResponse;
307 5 : const char *const apszOptions[] = {"HEADERS=Metadata: true", nullptr};
308 : CPLHTTPResult *psResult =
309 5 : CPLHTTPFetch((osRootURL + osURLResource).c_str(), apszOptions);
310 5 : if (psResult)
311 : {
312 5 : if (psResult->nStatus == 0 && psResult->pabyData != nullptr)
313 : {
314 : const std::string osJSon =
315 10 : reinterpret_cast<char *>(psResult->pabyData);
316 5 : oResponse = CPLParseKeyValueJson(osJSon.c_str());
317 5 : if (oResponse.FetchNameValue("error"))
318 : {
319 0 : CPLDebug("AZURE",
320 : "Cannot retrieve managed identities credentials: %s",
321 : osJSon.c_str());
322 : }
323 : }
324 5 : CPLHTTPDestroyResult(psResult);
325 : }
326 5 : osAccessToken = oResponse.FetchNameValueDef("access_token", "");
327 : const GIntBig nExpiresOn =
328 5 : CPLAtoGIntBig(oResponse.FetchNameValueDef("expires_on", ""));
329 5 : if (!osAccessToken.empty() && nExpiresOn > 0)
330 : {
331 10 : CPLAzureCachedToken cachedToken;
332 5 : cachedToken.osAccessToken = osAccessToken;
333 5 : cachedToken.nExpiresOn = nExpiresOn;
334 5 : goMapIMDSURLToCachedToken[osURLResource] = std::move(cachedToken);
335 5 : CPLDebug("AZURE", "Storing credentials for %s until " CPL_FRMT_GIB,
336 : osURLResource.c_str(), nExpiresOn);
337 : }
338 :
339 5 : return !osAccessToken.empty();
340 : }
341 :
342 : /************************************************************************/
343 : /* GetConfigurationFromWorkloadIdentity() */
344 : /************************************************************************/
345 :
346 : // Last timestamp AZURE_FEDERATED_TOKEN_FILE was read
347 : static GIntBig gnLastReadFederatedTokenFile = 0;
348 : static std::string gosFederatedToken{};
349 :
350 : // Azure Active Directory Workload Identity, typically for Azure Kubernetes
351 : // Cf https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/identity/azure-identity/azure/identity/_credentials/workload_identity.py
352 21 : static bool GetConfigurationFromWorkloadIdentity(std::string &osAccessToken)
353 : {
354 : const std::string AZURE_CLIENT_ID(
355 42 : CPLGetConfigOption("AZURE_CLIENT_ID", ""));
356 : const std::string AZURE_TENANT_ID(
357 42 : CPLGetConfigOption("AZURE_TENANT_ID", ""));
358 : const std::string AZURE_AUTHORITY_HOST(
359 42 : CPLGetConfigOption("AZURE_AUTHORITY_HOST", ""));
360 : const std::string AZURE_FEDERATED_TOKEN_FILE(
361 42 : CPLGetConfigOption("AZURE_FEDERATED_TOKEN_FILE", ""));
362 37 : if (AZURE_CLIENT_ID.empty() || AZURE_TENANT_ID.empty() ||
363 37 : AZURE_AUTHORITY_HOST.empty() || AZURE_FEDERATED_TOKEN_FILE.empty())
364 : {
365 13 : return false;
366 : }
367 :
368 16 : std::lock_guard<std::mutex> guard(gMutex);
369 :
370 : time_t nCurTime;
371 8 : time(&nCurTime);
372 :
373 : // Look for cached token corresponding to this request URL
374 8 : const std::string osURL(AZURE_AUTHORITY_HOST + AZURE_TENANT_ID +
375 16 : "/oauth2/v2.0/token");
376 8 : auto oIter = goMapIMDSURLToCachedToken.find(osURL);
377 8 : if (oIter != goMapIMDSURLToCachedToken.end())
378 : {
379 6 : const auto &oCachedToken = oIter->second;
380 : // Try to reuse credentials if they are still valid, but
381 : // keep one minute of margin...
382 6 : if (nCurTime < oCachedToken.nExpiresOn - 60)
383 : {
384 4 : osAccessToken = oCachedToken.osAccessToken;
385 4 : return true;
386 : }
387 : }
388 :
389 : // Ingest content of AZURE_FEDERATED_TOKEN_FILE if last time was more than
390 : // 600 seconds.
391 4 : if (nCurTime - gnLastReadFederatedTokenFile > 600)
392 : {
393 : auto fp = VSIVirtualHandleUniquePtr(
394 2 : VSIFOpenL(AZURE_FEDERATED_TOKEN_FILE.c_str(), "rb"));
395 2 : if (!fp)
396 : {
397 0 : CPLDebug("AZURE", "Cannot open AZURE_FEDERATED_TOKEN_FILE = %s",
398 : AZURE_FEDERATED_TOKEN_FILE.c_str());
399 0 : return false;
400 : }
401 2 : fp->Seek(0, SEEK_END);
402 2 : const auto nSize = fp->Tell();
403 2 : if (nSize == 0 || nSize > 100 * 1024)
404 : {
405 0 : CPLDebug(
406 : "AZURE",
407 : "Invalid size for AZURE_FEDERATED_TOKEN_FILE = " CPL_FRMT_GUIB,
408 : static_cast<GUIntBig>(nSize));
409 0 : return false;
410 : }
411 2 : fp->Seek(0, SEEK_SET);
412 2 : gosFederatedToken.resize(static_cast<size_t>(nSize));
413 2 : if (fp->Read(&gosFederatedToken[0], gosFederatedToken.size(), 1) != 1)
414 : {
415 0 : CPLDebug("AZURE", "Cannot read AZURE_FEDERATED_TOKEN_FILE");
416 0 : return false;
417 : }
418 2 : gnLastReadFederatedTokenFile = nCurTime;
419 : }
420 :
421 : /* -------------------------------------------------------------------- */
422 : /* Prepare POST request. */
423 : /* -------------------------------------------------------------------- */
424 8 : CPLStringList aosOptions;
425 :
426 : aosOptions.AddString(
427 4 : "HEADERS=Content-Type: application/x-www-form-urlencoded");
428 :
429 8 : std::string osItem("POSTFIELDS=client_assertion=");
430 4 : osItem += CPLAWSURLEncode(gosFederatedToken);
431 : osItem += "&client_assertion_type=urn:ietf:params:oauth:client-assertion-"
432 4 : "type:jwt-bearer";
433 4 : osItem += "&client_id=";
434 4 : osItem += CPLAWSURLEncode(AZURE_CLIENT_ID);
435 4 : osItem += "&grant_type=client_credentials";
436 4 : osItem += "&scope=https://storage.azure.com/.default";
437 4 : aosOptions.AddString(osItem.c_str());
438 :
439 : /* -------------------------------------------------------------------- */
440 : /* Submit request by HTTP. */
441 : /* -------------------------------------------------------------------- */
442 4 : CPLHTTPResult *psResult = CPLHTTPFetch(osURL.c_str(), aosOptions.List());
443 4 : if (!psResult)
444 0 : return false;
445 :
446 4 : if (!psResult->pabyData || psResult->pszErrBuf)
447 : {
448 0 : if (psResult->pszErrBuf)
449 0 : CPLDebug("AZURE", "%s", psResult->pszErrBuf);
450 0 : if (psResult->pabyData)
451 0 : CPLDebug("AZURE", "%s", psResult->pabyData);
452 :
453 0 : CPLDebug("AZURE",
454 : "Fetching OAuth2 access code from workload identity failed.");
455 0 : CPLHTTPDestroyResult(psResult);
456 0 : return false;
457 : }
458 :
459 : CPLStringList oResponse =
460 4 : CPLParseKeyValueJson(reinterpret_cast<char *>(psResult->pabyData));
461 4 : CPLHTTPDestroyResult(psResult);
462 :
463 4 : osAccessToken = oResponse.FetchNameValueDef("access_token", "");
464 4 : const int nExpiresIn = atoi(oResponse.FetchNameValueDef("expires_in", ""));
465 4 : if (!osAccessToken.empty() && nExpiresIn > 0)
466 : {
467 8 : CPLAzureCachedToken cachedToken;
468 4 : cachedToken.osAccessToken = osAccessToken;
469 4 : cachedToken.nExpiresOn = nCurTime + nExpiresIn;
470 4 : goMapIMDSURLToCachedToken[osURL] = cachedToken;
471 4 : CPLDebug("AZURE", "Storing credentials for %s until " CPL_FRMT_GIB,
472 : osURL.c_str(), cachedToken.nExpiresOn);
473 : }
474 :
475 4 : return !osAccessToken.empty();
476 : }
477 :
478 : /************************************************************************/
479 : /* GetConfigurationFromManagedIdentities() */
480 : /************************************************************************/
481 :
482 : static bool
483 21 : GetConfigurationFromManagedIdentities(const std::string &osPathForOption,
484 : std::string &osAccessToken)
485 : {
486 21 : if (GetConfigurationFromWorkloadIdentity(osAccessToken))
487 8 : return true;
488 13 : return GetConfigurationFromIMDSCredentials(osPathForOption, osAccessToken);
489 : }
490 :
491 : /************************************************************************/
492 : /* ClearCache() */
493 : /************************************************************************/
494 :
495 492 : void VSIAzureBlobHandleHelper::ClearCache()
496 : {
497 984 : std::lock_guard<std::mutex> guard(gMutex);
498 492 : goMapIMDSURLToCachedToken.clear();
499 492 : gnLastReadFederatedTokenFile = 0;
500 492 : gosFederatedToken.clear();
501 492 : }
502 :
503 : /************************************************************************/
504 : /* ParseStorageConnectionString() */
505 : /************************************************************************/
506 :
507 : static bool
508 266 : ParseStorageConnectionString(const std::string &osStorageConnectionString,
509 : const std::string &osServicePrefix,
510 : bool &bUseHTTPS, std::string &osEndpoint,
511 : std::string &osStorageAccount,
512 : std::string &osStorageKey, std::string &osSAS)
513 : {
514 : osStorageAccount =
515 266 : AzureCSGetParameter(osStorageConnectionString, "AccountName", false);
516 : osStorageKey =
517 266 : AzureCSGetParameter(osStorageConnectionString, "AccountKey", false);
518 :
519 : const std::string osProtocol(AzureCSGetParameter(
520 532 : osStorageConnectionString, "DefaultEndpointsProtocol", false));
521 266 : bUseHTTPS = (osProtocol != "http");
522 :
523 266 : if (osStorageAccount.empty() || osStorageKey.empty())
524 : {
525 3 : osStorageAccount.clear();
526 3 : osStorageKey.clear();
527 :
528 : const std::string osBlobEndpoint =
529 3 : RemoveTrailingSlash(AzureCSGetParameter(osStorageConnectionString,
530 6 : "BlobEndpoint", false));
531 6 : osSAS = AzureCSGetParameter(osStorageConnectionString,
532 3 : "SharedAccessSignature", false);
533 3 : if (!osBlobEndpoint.empty() && !osSAS.empty())
534 : {
535 2 : osEndpoint = osBlobEndpoint;
536 2 : return true;
537 : }
538 :
539 1 : return false;
540 : }
541 :
542 : const std::string osBlobEndpoint =
543 263 : AzureCSGetParameter(osStorageConnectionString, "BlobEndpoint", false);
544 263 : if (!osBlobEndpoint.empty())
545 : {
546 263 : osEndpoint = RemoveTrailingSlash(osBlobEndpoint);
547 : }
548 : else
549 : {
550 : const std::string osEndpointSuffix(AzureCSGetParameter(
551 0 : osStorageConnectionString, "EndpointSuffix", false));
552 0 : if (!osEndpointSuffix.empty())
553 0 : osEndpoint = (bUseHTTPS ? "https://" : "http://") +
554 0 : osStorageAccount + "." + osServicePrefix + "." +
555 0 : RemoveTrailingSlash(osEndpointSuffix);
556 : }
557 :
558 263 : return true;
559 : }
560 :
561 : /************************************************************************/
562 : /* GetConfigurationFromCLIConfigFile() */
563 : /************************************************************************/
564 :
565 9 : static bool GetConfigurationFromCLIConfigFile(
566 : const std::string &osPathForOption, const std::string &osServicePrefix,
567 : bool &bUseHTTPS, std::string &osEndpoint, std::string &osStorageAccount,
568 : std::string &osStorageKey, std::string &osSAS, std::string &osAccessToken,
569 : bool &bFromManagedIdentities)
570 : {
571 : #ifdef _WIN32
572 : const char *pszHome = CPLGetConfigOption("USERPROFILE", nullptr);
573 : constexpr char SEP_STRING[] = "\\";
574 : #else
575 9 : const char *pszHome = CPLGetConfigOption("HOME", nullptr);
576 9 : constexpr char SEP_STRING[] = "/";
577 : #endif
578 :
579 18 : std::string osDotAzure(pszHome ? pszHome : "");
580 9 : osDotAzure += SEP_STRING;
581 9 : osDotAzure += ".azure";
582 :
583 : const char *pszAzureConfigDir =
584 9 : CPLGetConfigOption("AZURE_CONFIG_DIR", osDotAzure.c_str());
585 9 : if (pszAzureConfigDir[0] == '\0')
586 4 : return false;
587 :
588 10 : std::string osConfigFilename = pszAzureConfigDir;
589 5 : osConfigFilename += SEP_STRING;
590 5 : osConfigFilename += "config";
591 :
592 5 : VSILFILE *fp = VSIFOpenL(osConfigFilename.c_str(), "rb");
593 10 : std::string osStorageConnectionString;
594 5 : if (fp == nullptr)
595 1 : return false;
596 :
597 4 : bool bInStorageSection = false;
598 25 : while (const char *pszLine = CPLReadLineL(fp))
599 : {
600 21 : if (pszLine[0] == '#' || pszLine[0] == ';')
601 : {
602 : // comment line
603 : }
604 21 : else if (strcmp(pszLine, "[storage]") == 0)
605 : {
606 4 : bInStorageSection = true;
607 : }
608 17 : else if (pszLine[0] == '[')
609 : {
610 4 : bInStorageSection = false;
611 : }
612 13 : else if (bInStorageSection)
613 : {
614 5 : char *pszKey = nullptr;
615 5 : const char *pszValue = CPLParseNameValue(pszLine, &pszKey);
616 5 : if (pszKey && pszValue)
617 : {
618 5 : if (EQUAL(pszKey, "account"))
619 : {
620 2 : osStorageAccount = pszValue;
621 : }
622 3 : else if (EQUAL(pszKey, "connection_string"))
623 : {
624 1 : osStorageConnectionString = pszValue;
625 : }
626 2 : else if (EQUAL(pszKey, "key"))
627 : {
628 1 : osStorageKey = pszValue;
629 : }
630 1 : else if (EQUAL(pszKey, "sas_token"))
631 : {
632 1 : osSAS = pszValue;
633 : // Az CLI apparently uses configparser with
634 : // BasicInterpolation where the % character has a special
635 : // meaning See
636 : // https://docs.python.org/3/library/configparser.html#configparser.BasicInterpolation
637 : // A token might end with %%3D which must be transformed to
638 : // %3D
639 1 : osSAS = CPLString(osSAS).replaceAll("%%", '%');
640 : }
641 : }
642 5 : CPLFree(pszKey);
643 : }
644 21 : }
645 4 : VSIFCloseL(fp);
646 :
647 4 : if (!osStorageConnectionString.empty())
648 : {
649 1 : return ParseStorageConnectionString(
650 : osStorageConnectionString, osServicePrefix, bUseHTTPS, osEndpoint,
651 1 : osStorageAccount, osStorageKey, osSAS);
652 : }
653 :
654 3 : if (osStorageAccount.empty())
655 : {
656 1 : CPLDebug("AZURE", "Missing storage.account in %s",
657 : osConfigFilename.c_str());
658 1 : return false;
659 : }
660 :
661 2 : if (osEndpoint.empty())
662 0 : osEndpoint = (bUseHTTPS ? "https://" : "http://") + osStorageAccount +
663 0 : "." + osServicePrefix + ".core.windows.net";
664 :
665 2 : osAccessToken = CPLGetConfigOption("AZURE_STORAGE_ACCESS_TOKEN", "");
666 2 : if (!osAccessToken.empty())
667 0 : return true;
668 :
669 2 : if (osStorageKey.empty() && osSAS.empty())
670 : {
671 0 : if (CPLTestBool(CPLGetConfigOption("AZURE_NO_SIGN_REQUEST", "NO")))
672 : {
673 0 : return true;
674 : }
675 :
676 0 : std::string osTmpAccessToken;
677 0 : if (GetConfigurationFromManagedIdentities(osPathForOption,
678 : osTmpAccessToken))
679 : {
680 0 : bFromManagedIdentities = true;
681 0 : return true;
682 : }
683 :
684 0 : CPLDebug("AZURE", "Missing storage.key or storage.sas_token in %s",
685 : osConfigFilename.c_str());
686 0 : return false;
687 : }
688 :
689 2 : return true;
690 : }
691 :
692 : /************************************************************************/
693 : /* GetConfiguration() */
694 : /************************************************************************/
695 :
696 303 : bool VSIAzureBlobHandleHelper::GetConfiguration(
697 : const std::string &osPathForOption, CSLConstList papszOptions,
698 : Service eService, bool &bUseHTTPS, std::string &osEndpoint,
699 : std::string &osStorageAccount, std::string &osStorageKey,
700 : std::string &osSAS, std::string &osAccessToken,
701 : bool &bFromManagedIdentities)
702 : {
703 303 : bFromManagedIdentities = false;
704 :
705 : const std::string osServicePrefix(
706 606 : eService == Service::SERVICE_BLOB ? "blob" : "dfs");
707 303 : bUseHTTPS = CPLTestBool(VSIGetPathSpecificOption(
708 : osPathForOption.c_str(), "CPL_AZURE_USE_HTTPS", "YES"));
709 606 : osEndpoint = RemoveTrailingSlash(VSIGetPathSpecificOption(
710 303 : osPathForOption.c_str(), "CPL_AZURE_ENDPOINT", ""));
711 :
712 : const std::string osStorageConnectionString(CSLFetchNameValueDef(
713 : papszOptions, "AZURE_STORAGE_CONNECTION_STRING",
714 : VSIGetPathSpecificOption(osPathForOption.c_str(),
715 606 : "AZURE_STORAGE_CONNECTION_STRING", "")));
716 303 : if (!osStorageConnectionString.empty())
717 : {
718 265 : return ParseStorageConnectionString(
719 : osStorageConnectionString, osServicePrefix, bUseHTTPS, osEndpoint,
720 265 : osStorageAccount, osStorageKey, osSAS);
721 : }
722 : else
723 : {
724 : osStorageAccount = CSLFetchNameValueDef(
725 : papszOptions, "AZURE_STORAGE_ACCOUNT",
726 : VSIGetPathSpecificOption(osPathForOption.c_str(),
727 38 : "AZURE_STORAGE_ACCOUNT", ""));
728 38 : if (!osStorageAccount.empty())
729 : {
730 29 : if (osEndpoint.empty())
731 30 : osEndpoint = (bUseHTTPS ? "https://" : "http://") +
732 30 : osStorageAccount + "." + osServicePrefix +
733 15 : ".core.windows.net";
734 :
735 : osAccessToken = CSLFetchNameValueDef(
736 : papszOptions, "AZURE_STORAGE_ACCESS_TOKEN",
737 : VSIGetPathSpecificOption(osPathForOption.c_str(),
738 29 : "AZURE_STORAGE_ACCESS_TOKEN", ""));
739 29 : if (!osAccessToken.empty())
740 1 : return true;
741 :
742 : osStorageKey = CSLFetchNameValueDef(
743 : papszOptions, "AZURE_STORAGE_ACCESS_KEY",
744 : VSIGetPathSpecificOption(osPathForOption.c_str(),
745 28 : "AZURE_STORAGE_ACCESS_KEY", ""));
746 28 : if (osStorageKey.empty())
747 : {
748 : osSAS = VSIGetPathSpecificOption(
749 : osPathForOption.c_str(), "AZURE_STORAGE_SAS_TOKEN",
750 : CPLGetConfigOption("AZURE_SAS",
751 24 : "")); // AZURE_SAS for GDAL < 3.5
752 24 : if (osSAS.empty())
753 : {
754 16 : if (CPLTestBool(VSIGetPathSpecificOption(
755 : osPathForOption.c_str(), "AZURE_NO_SIGN_REQUEST",
756 : "NO")))
757 : {
758 5 : return true;
759 : }
760 :
761 22 : std::string osTmpAccessToken;
762 11 : if (GetConfigurationFromManagedIdentities(osPathForOption,
763 : osTmpAccessToken))
764 : {
765 10 : bFromManagedIdentities = true;
766 10 : return true;
767 : }
768 :
769 1 : const char *pszMsg =
770 : "AZURE_STORAGE_ACCESS_KEY or AZURE_STORAGE_SAS_TOKEN "
771 : "or AZURE_NO_SIGN_REQUEST configuration option "
772 : "not defined";
773 1 : CPLDebug("AZURE", "%s", pszMsg);
774 1 : VSIError(VSIE_AWSInvalidCredentials, "%s", pszMsg);
775 1 : return false;
776 : }
777 : }
778 12 : return true;
779 : }
780 : }
781 :
782 9 : if (GetConfigurationFromCLIConfigFile(
783 : osPathForOption, osServicePrefix, bUseHTTPS, osEndpoint,
784 : osStorageAccount, osStorageKey, osSAS, osAccessToken,
785 : bFromManagedIdentities))
786 : {
787 3 : return true;
788 : }
789 :
790 6 : const char *pszMsg =
791 : "Missing AZURE_STORAGE_ACCOUNT+"
792 : "(AZURE_STORAGE_ACCESS_KEY or AZURE_STORAGE_SAS_TOKEN or "
793 : "AZURE_NO_SIGN_REQUEST) or "
794 : "AZURE_STORAGE_CONNECTION_STRING "
795 : "configuration options or Azure CLI configuration file";
796 6 : CPLDebug("AZURE", "%s", pszMsg);
797 6 : VSIError(VSIE_AWSInvalidCredentials, "%s", pszMsg);
798 6 : return false;
799 : }
800 :
801 : /************************************************************************/
802 : /* BuildFromURI() */
803 : /************************************************************************/
804 :
805 303 : VSIAzureBlobHandleHelper *VSIAzureBlobHandleHelper::BuildFromURI(
806 : const char *pszURI, const char *pszFSPrefix, CSLConstList papszOptions)
807 : {
808 303 : if (strcmp(pszFSPrefix, "/vsiaz/") != 0 &&
809 95 : strcmp(pszFSPrefix, "/vsiaz_streaming/") != 0 &&
810 89 : strcmp(pszFSPrefix, "/vsiadls/") != 0)
811 : {
812 0 : CPLError(CE_Failure, CPLE_AppDefined, "Unsupported FS prefix");
813 0 : return nullptr;
814 : }
815 :
816 606 : const auto eService = strcmp(pszFSPrefix, "/vsiaz/") == 0 ||
817 95 : strcmp(pszFSPrefix, "/vsiaz_streaming/") == 0
818 303 : ? Service::SERVICE_BLOB
819 : : Service::SERVICE_ADLS;
820 :
821 : std::string osPathForOption(
822 606 : eService == Service::SERVICE_BLOB ? "/vsiaz/" : "/vsiadls/");
823 303 : osPathForOption += pszURI;
824 :
825 303 : bool bUseHTTPS = true;
826 606 : std::string osStorageAccount;
827 606 : std::string osStorageKey;
828 606 : std::string osEndpoint;
829 606 : std::string osSAS;
830 606 : std::string osAccessToken;
831 303 : bool bFromManagedIdentities = false;
832 :
833 303 : if (!GetConfiguration(osPathForOption, papszOptions, eService, bUseHTTPS,
834 : osEndpoint, osStorageAccount, osStorageKey, osSAS,
835 : osAccessToken, bFromManagedIdentities))
836 : {
837 8 : return nullptr;
838 : }
839 :
840 295 : if (CPLTestBool(VSIGetPathSpecificOption(osPathForOption.c_str(),
841 : "AZURE_NO_SIGN_REQUEST", "NO")))
842 : {
843 6 : osStorageKey.clear();
844 6 : osSAS.clear();
845 6 : osAccessToken.clear();
846 : }
847 :
848 : // pszURI == bucket/object
849 590 : const std::string osBucketObject(pszURI);
850 590 : std::string osBucket(osBucketObject);
851 295 : std::string osObjectKey;
852 295 : size_t nSlashPos = osBucketObject.find('/');
853 295 : if (nSlashPos != std::string::npos)
854 : {
855 215 : osBucket = osBucketObject.substr(0, nSlashPos);
856 215 : osObjectKey = osBucketObject.substr(nSlashPos + 1);
857 : }
858 :
859 : return new VSIAzureBlobHandleHelper(
860 : osPathForOption, osEndpoint, osBucket, osObjectKey, osStorageAccount,
861 295 : osStorageKey, osSAS, osAccessToken, bFromManagedIdentities);
862 : }
863 :
864 : /************************************************************************/
865 : /* BuildURL() */
866 : /************************************************************************/
867 :
868 840 : std::string VSIAzureBlobHandleHelper::BuildURL(const std::string &osEndpoint,
869 : const std::string &osBucket,
870 : const std::string &osObjectKey,
871 : const std::string &osSAS)
872 : {
873 840 : std::string osURL = osEndpoint;
874 840 : osURL += "/";
875 840 : osURL += CPLAWSURLEncode(osBucket, false);
876 840 : if (!osObjectKey.empty())
877 461 : osURL += "/" + CPLAWSURLEncode(osObjectKey, false);
878 840 : if (!osSAS.empty())
879 11 : osURL += '?' + osSAS;
880 840 : return osURL;
881 : }
882 :
883 : /************************************************************************/
884 : /* RebuildURL() */
885 : /************************************************************************/
886 :
887 545 : void VSIAzureBlobHandleHelper::RebuildURL()
888 : {
889 545 : m_osURL = BuildURL(m_osEndpoint, m_osBucket, m_osObjectKey, std::string());
890 545 : m_osURL += GetQueryString(false);
891 545 : if (!m_osSAS.empty())
892 11 : m_osURL += (m_oMapQueryParameters.empty() ? '?' : '&') + m_osSAS;
893 545 : }
894 :
895 : /************************************************************************/
896 : /* GetSASQueryString() */
897 : /************************************************************************/
898 :
899 73 : std::string VSIAzureBlobHandleHelper::GetSASQueryString() const
900 : {
901 73 : if (!m_osSAS.empty())
902 4 : return '?' + m_osSAS;
903 69 : return std::string();
904 : }
905 :
906 : /************************************************************************/
907 : /* GetCurlHeaders() */
908 : /************************************************************************/
909 :
910 225 : struct curl_slist *VSIAzureBlobHandleHelper::GetCurlHeaders(
911 : const std::string &osVerb, const struct curl_slist *psExistingHeaders,
912 : const void *, size_t) const
913 : {
914 225 : if (m_bFromManagedIdentities || !m_osAccessToken.empty())
915 : {
916 22 : std::string osAccessToken;
917 11 : if (m_bFromManagedIdentities)
918 : {
919 10 : if (!GetConfigurationFromManagedIdentities(m_osPathForOption,
920 : osAccessToken))
921 0 : return nullptr;
922 : }
923 : else
924 : {
925 1 : osAccessToken = m_osAccessToken;
926 : }
927 :
928 11 : struct curl_slist *headers = nullptr;
929 :
930 : // Do not use CPLSPrintf() as we could get over the 8K character limit
931 : // with very large SAS tokens
932 11 : std::string osAuthorization = "Authorization: Bearer ";
933 11 : osAuthorization += osAccessToken;
934 11 : headers = curl_slist_append(headers, osAuthorization.c_str());
935 11 : headers = curl_slist_append(headers, "x-ms-version: 2019-12-12");
936 11 : return headers;
937 : }
938 :
939 428 : std::string osResource;
940 214 : const auto nSlashSlashPos = m_osEndpoint.find("//");
941 214 : if (nSlashSlashPos != std::string::npos)
942 : {
943 214 : const auto nResourcePos = m_osEndpoint.find('/', nSlashSlashPos + 2);
944 214 : if (nResourcePos != std::string::npos)
945 207 : osResource = m_osEndpoint.substr(nResourcePos);
946 : }
947 214 : osResource += "/" + m_osBucket;
948 214 : if (!m_osObjectKey.empty())
949 146 : osResource += "/" + CPLAWSURLEncode(m_osObjectKey, false);
950 :
951 428 : return GetAzureBlobHeaders(osVerb, psExistingHeaders, osResource,
952 214 : m_oMapQueryParameters, m_osStorageAccount,
953 214 : m_osStorageKey, m_bIncludeMSVersion);
954 : }
955 :
956 : /************************************************************************/
957 : /* GetSignedURL() */
958 : /************************************************************************/
959 :
960 7 : std::string VSIAzureBlobHandleHelper::GetSignedURL(CSLConstList papszOptions)
961 : {
962 7 : if (m_osStorageKey.empty())
963 3 : return m_osURL;
964 :
965 8 : std::string osStartDate(CPLGetAWS_SIGN4_Timestamp(time(nullptr)));
966 4 : const char *pszStartDate = CSLFetchNameValue(papszOptions, "START_DATE");
967 4 : if (pszStartDate)
968 2 : osStartDate = pszStartDate;
969 4 : int nYear, nMonth, nDay, nHour = 0, nMin = 0, nSec = 0;
970 4 : if (sscanf(osStartDate.c_str(), "%04d%02d%02dT%02d%02d%02dZ", &nYear,
971 4 : &nMonth, &nDay, &nHour, &nMin, &nSec) < 3)
972 : {
973 0 : return std::string();
974 : }
975 : osStartDate = CPLSPrintf("%04d-%02d-%02dT%02d:%02d:%02dZ", nYear, nMonth,
976 4 : nDay, nHour, nMin, nSec);
977 :
978 : struct tm brokendowntime;
979 4 : brokendowntime.tm_year = nYear - 1900;
980 4 : brokendowntime.tm_mon = nMonth - 1;
981 4 : brokendowntime.tm_mday = nDay;
982 4 : brokendowntime.tm_hour = nHour;
983 4 : brokendowntime.tm_min = nMin;
984 4 : brokendowntime.tm_sec = nSec;
985 4 : GIntBig nStartDate = CPLYMDHMSToUnixTime(&brokendowntime);
986 : GIntBig nEndDate =
987 : nStartDate +
988 4 : atoi(CSLFetchNameValueDef(papszOptions, "EXPIRATION_DELAY", "3600"));
989 4 : CPLUnixTimeToYMDHMS(nEndDate, &brokendowntime);
990 4 : nYear = brokendowntime.tm_year + 1900;
991 4 : nMonth = brokendowntime.tm_mon + 1;
992 4 : nDay = brokendowntime.tm_mday;
993 4 : nHour = brokendowntime.tm_hour;
994 4 : nMin = brokendowntime.tm_min;
995 4 : nSec = brokendowntime.tm_sec;
996 : std::string osEndDate = CPLSPrintf("%04d-%02d-%02dT%02d:%02d:%02dZ", nYear,
997 8 : nMonth, nDay, nHour, nMin, nSec);
998 :
999 8 : std::string osVerb(CSLFetchNameValueDef(papszOptions, "VERB", "GET"));
1000 : std::string osSignedPermissions(CSLFetchNameValueDef(
1001 : papszOptions, "SIGNEDPERMISSIONS",
1002 4 : (EQUAL(osVerb.c_str(), "GET") || EQUAL(osVerb.c_str(), "HEAD")) ? "r"
1003 12 : : "w"));
1004 :
1005 : std::string osSignedIdentifier(
1006 8 : CSLFetchNameValueDef(papszOptions, "SIGNEDIDENTIFIER", ""));
1007 :
1008 8 : const std::string osSignedVersion("2020-12-06");
1009 8 : const std::string osSignedProtocol("https");
1010 8 : const std::string osSignedResource("b"); // blob
1011 :
1012 8 : std::string osCanonicalizedResource("/blob/");
1013 4 : osCanonicalizedResource += CPLAWSURLEncode(m_osStorageAccount, false);
1014 4 : osCanonicalizedResource += '/';
1015 4 : osCanonicalizedResource += CPLAWSURLEncode(m_osBucket, false);
1016 4 : osCanonicalizedResource += '/';
1017 4 : osCanonicalizedResource += CPLAWSURLEncode(m_osObjectKey, false);
1018 :
1019 : // Cf https://learn.microsoft.com/en-us/rest/api/storageservices/create-service-sas
1020 8 : std::string osStringToSign;
1021 4 : osStringToSign += osSignedPermissions + "\n";
1022 4 : osStringToSign += osStartDate + "\n";
1023 4 : osStringToSign += osEndDate + "\n";
1024 4 : osStringToSign += osCanonicalizedResource + "\n";
1025 4 : osStringToSign += osSignedIdentifier + "\n";
1026 4 : osStringToSign += "\n"; // signedIP
1027 4 : osStringToSign += osSignedProtocol + "\n";
1028 4 : osStringToSign += osSignedVersion + "\n";
1029 4 : osStringToSign += osSignedResource + "\n";
1030 4 : osStringToSign += "\n"; // signedSnapshotTime
1031 4 : osStringToSign += "\n"; // signedEncryptionScope
1032 4 : osStringToSign += "\n"; // rscc
1033 4 : osStringToSign += "\n"; // rscd
1034 4 : osStringToSign += "\n"; // rsce
1035 4 : osStringToSign += "\n"; // rscl
1036 :
1037 : #ifdef DEBUG_VERBOSE
1038 : CPLDebug("AZURE", "osStringToSign = %s", osStringToSign.c_str());
1039 : #endif
1040 :
1041 : /* -------------------------------------------------------------------- */
1042 : /* Compute signature. */
1043 : /* -------------------------------------------------------------------- */
1044 : std::string osSignature(
1045 8 : CPLAzureGetSignature(osStringToSign, m_osStorageKey));
1046 :
1047 4 : ResetQueryParameters();
1048 4 : AddQueryParameter("sv", osSignedVersion);
1049 4 : AddQueryParameter("st", osStartDate);
1050 4 : AddQueryParameter("se", osEndDate);
1051 4 : AddQueryParameter("sr", osSignedResource);
1052 4 : AddQueryParameter("sp", osSignedPermissions);
1053 4 : AddQueryParameter("spr", osSignedProtocol);
1054 4 : AddQueryParameter("sig", osSignature);
1055 4 : if (!osSignedIdentifier.empty())
1056 0 : AddQueryParameter("si", osSignedIdentifier);
1057 4 : return m_osURL;
1058 : }
1059 :
1060 : #endif // HAVE_CURL
1061 :
1062 : //! @endcond
|
