Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: GDAL TileDB Driver
4 : * Purpose: Implement GDAL TileDB multidimensional support based on https://www.tiledb.io
5 : * Author: TileDB, Inc
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2023, TileDB, Inc
9 : *
10 : * Permission is hereby granted, free of charge, to any person obtaining a
11 : * copy of this software and associated documentation files (the "Software"),
12 : * to deal in the Software without restriction, including without limitation
13 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 : * and/or sell copies of the Software, and to permit persons to whom the
15 : * Software is furnished to do so, subject to the following conditions:
16 : *
17 : * The above copyright notice and this permission notice shall be included
18 : * in all copies or substantial portions of the Software.
19 : *
20 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 : * DEALINGS IN THE SOFTWARE.
27 : ****************************************************************************/
28 :
29 : #include "tiledbmultidim.h"
30 :
31 : #include "memmultidim.h"
32 :
33 : /************************************************************************/
34 : /* TileDBGroup::~TileDBGroup() */
35 : /************************************************************************/
36 :
37 116 : TileDBGroup::~TileDBGroup()
38 : {
39 58 : m_oMapGroups.clear();
40 58 : m_oMapArrays.clear();
41 58 : if (m_poTileDBGroup)
42 : {
43 : try
44 : {
45 58 : m_poTileDBGroup->close();
46 58 : m_poTileDBGroup.reset();
47 : }
48 0 : catch (const std::exception &e)
49 : {
50 : // Will leak memory, but better that than crashing
51 : // Cf https://github.com/TileDB-Inc/TileDB/issues/4101
52 0 : CPL_IGNORE_RET_VAL(m_poTileDBGroup.release());
53 0 : CPLError(CE_Failure, CPLE_AppDefined,
54 0 : "TileDBGroup::~TileDBGroup(): %s", e.what());
55 : }
56 : }
57 116 : }
58 :
59 : /************************************************************************/
60 : /* TileDBGroup::GetGroupNames() */
61 : /************************************************************************/
62 :
63 : std::vector<std::string>
64 7 : TileDBGroup::GetGroupNames(CSLConstList /*papszOptions*/) const
65 : {
66 7 : if (!EnsureOpenAs(TILEDB_READ))
67 0 : return {};
68 :
69 14 : std::vector<std::string> aosNames;
70 17 : for (uint64_t i = 0; i < m_poTileDBGroup->member_count(); ++i)
71 : {
72 20 : auto obj = m_poTileDBGroup->member(i);
73 10 : if (obj.type() == tiledb::Object::Type::Group)
74 : {
75 3 : if (obj.name().has_value())
76 3 : aosNames.push_back(*(obj.name()));
77 : else
78 0 : aosNames.push_back(CPLGetFilename(obj.uri().c_str()));
79 : }
80 : }
81 7 : return aosNames;
82 : }
83 :
84 : /************************************************************************/
85 : /* TileDBGroup::OpenFromDisk() */
86 : /************************************************************************/
87 :
88 : /* static */
89 31 : std::shared_ptr<TileDBGroup> TileDBGroup::OpenFromDisk(
90 : const std::shared_ptr<TileDBSharedResource> &poSharedResource,
91 : const std::string &osParentName, const std::string &osName,
92 : const std::string &osPath)
93 : {
94 : const auto eType =
95 31 : tiledb::Object::object(poSharedResource->GetCtx(), osPath).type();
96 31 : if (eType != tiledb::Object::Type::Group)
97 : {
98 0 : CPLError(CE_Failure, CPLE_AppDefined, "%s is not a TileDB group",
99 : osPath.c_str());
100 0 : return nullptr;
101 : }
102 :
103 : auto poTileDBGroup = std::make_unique<tiledb::Group>(
104 62 : poSharedResource->GetCtx(), osPath, TILEDB_READ);
105 :
106 : auto poGroup =
107 62 : TileDBGroup::Create(poSharedResource, osParentName, osName, osPath);
108 31 : poGroup->m_poTileDBGroup = std::move(poTileDBGroup);
109 31 : return poGroup;
110 : }
111 :
112 : /************************************************************************/
113 : /* TileDBGroup::CreateOnDisk() */
114 : /************************************************************************/
115 :
116 : /* static */
117 27 : std::shared_ptr<TileDBGroup> TileDBGroup::CreateOnDisk(
118 : const std::shared_ptr<TileDBSharedResource> &poSharedResource,
119 : const std::string &osParentName, const std::string &osName,
120 : const std::string &osPath)
121 : {
122 : try
123 : {
124 27 : tiledb::create_group(poSharedResource->GetCtx(), osPath);
125 : }
126 0 : catch (const std::exception &e)
127 : {
128 0 : CPLError(CE_Failure, CPLE_AppDefined, "%s", e.what());
129 0 : return nullptr;
130 : }
131 :
132 : auto poTileDBGroup = std::make_unique<tiledb::Group>(
133 54 : poSharedResource->GetCtx(), osPath, TILEDB_WRITE);
134 :
135 : auto poGroup =
136 54 : TileDBGroup::Create(poSharedResource, osParentName, osName, osPath);
137 27 : poGroup->m_poTileDBGroup = std::move(poTileDBGroup);
138 27 : return poGroup;
139 : }
140 :
141 : /************************************************************************/
142 : /* TileDBGroup::EnsureOpenAs() */
143 : /************************************************************************/
144 :
145 160 : bool TileDBGroup::EnsureOpenAs(tiledb_query_type_t mode) const
146 : {
147 160 : if (!m_poTileDBGroup)
148 0 : return false;
149 160 : if (m_poTileDBGroup->query_type() == mode && m_poTileDBGroup->is_open())
150 72 : return true;
151 : try
152 : {
153 88 : m_poTileDBGroup->close();
154 88 : m_poTileDBGroup->open(mode);
155 : }
156 0 : catch (const std::exception &e)
157 : {
158 0 : CPLError(CE_Failure, CPLE_AppDefined, "%s", e.what());
159 0 : m_poTileDBGroup.reset();
160 0 : return false;
161 : }
162 88 : return true;
163 : }
164 :
165 : /************************************************************************/
166 : /* TileDBGroup::HasObjectOfSameName() */
167 : /************************************************************************/
168 :
169 39 : bool TileDBGroup::HasObjectOfSameName(const std::string &osName) const
170 : {
171 39 : if (m_oMapGroups.find(osName) != m_oMapGroups.end())
172 : {
173 2 : CPLError(CE_Failure, CPLE_AppDefined, "A group named %s already exists",
174 : osName.c_str());
175 2 : return true;
176 : }
177 37 : if (m_oMapArrays.find(osName) != m_oMapArrays.end())
178 : {
179 1 : CPLError(CE_Failure, CPLE_AppDefined,
180 : "An array named %s already exists", osName.c_str());
181 1 : return true;
182 : }
183 :
184 36 : if (!EnsureOpenAs(TILEDB_READ))
185 0 : return {};
186 47 : for (uint64_t i = 0; i < m_poTileDBGroup->member_count(); ++i)
187 : {
188 11 : auto obj = m_poTileDBGroup->member(i);
189 11 : std::string osObjName = obj.name().has_value()
190 22 : ? *(obj.name())
191 22 : : CPLGetFilename(obj.uri().c_str());
192 11 : if (osName == osObjName)
193 : {
194 0 : if (obj.type() == tiledb::Object::Type::Group)
195 : {
196 0 : CPLError(CE_Failure, CPLE_AppDefined,
197 : "A group named %s already exists", osName.c_str());
198 0 : return true;
199 : }
200 0 : else if (obj.type() == tiledb::Object::Type::Array)
201 : {
202 0 : CPLError(CE_Failure, CPLE_AppDefined,
203 : "An array named %s already exists", osName.c_str());
204 0 : return true;
205 : }
206 : }
207 : }
208 36 : return false;
209 : }
210 :
211 : /************************************************************************/
212 : /* TileDBGroup::OpenGroup() */
213 : /************************************************************************/
214 :
215 : std::shared_ptr<GDALGroup>
216 3 : TileDBGroup::OpenGroup(const std::string &osName,
217 : CSLConstList /*papszOptions*/) const
218 : {
219 3 : auto oIter = m_oMapGroups.find(osName);
220 3 : if (oIter != m_oMapGroups.end())
221 : {
222 1 : return oIter->second;
223 : }
224 2 : if (!m_poTileDBGroup)
225 0 : return nullptr;
226 :
227 : // Try to match by member name property first, and if not use the
228 : // last part of their URI
229 4 : std::string osSubPath;
230 4 : std::string osSubPathCandidate;
231 2 : for (uint64_t i = 0; i < m_poTileDBGroup->member_count(); ++i)
232 : {
233 1 : auto obj = m_poTileDBGroup->member(i);
234 1 : if (obj.type() == tiledb::Object::Type::Group)
235 : {
236 1 : if (obj.name().has_value() && *(obj.name()) == osName)
237 : {
238 1 : osSubPath = obj.uri();
239 1 : break;
240 : }
241 0 : else if (CPLGetFilename(obj.uri().c_str()) == osName)
242 : {
243 0 : osSubPathCandidate = osSubPath;
244 : }
245 : }
246 : }
247 2 : if (osSubPath.empty())
248 1 : osSubPath = std::move(osSubPathCandidate);
249 2 : if (osSubPath.empty())
250 1 : return nullptr;
251 :
252 : auto poSubGroup = TileDBGroup::OpenFromDisk(
253 2 : m_poSharedResource, m_osFullName, osName, osSubPath);
254 1 : if (!poSubGroup)
255 0 : return nullptr;
256 :
257 1 : m_oMapGroups[osName] = poSubGroup;
258 :
259 1 : return poSubGroup;
260 : }
261 :
262 : /************************************************************************/
263 : /* TileDBGroup::CreateGroup() */
264 : /************************************************************************/
265 :
266 5 : std::shared_ptr<GDALGroup> TileDBGroup::CreateGroup(const std::string &osName,
267 : CSLConstList papszOptions)
268 : {
269 5 : if (!m_poSharedResource->IsUpdatable())
270 : {
271 1 : CPLError(CE_Failure, CPLE_NotSupported,
272 : "Dataset not open in update mode");
273 1 : return nullptr;
274 : }
275 :
276 4 : if (HasObjectOfSameName(osName))
277 2 : return nullptr;
278 :
279 4 : std::string osSubPath = m_poTileDBGroup->uri() + "/" +
280 6 : TileDBSharedResource::SanitizeNameForPath(osName);
281 2 : const char *pszURI = CSLFetchNameValue(papszOptions, "URI");
282 2 : if (pszURI)
283 0 : osSubPath = pszURI;
284 : auto poSubGroup =
285 4 : CreateOnDisk(m_poSharedResource, m_osFullName, osName, osSubPath);
286 2 : if (!poSubGroup)
287 0 : return nullptr;
288 :
289 2 : if (!AddMember(osSubPath, osName))
290 0 : return nullptr;
291 2 : m_oMapGroups[osName] = poSubGroup;
292 :
293 2 : return poSubGroup;
294 : }
295 :
296 : /************************************************************************/
297 : /* TileDBGroup::AddMember() */
298 : /************************************************************************/
299 :
300 32 : bool TileDBGroup::AddMember(const std::string &osPath,
301 : const std::string &osName)
302 : {
303 32 : if (!EnsureOpenAs(TILEDB_WRITE))
304 0 : return false;
305 :
306 : try
307 : {
308 32 : m_poTileDBGroup->add_member(osPath, /* relative= */ false, osName);
309 : }
310 0 : catch (const std::exception &e)
311 : {
312 0 : CPLError(CE_Failure, CPLE_AppDefined, "AddMember() failed with: %s",
313 0 : e.what());
314 0 : return false;
315 : }
316 : // Force close() and re-open() to avoid
317 : // https://github.com/TileDB-Inc/TileDB/issues/4101
318 : try
319 : {
320 32 : m_poTileDBGroup->close();
321 32 : m_poTileDBGroup->open(TILEDB_WRITE);
322 : }
323 0 : catch (const std::exception &e)
324 : {
325 0 : CPLError(CE_Failure, CPLE_AppDefined, "AddMember() failed with: %s",
326 0 : e.what());
327 0 : m_poTileDBGroup.reset();
328 0 : return false;
329 : }
330 32 : return true;
331 : }
332 :
333 : /************************************************************************/
334 : /* TileDBGroup::CreateDimension() */
335 : /************************************************************************/
336 :
337 36 : std::shared_ptr<GDALDimension> TileDBGroup::CreateDimension(
338 : const std::string &osName, const std::string &osType,
339 : const std::string &osDirection, GUInt64 nSize, CSLConstList)
340 : {
341 36 : if (osName.empty())
342 : {
343 0 : CPLError(CE_Failure, CPLE_NotSupported,
344 : "Empty dimension name not supported");
345 0 : return nullptr;
346 : }
347 :
348 36 : if (m_oMapDimensions.find(osName) != m_oMapDimensions.end())
349 : {
350 0 : CPLError(CE_Failure, CPLE_AppDefined,
351 : "A dimension with same name already exists");
352 0 : return nullptr;
353 : }
354 : auto newDim(std::make_shared<GDALDimensionWeakIndexingVar>(
355 72 : GetFullName(), osName, osType, osDirection, nSize));
356 36 : m_oMapDimensions[osName] = newDim;
357 36 : return newDim;
358 : }
359 :
360 : /************************************************************************/
361 : /* TileDBGroup::GetMDArrayNames() */
362 : /************************************************************************/
363 :
364 : std::vector<std::string>
365 27 : TileDBGroup::GetMDArrayNames(CSLConstList /*papszOptions */) const
366 : {
367 27 : if (!EnsureOpenAs(TILEDB_READ))
368 0 : return {};
369 :
370 54 : std::vector<std::string> aosNames;
371 85 : for (uint64_t i = 0; i < m_poTileDBGroup->member_count(); ++i)
372 : {
373 116 : auto obj = m_poTileDBGroup->member(i);
374 58 : if (obj.type() == tiledb::Object::Type::Array)
375 : {
376 114 : tiledb::ArraySchema schema(m_poSharedResource->GetCtx(), obj.uri());
377 57 : if (schema.array_type() == TILEDB_DENSE)
378 : {
379 : const std::string osName =
380 171 : obj.name().has_value() ? *(obj.name())
381 171 : : CPLGetFilename(obj.uri().c_str());
382 57 : const auto nAttributes = schema.attribute_num();
383 57 : if (nAttributes != 1)
384 : {
385 0 : for (uint32_t iAttr = 0; iAttr < nAttributes; ++iAttr)
386 : {
387 0 : aosNames.push_back(osName + "." +
388 0 : schema.attribute(iAttr).name());
389 : }
390 : }
391 : else
392 : {
393 57 : aosNames.push_back(osName);
394 : }
395 : }
396 : }
397 : }
398 :
399 : // As array creation is deferred, the above loop didn't get freshly
400 : // created arrays
401 46 : for (const auto &kv : m_oMapArrays)
402 : {
403 19 : if (std::find(aosNames.begin(), aosNames.end(), kv.first) ==
404 38 : aosNames.end())
405 : {
406 1 : aosNames.push_back(kv.first);
407 : }
408 : }
409 :
410 27 : return aosNames;
411 : }
412 :
413 : /************************************************************************/
414 : /* TileDBGroup::OpenMDArray() */
415 : /************************************************************************/
416 :
417 : std::shared_ptr<GDALMDArray>
418 62 : TileDBGroup::OpenMDArray(const std::string &osName,
419 : CSLConstList papszOptions) const
420 : {
421 62 : auto oIter = m_oMapArrays.find(osName);
422 62 : if (oIter != m_oMapArrays.end())
423 : {
424 22 : return oIter->second;
425 : }
426 40 : if (!m_poTileDBGroup)
427 0 : return nullptr;
428 :
429 80 : std::string osNamePrefix = osName;
430 80 : std::string osNameSuffix;
431 40 : const auto nLastDot = osName.rfind('.');
432 40 : if (nLastDot != std::string::npos)
433 : {
434 0 : osNamePrefix = osName.substr(0, nLastDot);
435 0 : osNameSuffix = osName.substr(nLastDot + 1);
436 : }
437 :
438 : // Try to match by member name property first, and if not use the
439 : // last part of their URI
440 80 : std::string osSubPath;
441 80 : std::string osSubPathCandidate;
442 63 : for (uint64_t i = 0; i < m_poTileDBGroup->member_count(); ++i)
443 : {
444 58 : auto obj = m_poTileDBGroup->member(i);
445 :
446 : const auto MatchNameSuffix =
447 81 : [this, &obj, &osName, &osNamePrefix,
448 370 : &osNameSuffix](const std::string &osObjName)
449 : {
450 162 : tiledb::ArraySchema schema(m_poSharedResource->GetCtx(), obj.uri());
451 81 : if (osNameSuffix.empty() && osObjName == osName)
452 : {
453 35 : return true;
454 : }
455 46 : else if (osObjName == osNamePrefix && !osNameSuffix.empty() &&
456 0 : schema.has_attribute(osNameSuffix))
457 : {
458 0 : return true;
459 : }
460 46 : return false;
461 58 : };
462 :
463 58 : if (obj.type() == tiledb::Object::Type::Array)
464 : {
465 58 : if (obj.name().has_value() && MatchNameSuffix(*(obj.name())))
466 : {
467 35 : osSubPath = obj.uri();
468 35 : break;
469 : }
470 23 : else if (MatchNameSuffix(CPLGetFilename(obj.uri().c_str())))
471 : {
472 0 : osSubPathCandidate = obj.uri();
473 : }
474 : }
475 : }
476 40 : if (osSubPath.empty())
477 5 : osSubPath = std::move(osSubPathCandidate);
478 40 : if (osSubPath.empty())
479 5 : return nullptr;
480 :
481 35 : if (m_oSetArrayInOpening.find(osName) != m_oSetArrayInOpening.end())
482 2 : return nullptr;
483 33 : m_oSetArrayInOpening.insert(osName);
484 0 : auto poArray = TileDBArray::OpenFromDisk(m_poSharedResource, m_pSelf.lock(),
485 33 : m_osFullName, osName, osNameSuffix,
486 66 : osSubPath, papszOptions);
487 33 : m_oSetArrayInOpening.erase(osName);
488 33 : if (!poArray)
489 0 : return nullptr;
490 :
491 33 : m_oMapArrays[osName] = poArray;
492 :
493 33 : return poArray;
494 : }
495 :
496 : /************************************************************************/
497 : /* TileDBGroup::CreateMDArray() */
498 : /************************************************************************/
499 :
500 40 : std::shared_ptr<GDALMDArray> TileDBGroup::CreateMDArray(
501 : const std::string &osName,
502 : const std::vector<std::shared_ptr<GDALDimension>> &aoDimensions,
503 : const GDALExtendedDataType &oDataType, CSLConstList papszOptions)
504 : {
505 40 : if (CPLTestBool(CSLFetchNameValueDef(papszOptions, "IN_MEMORY", "NO")))
506 : {
507 : auto poArray =
508 8 : MEMMDArray::Create(std::string(), osName, aoDimensions, oDataType);
509 4 : if (!poArray || !poArray->Init())
510 0 : return nullptr;
511 4 : return poArray;
512 : }
513 :
514 36 : if (!m_poSharedResource->IsUpdatable())
515 : {
516 1 : CPLError(CE_Failure, CPLE_NotSupported,
517 : "Dataset not open in update mode");
518 1 : return nullptr;
519 : }
520 :
521 35 : if (HasObjectOfSameName(osName))
522 1 : return nullptr;
523 :
524 34 : if (!EnsureOpenAs(TILEDB_WRITE))
525 0 : return nullptr;
526 :
527 68 : auto poSelf = std::dynamic_pointer_cast<TileDBGroup>(m_pSelf.lock());
528 34 : CPLAssert(poSelf);
529 : auto poArray =
530 34 : TileDBArray::CreateOnDisk(m_poSharedResource, poSelf, osName,
531 68 : aoDimensions, oDataType, papszOptions);
532 34 : if (!poArray)
533 4 : return nullptr;
534 :
535 30 : m_oMapArrays[osName] = poArray;
536 30 : return poArray;
537 : }
538 :
539 : /************************************************************************/
540 : /* TileDBGroup::CreateAttribute() */
541 : /************************************************************************/
542 :
543 5 : std::shared_ptr<GDALAttribute> TileDBGroup::CreateAttribute(
544 : const std::string &osName, const std::vector<GUInt64> &anDimensions,
545 : const GDALExtendedDataType &oDataType, CSLConstList papszOptions)
546 : {
547 5 : return CreateAttributeImpl(osName, anDimensions, oDataType, papszOptions);
548 : }
549 :
550 : /************************************************************************/
551 : /* TileDBGroup::GetAttribute() */
552 : /************************************************************************/
553 :
554 : std::shared_ptr<GDALAttribute>
555 4 : TileDBGroup::GetAttribute(const std::string &osName) const
556 : {
557 4 : return GetAttributeImpl(osName);
558 : }
559 :
560 : /************************************************************************/
561 : /* TileDBGroup::GetAttributes() */
562 : /************************************************************************/
563 :
564 : std::vector<std::shared_ptr<GDALAttribute>>
565 4 : TileDBGroup::GetAttributes(CSLConstList /* papszOptions */) const
566 : {
567 4 : return GetAttributesImpl();
568 : }
569 :
570 : /************************************************************************/
571 : /* TileDBGroup::DeleteAttribute() */
572 : /************************************************************************/
573 :
574 3 : bool TileDBGroup::DeleteAttribute(const std::string &osName,
575 : CSLConstList papszOptions)
576 : {
577 3 : return DeleteAttributeImpl(osName, papszOptions);
578 : }
|
