Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: VSI Virtual File System
4 : * Purpose: Implementation of Memory Buffer virtual IO functions.
5 : * Author: Frank Warmerdam, warmerdam@pobox.com
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
9 : * Copyright (c) 2007-2014, Even Rouault <even dot rouault at spatialys.com>
10 : *
11 : * Permission is hereby granted, free of charge, to any person obtaining a
12 : * copy of this software and associated documentation files (the "Software"),
13 : * to deal in the Software without restriction, including without limitation
14 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 : * and/or sell copies of the Software, and to permit persons to whom the
16 : * Software is furnished to do so, subject to the following conditions:
17 : *
18 : * The above copyright notice and this permission notice shall be included
19 : * in all copies or substantial portions of the Software.
20 : *
21 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
22 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
24 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 : * DEALINGS IN THE SOFTWARE.
28 : ****************************************************************************/
29 :
30 : #include "cpl_port.h"
31 : #include "cpl_vsi.h"
32 : #include "cpl_vsi_virtual.h"
33 :
34 : #include <cerrno>
35 : #include <cstddef>
36 : #include <cstring>
37 : #include <ctime>
38 : #if HAVE_FCNTL_H
39 : #include <fcntl.h>
40 : #endif
41 : #if HAVE_SYS_STAT_H
42 : #include <sys/stat.h>
43 : #endif
44 :
45 : #include <algorithm>
46 : #include <map>
47 : #include <string>
48 : #include <utility>
49 : #include <memory>
50 :
51 : #include <mutex>
52 : // c++17 or VS2017
53 : #if defined(HAVE_SHARED_MUTEX) || _MSC_VER >= 1910
54 : #include <shared_mutex>
55 : #define CPL_SHARED_MUTEX_TYPE std::shared_mutex
56 : #define CPL_SHARED_LOCK std::shared_lock<std::shared_mutex>
57 : #define CPL_EXCLUSIVE_LOCK std::unique_lock<std::shared_mutex>
58 : #else
59 : // Poor-man implementation of std::shared_mutex with an exclusive mutex
60 : #define CPL_SHARED_MUTEX_TYPE std::mutex
61 : #define CPL_SHARED_LOCK std::lock_guard<std::mutex>
62 : #define CPL_EXCLUSIVE_LOCK std::lock_guard<std::mutex>
63 : #endif
64 :
65 : #include "cpl_atomic_ops.h"
66 : #include "cpl_conv.h"
67 : #include "cpl_error.h"
68 : #include "cpl_multiproc.h"
69 : #include "cpl_string.h"
70 :
71 : //! @cond Doxygen_Suppress
72 :
73 : /*
74 : ** Notes on Multithreading:
75 : **
76 : ** VSIMemFilesystemHandler: This class maintains a mutex to protect
77 : ** access and update of the oFileList array which has all the "files" in
78 : ** the memory filesystem area. It is expected that multiple threads would
79 : ** want to create and read different files at the same time and so might
80 : ** collide access oFileList without the mutex.
81 : **
82 : ** VSIMemFile: A mutex protects accesses to the file
83 : **
84 : ** VSIMemHandle: This is essentially a "current location" representing
85 : ** on accessor to a file, and is inherently intended only to be used in
86 : ** a single thread.
87 : **
88 : ** In General:
89 : **
90 : ** Multiple threads accessing the memory filesystem are ok as long as
91 : ** a given VSIMemHandle (i.e. FILE * at app level) isn't used by multiple
92 : ** threads at once.
93 : */
94 :
95 : /************************************************************************/
96 : /* ==================================================================== */
97 : /* VSIMemFile */
98 : /* ==================================================================== */
99 : /************************************************************************/
100 :
101 : class VSIMemFile
102 : {
103 : CPL_DISALLOW_COPY_ASSIGN(VSIMemFile)
104 :
105 : public:
106 : CPLString osFilename{};
107 :
108 : bool bIsDirectory = false;
109 :
110 : bool bOwnData = true;
111 : GByte *pabyData = nullptr;
112 : vsi_l_offset nLength = 0;
113 : vsi_l_offset nAllocLength = 0;
114 : vsi_l_offset nMaxLength = GUINTBIG_MAX;
115 :
116 : time_t mTime = 0;
117 : CPL_SHARED_MUTEX_TYPE m_oMutex{};
118 :
119 : VSIMemFile();
120 : virtual ~VSIMemFile();
121 :
122 : bool SetLength(vsi_l_offset nNewSize);
123 : };
124 :
125 : /************************************************************************/
126 : /* ==================================================================== */
127 : /* VSIMemHandle */
128 : /* ==================================================================== */
129 : /************************************************************************/
130 :
131 : class VSIMemHandle final : public VSIVirtualHandle
132 : {
133 : CPL_DISALLOW_COPY_ASSIGN(VSIMemHandle)
134 :
135 : public:
136 : std::shared_ptr<VSIMemFile> poFile = nullptr;
137 : vsi_l_offset m_nOffset = 0;
138 : bool bUpdate = false;
139 : bool bEOF = false;
140 : bool bExtendFileAtNextWrite = false;
141 :
142 158657 : VSIMemHandle() = default;
143 : ~VSIMemHandle() override;
144 :
145 : int Seek(vsi_l_offset nOffset, int nWhence) override;
146 : vsi_l_offset Tell() override;
147 : size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
148 : size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
149 : int Eof() override;
150 : int Close() override;
151 : int Truncate(vsi_l_offset nNewSize) override;
152 :
153 3 : bool HasPRead() const override
154 : {
155 3 : return true;
156 : }
157 :
158 : size_t PRead(void * /*pBuffer*/, size_t /* nSize */,
159 : vsi_l_offset /*nOffset*/) const override;
160 : };
161 :
162 : /************************************************************************/
163 : /* ==================================================================== */
164 : /* VSIMemFilesystemHandler */
165 : /* ==================================================================== */
166 : /************************************************************************/
167 :
168 : class VSIMemFilesystemHandler final : public VSIFilesystemHandler
169 : {
170 : const std::string m_osPrefix;
171 : CPL_DISALLOW_COPY_ASSIGN(VSIMemFilesystemHandler)
172 :
173 : public:
174 : std::map<CPLString, std::shared_ptr<VSIMemFile>> oFileList{};
175 : CPLMutex *hMutex = nullptr;
176 :
177 1229 : explicit VSIMemFilesystemHandler(const char *pszPrefix)
178 1229 : : m_osPrefix(pszPrefix)
179 : {
180 1229 : }
181 :
182 : ~VSIMemFilesystemHandler() override;
183 :
184 : // TODO(schwehr): Fix VSIFileFromMemBuffer so that using is not needed.
185 : using VSIFilesystemHandler::Open;
186 :
187 : VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
188 : bool bSetError,
189 : CSLConstList /* papszOptions */) override;
190 : int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
191 : int nFlags) override;
192 : int Unlink(const char *pszFilename) override;
193 : int Mkdir(const char *pszDirname, long nMode) override;
194 : int Rmdir(const char *pszDirname) override;
195 : char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
196 : int Rename(const char *oldpath, const char *newpath) override;
197 : GIntBig GetDiskFreeSpace(const char *pszDirname) override;
198 :
199 : static std::string NormalizePath(const std::string &in);
200 :
201 : int Unlink_unlocked(const char *pszFilename);
202 :
203 1 : VSIFilesystemHandler *Duplicate(const char *pszPrefix) override
204 : {
205 1 : return new VSIMemFilesystemHandler(pszPrefix);
206 : }
207 : };
208 :
209 : /************************************************************************/
210 : /* ==================================================================== */
211 : /* VSIMemFile */
212 : /* ==================================================================== */
213 : /************************************************************************/
214 :
215 : /************************************************************************/
216 : /* VSIMemFile() */
217 : /************************************************************************/
218 :
219 87410 : VSIMemFile::VSIMemFile()
220 : {
221 87410 : time(&mTime);
222 87410 : }
223 :
224 : /************************************************************************/
225 : /* ~VSIMemFile() */
226 : /************************************************************************/
227 :
228 87115 : VSIMemFile::~VSIMemFile()
229 : {
230 87115 : if (bOwnData && pabyData)
231 72967 : CPLFree(pabyData);
232 87115 : }
233 :
234 : /************************************************************************/
235 : /* SetLength() */
236 : /************************************************************************/
237 :
238 : // Must be called under exclusive lock
239 1118710 : bool VSIMemFile::SetLength(vsi_l_offset nNewLength)
240 :
241 : {
242 1118710 : if (nNewLength > nMaxLength)
243 : {
244 4632 : CPLError(CE_Failure, CPLE_NotSupported, "Maximum file size reached!");
245 4632 : return false;
246 : }
247 :
248 : /* -------------------------------------------------------------------- */
249 : /* Grow underlying array if needed. */
250 : /* -------------------------------------------------------------------- */
251 1114080 : if (nNewLength > nAllocLength)
252 : {
253 : // If we don't own the buffer, we cannot reallocate it because
254 : // the return address might be different from the one passed by
255 : // the caller. Hence, the caller would not be able to free
256 : // the buffer.
257 116551 : if (!bOwnData)
258 : {
259 1 : CPLError(CE_Failure, CPLE_NotSupported,
260 : "Cannot extended in-memory file whose ownership was not "
261 : "transferred");
262 1 : return false;
263 : }
264 :
265 116550 : const vsi_l_offset nNewAlloc = (nNewLength + nNewLength / 10) + 5000;
266 116550 : GByte *pabyNewData = nullptr;
267 : if (static_cast<vsi_l_offset>(static_cast<size_t>(nNewAlloc)) ==
268 : nNewAlloc)
269 : {
270 : pabyNewData = static_cast<GByte *>(
271 116550 : VSIRealloc(pabyData, static_cast<size_t>(nNewAlloc)));
272 : }
273 116551 : if (pabyNewData == nullptr)
274 : {
275 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
276 : "Cannot extend in-memory file to " CPL_FRMT_GUIB
277 : " bytes due to out-of-memory situation",
278 : nNewAlloc);
279 1 : return false;
280 : }
281 :
282 : // Clear the new allocated part of the buffer.
283 116552 : memset(pabyNewData + nAllocLength, 0,
284 116552 : static_cast<size_t>(nNewAlloc - nAllocLength));
285 :
286 116552 : pabyData = pabyNewData;
287 116552 : nAllocLength = nNewAlloc;
288 : }
289 997531 : else if (nNewLength < nLength)
290 : {
291 5590 : memset(pabyData + nNewLength, 0,
292 5590 : static_cast<size_t>(nLength - nNewLength));
293 : }
294 :
295 1114080 : nLength = nNewLength;
296 1114080 : time(&mTime);
297 :
298 1114080 : return true;
299 : }
300 :
301 : /************************************************************************/
302 : /* ==================================================================== */
303 : /* VSIMemHandle */
304 : /* ==================================================================== */
305 : /************************************************************************/
306 :
307 : /************************************************************************/
308 : /* ~VSIMemHandle() */
309 : /************************************************************************/
310 :
311 317270 : VSIMemHandle::~VSIMemHandle()
312 : {
313 158629 : VSIMemHandle::Close();
314 317281 : }
315 :
316 : /************************************************************************/
317 : /* Close() */
318 : /************************************************************************/
319 :
320 317289 : int VSIMemHandle::Close()
321 :
322 : {
323 317289 : if (poFile)
324 : {
325 : #ifdef DEBUG_VERBOSE
326 : CPLDebug("VSIMEM", "Closing handle %p on %s: ref_count=%d (before)",
327 : this, poFile->osFilename.c_str(),
328 : static_cast<int>(poFile.use_count()));
329 : #endif
330 158638 : poFile = nullptr;
331 : }
332 :
333 317273 : return 0;
334 : }
335 :
336 : /************************************************************************/
337 : /* Seek() */
338 : /************************************************************************/
339 :
340 4793460 : int VSIMemHandle::Seek(vsi_l_offset nOffset, int nWhence)
341 :
342 : {
343 9586900 : CPL_SHARED_LOCK oLock(poFile->m_oMutex);
344 :
345 4793460 : bExtendFileAtNextWrite = false;
346 4793460 : if (nWhence == SEEK_CUR)
347 : {
348 : if (nOffset > INT_MAX)
349 : {
350 : // printf("likely negative offset intended\n");
351 : }
352 376807 : m_nOffset += nOffset;
353 : }
354 4416650 : else if (nWhence == SEEK_SET)
355 : {
356 4235460 : m_nOffset = nOffset;
357 : }
358 181184 : else if (nWhence == SEEK_END)
359 : {
360 181181 : m_nOffset = poFile->nLength + nOffset;
361 : }
362 : else
363 : {
364 3 : errno = EINVAL;
365 3 : return -1;
366 : }
367 :
368 4793450 : bEOF = false;
369 :
370 4793450 : if (m_nOffset > poFile->nLength)
371 : {
372 53664 : if (bUpdate) // Writable files are zero-extended by seek past end.
373 : {
374 53612 : bExtendFileAtNextWrite = true;
375 : }
376 : }
377 :
378 4793440 : return 0;
379 : }
380 :
381 : /************************************************************************/
382 : /* Tell() */
383 : /************************************************************************/
384 :
385 3168340 : vsi_l_offset VSIMemHandle::Tell()
386 :
387 : {
388 3168340 : return m_nOffset;
389 : }
390 :
391 : /************************************************************************/
392 : /* Read() */
393 : /************************************************************************/
394 :
395 6863470 : size_t VSIMemHandle::Read(void *pBuffer, size_t nSize, size_t nCount)
396 :
397 : {
398 13726900 : CPL_SHARED_LOCK oLock(poFile->m_oMutex);
399 :
400 6863470 : size_t nBytesToRead = nSize * nCount;
401 6863470 : if (nBytesToRead == 0)
402 1560 : return 0;
403 :
404 6861910 : if (nCount > 0 && nBytesToRead / nCount != nSize)
405 : {
406 0 : bEOF = true;
407 0 : return 0;
408 : }
409 :
410 6861910 : if (poFile->nLength <= m_nOffset || nBytesToRead + m_nOffset < nBytesToRead)
411 : {
412 25555 : bEOF = true;
413 25555 : return 0;
414 : }
415 6836360 : if (nBytesToRead + m_nOffset > poFile->nLength)
416 : {
417 45741 : nBytesToRead = static_cast<size_t>(poFile->nLength - m_nOffset);
418 45744 : nCount = nBytesToRead / nSize;
419 45744 : bEOF = true;
420 : }
421 :
422 6836360 : if (nBytesToRead)
423 6836360 : memcpy(pBuffer, poFile->pabyData + m_nOffset,
424 : static_cast<size_t>(nBytesToRead));
425 6836350 : m_nOffset += nBytesToRead;
426 :
427 6836350 : return nCount;
428 : }
429 :
430 : /************************************************************************/
431 : /* PRead() */
432 : /************************************************************************/
433 :
434 35 : size_t VSIMemHandle::PRead(void *pBuffer, size_t nSize,
435 : vsi_l_offset nOffset) const
436 : {
437 70 : CPL_SHARED_LOCK oLock(poFile->m_oMutex);
438 :
439 35 : if (nOffset < poFile->nLength)
440 : {
441 : const size_t nToCopy = static_cast<size_t>(
442 102 : std::min(static_cast<vsi_l_offset>(poFile->nLength - nOffset),
443 34 : static_cast<vsi_l_offset>(nSize)));
444 34 : memcpy(pBuffer, poFile->pabyData + static_cast<size_t>(nOffset),
445 : nToCopy);
446 34 : return nToCopy;
447 : }
448 1 : return 0;
449 : }
450 :
451 : /************************************************************************/
452 : /* Write() */
453 : /************************************************************************/
454 :
455 1532100 : size_t VSIMemHandle::Write(const void *pBuffer, size_t nSize, size_t nCount)
456 :
457 : {
458 3064190 : CPL_EXCLUSIVE_LOCK oLock(poFile->m_oMutex);
459 :
460 1532100 : if (!bUpdate)
461 : {
462 1 : errno = EACCES;
463 1 : return 0;
464 : }
465 1532100 : if (bExtendFileAtNextWrite)
466 : {
467 40051 : bExtendFileAtNextWrite = false;
468 40051 : if (!poFile->SetLength(m_nOffset))
469 180 : return 0;
470 : }
471 :
472 1531920 : const size_t nBytesToWrite = nSize * nCount;
473 1531920 : if (nCount > 0 && nBytesToWrite / nCount != nSize)
474 : {
475 0 : return 0;
476 : }
477 1531920 : if (nBytesToWrite + m_nOffset < nBytesToWrite)
478 : {
479 0 : return 0;
480 : }
481 :
482 1531920 : if (nBytesToWrite + m_nOffset > poFile->nLength)
483 : {
484 1072410 : if (!poFile->SetLength(nBytesToWrite + m_nOffset))
485 4434 : return 0;
486 : }
487 :
488 1527470 : if (nBytesToWrite)
489 1517470 : memcpy(poFile->pabyData + m_nOffset, pBuffer, nBytesToWrite);
490 : // Coverity seems to be confused by the fact that we access m_nOffset
491 : // under a shared lock in most places, except here under an exclusive lock
492 : // which is fine
493 : // coverity[missing_lock]
494 1527470 : m_nOffset += nBytesToWrite;
495 :
496 1527470 : time(&poFile->mTime);
497 :
498 1527480 : return nCount;
499 : }
500 :
501 : /************************************************************************/
502 : /* Eof() */
503 : /************************************************************************/
504 :
505 147650 : int VSIMemHandle::Eof()
506 :
507 : {
508 147650 : CPL_SHARED_LOCK oLock(poFile->m_oMutex);
509 295300 : return bEOF;
510 : }
511 :
512 : /************************************************************************/
513 : /* Truncate() */
514 : /************************************************************************/
515 :
516 688 : int VSIMemHandle::Truncate(vsi_l_offset nNewSize)
517 : {
518 688 : if (!bUpdate)
519 : {
520 1 : errno = EACCES;
521 1 : return -1;
522 : }
523 :
524 687 : bExtendFileAtNextWrite = false;
525 :
526 1374 : CPL_EXCLUSIVE_LOCK oLock(poFile->m_oMutex);
527 687 : if (poFile->SetLength(nNewSize))
528 667 : return 0;
529 :
530 20 : return -1;
531 : }
532 :
533 : /************************************************************************/
534 : /* ==================================================================== */
535 : /* VSIMemFilesystemHandler */
536 : /* ==================================================================== */
537 : /************************************************************************/
538 :
539 : /************************************************************************/
540 : /* ~VSIMemFilesystemHandler() */
541 : /************************************************************************/
542 :
543 1706 : VSIMemFilesystemHandler::~VSIMemFilesystemHandler()
544 :
545 : {
546 853 : oFileList.clear();
547 :
548 853 : if (hMutex != nullptr)
549 206 : CPLDestroyMutex(hMutex);
550 853 : hMutex = nullptr;
551 1706 : }
552 :
553 : /************************************************************************/
554 : /* Open() */
555 : /************************************************************************/
556 :
557 203207 : VSIVirtualHandle *VSIMemFilesystemHandler::Open(const char *pszFilename,
558 : const char *pszAccess,
559 : bool bSetError,
560 : CSLConstList /* papszOptions */)
561 :
562 : {
563 406416 : CPLMutexHolder oHolder(&hMutex);
564 609627 : const CPLString osFilename = NormalizePath(pszFilename);
565 203209 : if (osFilename.empty())
566 0 : return nullptr;
567 :
568 203209 : vsi_l_offset nMaxLength = GUINTBIG_MAX;
569 203209 : const size_t iPos = osFilename.find("||maxlength=");
570 203209 : if (iPos != std::string::npos)
571 : {
572 3270 : nMaxLength = static_cast<vsi_l_offset>(CPLAtoGIntBig(
573 6540 : osFilename.substr(iPos + strlen("||maxlength=")).c_str()));
574 : }
575 :
576 : /* -------------------------------------------------------------------- */
577 : /* Get the filename we are opening, create if needed. */
578 : /* -------------------------------------------------------------------- */
579 406418 : std::shared_ptr<VSIMemFile> poFile = nullptr;
580 203209 : if (oFileList.find(osFilename) != oFileList.end())
581 : {
582 76558 : poFile = oFileList[osFilename];
583 : }
584 :
585 : // If no file and opening in read, error out.
586 530266 : if (strstr(pszAccess, "w") == nullptr &&
587 203209 : strstr(pszAccess, "a") == nullptr && poFile == nullptr)
588 : {
589 52815 : if (bSetError)
590 : {
591 6831 : VSIError(VSIE_FileError, "No such file or directory");
592 : }
593 52815 : errno = ENOENT;
594 52815 : return nullptr;
595 : }
596 :
597 : // Create.
598 150394 : if (poFile == nullptr)
599 : {
600 73836 : const char *pszFileDir = CPLGetPath(osFilename.c_str());
601 73836 : if (VSIMkdirRecursive(pszFileDir, 0755) == -1)
602 : {
603 1 : if (bSetError)
604 : {
605 0 : VSIError(VSIE_FileError,
606 : "Could not create directory %s for writing",
607 : pszFileDir);
608 : }
609 1 : errno = ENOENT;
610 1 : return nullptr;
611 : }
612 :
613 73835 : poFile = std::make_shared<VSIMemFile>();
614 73835 : poFile->osFilename = osFilename;
615 73835 : oFileList[poFile->osFilename] = poFile;
616 : #ifdef DEBUG_VERBOSE
617 : CPLDebug("VSIMEM", "Creating file %s: ref_count=%d", pszFilename,
618 : static_cast<int>(poFile.use_count()));
619 : #endif
620 73835 : poFile->nMaxLength = nMaxLength;
621 : }
622 : // Overwrite
623 76558 : else if (strstr(pszAccess, "w"))
624 : {
625 5563 : CPL_EXCLUSIVE_LOCK oLock(poFile->m_oMutex);
626 5563 : poFile->SetLength(0);
627 5563 : poFile->nMaxLength = nMaxLength;
628 : }
629 :
630 150393 : if (poFile->bIsDirectory)
631 : {
632 947 : errno = EISDIR;
633 947 : return nullptr;
634 : }
635 :
636 : /* -------------------------------------------------------------------- */
637 : /* Setup the file handle on this file. */
638 : /* -------------------------------------------------------------------- */
639 149446 : VSIMemHandle *poHandle = new VSIMemHandle;
640 :
641 149446 : poHandle->poFile = poFile;
642 149446 : poHandle->m_nOffset = 0;
643 149446 : poHandle->bEOF = false;
644 200805 : poHandle->bUpdate = strstr(pszAccess, "w") || strstr(pszAccess, "+") ||
645 51359 : strstr(pszAccess, "a");
646 :
647 : #ifdef DEBUG_VERBOSE
648 : CPLDebug("VSIMEM", "Opening handle %p on %s: ref_count=%d", poHandle,
649 : pszFilename, static_cast<int>(poFile.use_count()));
650 : #endif
651 149446 : if (strstr(pszAccess, "a"))
652 : {
653 51 : CPL_SHARED_LOCK oLock(poFile->m_oMutex);
654 51 : poHandle->m_nOffset = poFile->nLength;
655 : }
656 :
657 149446 : return poHandle;
658 : }
659 :
660 : /************************************************************************/
661 : /* Stat() */
662 : /************************************************************************/
663 :
664 398789 : int VSIMemFilesystemHandler::Stat(const char *pszFilename,
665 : VSIStatBufL *pStatBuf, int /* nFlags */)
666 :
667 : {
668 797578 : CPLMutexHolder oHolder(&hMutex);
669 :
670 1196370 : const CPLString osFilename = NormalizePath(pszFilename);
671 :
672 398789 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
673 :
674 398789 : if (osFilename + '/' == m_osPrefix || osFilename == m_osPrefix)
675 : {
676 26956 : pStatBuf->st_size = 0;
677 26956 : pStatBuf->st_mode = S_IFDIR;
678 26956 : return 0;
679 : }
680 :
681 371833 : auto oIter = oFileList.find(osFilename);
682 371833 : if (oIter == oFileList.end())
683 : {
684 271246 : errno = ENOENT;
685 271246 : return -1;
686 : }
687 :
688 201174 : std::shared_ptr<VSIMemFile> poFile = oIter->second;
689 :
690 100587 : memset(pStatBuf, 0, sizeof(VSIStatBufL));
691 :
692 100587 : CPL_SHARED_LOCK oLock(poFile->m_oMutex);
693 100587 : if (poFile->bIsDirectory)
694 : {
695 61432 : pStatBuf->st_size = 0;
696 61432 : pStatBuf->st_mode = S_IFDIR;
697 : }
698 : else
699 : {
700 39155 : pStatBuf->st_size = poFile->nLength;
701 39155 : pStatBuf->st_mode = S_IFREG;
702 39155 : pStatBuf->st_mtime = poFile->mTime;
703 : }
704 :
705 100587 : return 0;
706 : }
707 :
708 : /************************************************************************/
709 : /* Unlink() */
710 : /************************************************************************/
711 :
712 88541 : int VSIMemFilesystemHandler::Unlink(const char *pszFilename)
713 :
714 : {
715 177084 : CPLMutexHolder oHolder(&hMutex);
716 177086 : return Unlink_unlocked(pszFilename);
717 : }
718 :
719 : /************************************************************************/
720 : /* Unlink_unlocked() */
721 : /************************************************************************/
722 :
723 97747 : int VSIMemFilesystemHandler::Unlink_unlocked(const char *pszFilename)
724 :
725 : {
726 293241 : const CPLString osFilename = NormalizePath(pszFilename);
727 :
728 97747 : auto oIter = oFileList.find(osFilename);
729 97747 : if (oIter == oFileList.end())
730 : {
731 16652 : errno = ENOENT;
732 16652 : return -1;
733 : }
734 :
735 : #ifdef DEBUG_VERBOSE
736 : std::shared_ptr<VSIMemFile> poFile = oIter->second;
737 : CPLDebug("VSIMEM", "Unlink %s: ref_count=%d (before)", pszFilename,
738 : static_cast<int>(poFile.use_count()));
739 : #endif
740 81095 : oFileList.erase(oIter);
741 :
742 81095 : return 0;
743 : }
744 :
745 : /************************************************************************/
746 : /* Mkdir() */
747 : /************************************************************************/
748 :
749 4511 : int VSIMemFilesystemHandler::Mkdir(const char *pszPathname, long /* nMode */)
750 :
751 : {
752 9022 : CPLMutexHolder oHolder(&hMutex);
753 :
754 13533 : const CPLString osPathname = NormalizePath(pszPathname);
755 :
756 4511 : if (oFileList.find(osPathname) != oFileList.end())
757 : {
758 147 : errno = EEXIST;
759 147 : return -1;
760 : }
761 :
762 4364 : std::shared_ptr<VSIMemFile> poFile = std::make_shared<VSIMemFile>();
763 4364 : poFile->osFilename = osPathname;
764 4364 : poFile->bIsDirectory = true;
765 4364 : oFileList[osPathname] = poFile;
766 : #ifdef DEBUG_VERBOSE
767 : CPLDebug("VSIMEM", "Mkdir on %s: ref_count=%d", pszPathname,
768 : static_cast<int>(poFile.use_count()));
769 : #endif
770 4364 : CPL_IGNORE_RET_VAL(poFile);
771 4364 : return 0;
772 : }
773 :
774 : /************************************************************************/
775 : /* Rmdir() */
776 : /************************************************************************/
777 :
778 3713 : int VSIMemFilesystemHandler::Rmdir(const char *pszPathname)
779 :
780 : {
781 3713 : return Unlink(pszPathname);
782 : }
783 :
784 : /************************************************************************/
785 : /* ReadDirEx() */
786 : /************************************************************************/
787 :
788 26438 : char **VSIMemFilesystemHandler::ReadDirEx(const char *pszPath, int nMaxFiles)
789 :
790 : {
791 52876 : CPLMutexHolder oHolder(&hMutex);
792 :
793 52876 : const CPLString osPath = NormalizePath(pszPath);
794 :
795 26438 : char **papszDir = nullptr;
796 26438 : size_t nPathLen = osPath.size();
797 :
798 26438 : if (nPathLen > 0 && osPath.back() == '/')
799 0 : nPathLen--;
800 :
801 : // In case of really big number of files in the directory, CSLAddString
802 : // can be slow (see #2158). We then directly build the list.
803 26438 : int nItems = 0;
804 26438 : int nAllocatedItems = 0;
805 :
806 1566230 : for (const auto &iter : oFileList)
807 : {
808 1539790 : const char *pszFilePath = iter.second->osFilename.c_str();
809 1539790 : if (EQUALN(osPath, pszFilePath, nPathLen) &&
810 2109920 : pszFilePath[nPathLen] == '/' &&
811 570130 : strstr(pszFilePath + nPathLen + 1, "/") == nullptr)
812 : {
813 367589 : if (nItems == 0)
814 : {
815 24495 : papszDir = static_cast<char **>(CPLCalloc(2, sizeof(char *)));
816 24495 : nAllocatedItems = 1;
817 : }
818 343094 : else if (nItems >= nAllocatedItems)
819 : {
820 78566 : nAllocatedItems = nAllocatedItems * 2;
821 78566 : papszDir = static_cast<char **>(CPLRealloc(
822 78566 : papszDir, (nAllocatedItems + 2) * sizeof(char *)));
823 : }
824 :
825 367589 : papszDir[nItems] = CPLStrdup(pszFilePath + nPathLen + 1);
826 367589 : papszDir[nItems + 1] = nullptr;
827 :
828 367589 : nItems++;
829 367589 : if (nMaxFiles > 0 && nItems > nMaxFiles)
830 1 : break;
831 : }
832 : }
833 :
834 52876 : return papszDir;
835 : }
836 :
837 : /************************************************************************/
838 : /* Rename() */
839 : /************************************************************************/
840 :
841 162 : int VSIMemFilesystemHandler::Rename(const char *pszOldPath,
842 : const char *pszNewPath)
843 :
844 : {
845 324 : CPLMutexHolder oHolder(&hMutex);
846 :
847 486 : const CPLString osOldPath = NormalizePath(pszOldPath);
848 486 : const CPLString osNewPath = NormalizePath(pszNewPath);
849 162 : if (!STARTS_WITH(pszNewPath, m_osPrefix.c_str()))
850 3 : return -1;
851 :
852 159 : if (osOldPath.compare(osNewPath) == 0)
853 0 : return 0;
854 :
855 159 : if (oFileList.find(osOldPath) == oFileList.end())
856 : {
857 0 : errno = ENOENT;
858 0 : return -1;
859 : }
860 :
861 : std::map<CPLString, std::shared_ptr<VSIMemFile>>::iterator it =
862 159 : oFileList.find(osOldPath);
863 502 : while (it != oFileList.end() && it->first.ifind(osOldPath) == 0)
864 : {
865 686 : const CPLString osRemainder = it->first.substr(osOldPath.size());
866 343 : if (osRemainder.empty() || osRemainder[0] == '/')
867 : {
868 444 : const CPLString osNewFullPath = osNewPath + osRemainder;
869 222 : Unlink_unlocked(osNewFullPath);
870 222 : oFileList[osNewFullPath] = it->second;
871 222 : it->second->osFilename = osNewFullPath;
872 222 : oFileList.erase(it++);
873 : }
874 : else
875 : {
876 121 : ++it;
877 : }
878 : }
879 :
880 159 : return 0;
881 : }
882 :
883 : /************************************************************************/
884 : /* NormalizePath() */
885 : /************************************************************************/
886 :
887 775676 : std::string VSIMemFilesystemHandler::NormalizePath(const std::string &in)
888 : {
889 1551350 : CPLString s(in);
890 775660 : std::replace(s.begin(), s.end(), '\\', '/');
891 775680 : s.replaceAll("//", '/');
892 775672 : if (!s.empty() && s.back() == '/')
893 1533 : s.resize(s.size() - 1);
894 : #if __GNUC__ >= 13
895 : // gcc 13 complains about below explicit std::move()
896 : return s;
897 : #else
898 : // Android NDK (and probably other compilers) warn about
899 : // "warning: local variable 's' will be copied despite being returned by name [-Wreturn-std-move]"
900 : // if not specifying std::move()
901 1551340 : return std::move(s);
902 : #endif
903 : }
904 :
905 : /************************************************************************/
906 : /* GetDiskFreeSpace() */
907 : /************************************************************************/
908 :
909 70 : GIntBig VSIMemFilesystemHandler::GetDiskFreeSpace(const char * /*pszDirname*/)
910 : {
911 70 : const GIntBig nRet = CPLGetUsablePhysicalRAM();
912 70 : if (nRet <= 0)
913 0 : return -1;
914 70 : return nRet;
915 : }
916 :
917 : //! @endcond
918 :
919 : /************************************************************************/
920 : /* VSIInstallMemFileHandler() */
921 : /************************************************************************/
922 :
923 : /**
924 : * \brief Install "memory" file system handler.
925 : *
926 : * A special file handler is installed that allows block of memory to be
927 : * treated as files. All portions of the file system underneath the base
928 : * path "/vsimem/" will be handled by this driver.
929 : *
930 : * Normal VSI*L functions can be used freely to create and destroy memory
931 : * arrays treating them as if they were real file system objects. Some
932 : * additional methods exist to efficient create memory file system objects
933 : * without duplicating original copies of the data or to "steal" the block
934 : * of memory associated with a memory file.
935 : *
936 : * Directory related functions are supported.
937 : *
938 : * This code example demonstrates using GDAL to translate from one memory
939 : * buffer to another.
940 : *
941 : * \code
942 : * GByte *ConvertBufferFormat( GByte *pabyInData, vsi_l_offset nInDataLength,
943 : * vsi_l_offset *pnOutDataLength )
944 : * {
945 : * // create memory file system object from buffer.
946 : * VSIFCloseL( VSIFileFromMemBuffer( "/vsimem/work.dat", pabyInData,
947 : * nInDataLength, FALSE ) );
948 : *
949 : * // Open memory buffer for read.
950 : * GDALDatasetH hDS = GDALOpen( "/vsimem/work.dat", GA_ReadOnly );
951 : *
952 : * // Get output format driver.
953 : * GDALDriverH hDriver = GDALGetDriverByName( "GTiff" );
954 : * GDALDatasetH hOutDS;
955 : *
956 : * hOutDS = GDALCreateCopy( hDriver, "/vsimem/out.tif", hDS, TRUE, NULL,
957 : * NULL, NULL );
958 : *
959 : * // close source file, and "unlink" it.
960 : * GDALClose( hDS );
961 : * VSIUnlink( "/vsimem/work.dat" );
962 : *
963 : * // seize the buffer associated with the output file.
964 : *
965 : * return VSIGetMemFileBuffer( "/vsimem/out.tif", pnOutDataLength, TRUE );
966 : * }
967 : * \endcode
968 : */
969 :
970 1228 : void VSIInstallMemFileHandler()
971 : {
972 1228 : VSIFileManager::InstallHandler("/vsimem/",
973 1228 : new VSIMemFilesystemHandler("/vsimem/"));
974 1228 : }
975 :
976 : /************************************************************************/
977 : /* VSIFileFromMemBuffer() */
978 : /************************************************************************/
979 :
980 : /**
981 : * \brief Create memory "file" from a buffer.
982 : *
983 : * A virtual memory file is created from the passed buffer with the indicated
984 : * filename. Under normal conditions the filename would need to be absolute
985 : * and within the /vsimem/ portion of the filesystem.
986 : * Starting with GDAL 3.6, nullptr can also be passed as pszFilename to mean
987 : * an anonymous file, that is destroyed when the handle is closed.
988 : *
989 : * If bTakeOwnership is TRUE, then the memory file system handler will take
990 : * ownership of the buffer, freeing it when the file is deleted. Otherwise
991 : * it remains the responsibility of the caller, but should not be freed as
992 : * long as it might be accessed as a file. In no circumstances does this
993 : * function take a copy of the pabyData contents.
994 : *
995 : * @param pszFilename the filename to be created, or nullptr
996 : * @param pabyData the data buffer for the file.
997 : * @param nDataLength the length of buffer in bytes.
998 : * @param bTakeOwnership TRUE to transfer "ownership" of buffer or FALSE.
999 : *
1000 : * @return open file handle on created file (see VSIFOpenL()).
1001 : */
1002 :
1003 9212 : VSILFILE *VSIFileFromMemBuffer(const char *pszFilename, GByte *pabyData,
1004 : vsi_l_offset nDataLength, int bTakeOwnership)
1005 :
1006 : {
1007 9212 : if (VSIFileManager::GetHandler("") ==
1008 9212 : VSIFileManager::GetHandler("/vsimem/"))
1009 0 : VSIInstallMemFileHandler();
1010 :
1011 : VSIMemFilesystemHandler *poHandler = static_cast<VSIMemFilesystemHandler *>(
1012 9212 : VSIFileManager::GetHandler("/vsimem/"));
1013 :
1014 : const CPLString osFilename =
1015 27408 : pszFilename ? VSIMemFilesystemHandler::NormalizePath(pszFilename)
1016 18424 : : std::string();
1017 9212 : if (osFilename == "/vsimem/")
1018 : {
1019 1 : CPLDebug("VSIMEM", "VSIFileFromMemBuffer(): illegal filename: %s",
1020 : pszFilename);
1021 1 : return nullptr;
1022 : }
1023 :
1024 : // Try to create the parent directory, if needed, before taking
1025 : // ownership of pabyData.
1026 9211 : if (!osFilename.empty())
1027 : {
1028 8982 : const char *pszFileDir = CPLGetPath(osFilename.c_str());
1029 8982 : if (VSIMkdirRecursive(pszFileDir, 0755) == -1)
1030 : {
1031 0 : VSIError(VSIE_FileError,
1032 : "Could not create directory %s for writing", pszFileDir);
1033 0 : errno = ENOENT;
1034 0 : return nullptr;
1035 : }
1036 : }
1037 :
1038 9211 : std::shared_ptr<VSIMemFile> poFile = std::make_shared<VSIMemFile>();
1039 :
1040 9211 : poFile->osFilename = osFilename;
1041 9211 : poFile->bOwnData = CPL_TO_BOOL(bTakeOwnership);
1042 9211 : poFile->pabyData = pabyData;
1043 9211 : poFile->nLength = nDataLength;
1044 9211 : poFile->nAllocLength = nDataLength;
1045 :
1046 9211 : if (!osFilename.empty())
1047 : {
1048 17964 : CPLMutexHolder oHolder(&poHandler->hMutex);
1049 8982 : poHandler->Unlink_unlocked(osFilename);
1050 8982 : poHandler->oFileList[poFile->osFilename] = poFile;
1051 : #ifdef DEBUG_VERBOSE
1052 : CPLDebug("VSIMEM", "VSIFileFromMemBuffer() %s: ref_count=%d (after)",
1053 : poFile->osFilename.c_str(),
1054 : static_cast<int>(poFile.use_count()));
1055 : #endif
1056 : }
1057 :
1058 : /* -------------------------------------------------------------------- */
1059 : /* Setup the file handle on this file. */
1060 : /* -------------------------------------------------------------------- */
1061 9211 : VSIMemHandle *poHandle = new VSIMemHandle;
1062 :
1063 9211 : poHandle->poFile = std::move(poFile);
1064 9211 : poHandle->bUpdate = true;
1065 9211 : return poHandle;
1066 : }
1067 :
1068 : /************************************************************************/
1069 : /* VSIGetMemFileBuffer() */
1070 : /************************************************************************/
1071 :
1072 : /**
1073 : * \brief Fetch buffer underlying memory file.
1074 : *
1075 : * This function returns a pointer to the memory buffer underlying a
1076 : * virtual "in memory" file. If bUnlinkAndSeize is TRUE the filesystem
1077 : * object will be deleted, and ownership of the buffer will pass to the
1078 : * caller otherwise the underlying file will remain in existence.
1079 : *
1080 : * @param pszFilename the name of the file to grab the buffer of.
1081 : * @param pnDataLength (file) length returned in this variable.
1082 : * @param bUnlinkAndSeize TRUE to remove the file, or FALSE to leave unaltered.
1083 : *
1084 : * @return pointer to memory buffer or NULL on failure.
1085 : */
1086 :
1087 35675 : GByte *VSIGetMemFileBuffer(const char *pszFilename, vsi_l_offset *pnDataLength,
1088 : int bUnlinkAndSeize)
1089 :
1090 : {
1091 : VSIMemFilesystemHandler *poHandler = static_cast<VSIMemFilesystemHandler *>(
1092 35675 : VSIFileManager::GetHandler("/vsimem/"));
1093 :
1094 35677 : if (pszFilename == nullptr)
1095 0 : return nullptr;
1096 :
1097 : const CPLString osFilename =
1098 107029 : VSIMemFilesystemHandler::NormalizePath(pszFilename);
1099 :
1100 71337 : CPLMutexHolder oHolder(&poHandler->hMutex);
1101 :
1102 35679 : if (poHandler->oFileList.find(osFilename) == poHandler->oFileList.end())
1103 126 : return nullptr;
1104 :
1105 35553 : std::shared_ptr<VSIMemFile> poFile = poHandler->oFileList[osFilename];
1106 35553 : GByte *pabyData = poFile->pabyData;
1107 35553 : if (pnDataLength != nullptr)
1108 32840 : *pnDataLength = poFile->nLength;
1109 :
1110 35553 : if (bUnlinkAndSeize)
1111 : {
1112 4399 : if (!poFile->bOwnData)
1113 0 : CPLDebug("VSIMemFile",
1114 : "File doesn't own data in VSIGetMemFileBuffer!");
1115 : else
1116 4399 : poFile->bOwnData = false;
1117 :
1118 4399 : poHandler->oFileList.erase(poHandler->oFileList.find(osFilename));
1119 : #ifdef DEBUG_VERBOSE
1120 : CPLDebug("VSIMEM", "VSIGetMemFileBuffer() %s: ref_count=%d (before)",
1121 : poFile->osFilename.c_str(),
1122 : static_cast<int>(poFile.use_count()));
1123 : #endif
1124 4399 : poFile->pabyData = nullptr;
1125 4399 : poFile->nLength = 0;
1126 4399 : poFile->nAllocLength = 0;
1127 : }
1128 :
1129 35553 : return pabyData;
1130 : }
|
