Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: VSI Virtual File System
4 : * Purpose: Implementation VSI*L File API and other file system access
5 : * methods going through file virtualization.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
10 : * Copyright (c) 2008-2014, Even Rouault <even dot rouault at spatialys.com>
11 : *
12 : * SPDX-License-Identifier: MIT
13 : ****************************************************************************/
14 :
15 : #include "cpl_port.h"
16 : #include "cpl_vsi.h"
17 :
18 : #include <cassert>
19 : #include <cinttypes>
20 : #include <cstdarg>
21 : #include <cstddef>
22 : #include <cstring>
23 : #if HAVE_FCNTL_H
24 : #include <fcntl.h>
25 : #endif
26 :
27 : #include <algorithm>
28 : #include <limits>
29 : #include <map>
30 : #include <memory>
31 : #include <mutex>
32 : #include <set>
33 : #include <string>
34 : #include <utility>
35 : #include <vector>
36 :
37 : #include "cpl_conv.h"
38 : #include "cpl_error.h"
39 : #include "cpl_multiproc.h"
40 : #include "cpl_string.h"
41 : #include "cpl_vsi_virtual.h"
42 : #include "cpl_vsil_curl_class.h"
43 :
44 : // To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
45 : #ifdef GetDiskFreeSpace
46 : #undef GetDiskFreeSpace
47 : #endif
48 :
49 : /************************************************************************/
50 : /* VSIReadDir() */
51 : /************************************************************************/
52 :
53 : /**
54 : * \brief Read names in a directory.
55 : *
56 : * This function abstracts access to directory contains. It returns a
57 : * list of strings containing the names of files, and directories in this
58 : * directory. The resulting string list becomes the responsibility of the
59 : * application and should be freed with CSLDestroy() when no longer needed.
60 : *
61 : * Note that no error is issued via CPLError() if the directory path is
62 : * invalid, though NULL is returned.
63 : *
64 : * This function used to be known as CPLReadDir(), but the old name is now
65 : * deprecated.
66 : *
67 : * @param pszPath the relative, or absolute path of a directory to read.
68 : * UTF-8 encoded.
69 : * @return The list of entries in the directory, or NULL if the directory
70 : * doesn't exist. Filenames are returned in UTF-8 encoding.
71 : */
72 :
73 14060 : char **VSIReadDir(const char *pszPath)
74 : {
75 14060 : return VSIReadDirEx(pszPath, 0);
76 : }
77 :
78 : /************************************************************************/
79 : /* VSIReadDirEx() */
80 : /************************************************************************/
81 :
82 : /**
83 : * \brief Read names in a directory.
84 : *
85 : * This function abstracts access to directory contains. It returns a
86 : * list of strings containing the names of files, and directories in this
87 : * directory. The resulting string list becomes the responsibility of the
88 : * application and should be freed with CSLDestroy() when no longer needed.
89 : *
90 : * Note that no error is issued via CPLError() if the directory path is
91 : * invalid, though NULL is returned.
92 : *
93 : * If nMaxFiles is set to a positive number, directory listing will stop after
94 : * that limit has been reached. Note that to indicate truncate, at least one
95 : * element more than the nMaxFiles limit will be returned. If CSLCount() on the
96 : * result is lesser or equal to nMaxFiles, then no truncation occurred.
97 : *
98 : * @param pszPath the relative, or absolute path of a directory to read.
99 : * UTF-8 encoded.
100 : * @param nMaxFiles maximum number of files after which to stop, or 0 for no
101 : * limit.
102 : * @return The list of entries in the directory, or NULL if the directory
103 : * doesn't exist. Filenames are returned in UTF-8 encoding.
104 : * @since GDAL 2.1
105 : */
106 :
107 46915 : char **VSIReadDirEx(const char *pszPath, int nMaxFiles)
108 : {
109 46915 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
110 :
111 46915 : return poFSHandler->ReadDirEx(pszPath, nMaxFiles);
112 : }
113 :
114 : /************************************************************************/
115 : /* VSISiblingFiles() */
116 : /************************************************************************/
117 :
118 : /**
119 : * \brief Return related filenames
120 : *
121 : * This function is essentially meant at being used by GDAL internals.
122 : *
123 : * @param pszFilename the path of a filename to inspect
124 : * UTF-8 encoded.
125 : * @return The list of entries, relative to the directory, of all sidecar
126 : * files available or NULL if the list is not known.
127 : * Filenames are returned in UTF-8 encoding.
128 : * Most implementations will return NULL, and a subsequent ReadDir will
129 : * list all files available in the file's directory. This function will be
130 : * overridden by VSI FilesystemHandlers that wish to force e.g. an empty list
131 : * to avoid opening non-existent files on slow filesystems. The return value
132 : * shall be destroyed with CSLDestroy()
133 : * @since GDAL 3.2
134 : */
135 69036 : char **VSISiblingFiles(const char *pszFilename)
136 : {
137 69036 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
138 :
139 69037 : return poFSHandler->SiblingFiles(pszFilename);
140 : }
141 :
142 : /************************************************************************/
143 : /* VSIGetDirectorySeparator() */
144 : /************************************************************************/
145 :
146 : /** Return the directory separator for the specified path.
147 : *
148 : * Default is forward slash. The only exception currently is the Windows
149 : * file system which returns anti-slash, unless the specified path is of the
150 : * form "{drive_letter}:/{rest_of_the_path}".
151 : *
152 : * @since 3.9
153 : */
154 866163 : const char *VSIGetDirectorySeparator(const char *pszPath)
155 : {
156 866163 : if (STARTS_WITH(pszPath, "http://") || STARTS_WITH(pszPath, "https://"))
157 303 : return "/";
158 :
159 865860 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
160 865908 : return poFSHandler->GetDirectorySeparator(pszPath);
161 : }
162 :
163 : /************************************************************************/
164 : /* VSIReadRecursive() */
165 : /************************************************************************/
166 :
167 : /**
168 : * \brief Read names in a directory recursively.
169 : *
170 : * This function abstracts access to directory contents and subdirectories.
171 : * It returns a list of strings containing the names of files and directories
172 : * in this directory and all subdirectories. The resulting string list becomes
173 : * the responsibility of the application and should be freed with CSLDestroy()
174 : * when no longer needed.
175 : *
176 : * Note that no error is issued via CPLError() if the directory path is
177 : * invalid, though NULL is returned.
178 : *
179 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
180 : * longer be always forward slash, but will be the one returned by
181 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
182 : * file systems.
183 : *
184 : * @param pszPathIn the relative, or absolute path of a directory to read.
185 : * UTF-8 encoded.
186 : *
187 : * @return The list of entries in the directory and subdirectories
188 : * or NULL if the directory doesn't exist. Filenames are returned in UTF-8
189 : * encoding.
190 : * @since GDAL 1.10.0
191 : *
192 : */
193 :
194 1245 : char **VSIReadDirRecursive(const char *pszPathIn)
195 : {
196 1245 : const char SEP = VSIGetDirectorySeparator(pszPathIn)[0];
197 :
198 1245 : const char *const apszOptions[] = {"NAME_AND_TYPE_ONLY=YES", nullptr};
199 1245 : VSIDIR *psDir = VSIOpenDir(pszPathIn, -1, apszOptions);
200 1245 : if (!psDir)
201 3 : return nullptr;
202 2484 : CPLStringList oFiles;
203 3839 : while (auto psEntry = VSIGetNextDirEntry(psDir))
204 : {
205 2597 : if (VSI_ISDIR(psEntry->nMode) && psEntry->pszName[0] &&
206 1131 : psEntry->pszName[strlen(psEntry->pszName) - 1] != SEP)
207 : {
208 1131 : oFiles.AddString((std::string(psEntry->pszName) + SEP).c_str());
209 : }
210 : else
211 : {
212 1466 : oFiles.AddString(psEntry->pszName);
213 : }
214 2597 : }
215 1242 : VSICloseDir(psDir);
216 :
217 1242 : return oFiles.StealList();
218 : }
219 :
220 : /************************************************************************/
221 : /* CPLReadDir() */
222 : /* */
223 : /* This is present only to provide ABI compatibility with older */
224 : /* versions. */
225 : /************************************************************************/
226 : #undef CPLReadDir
227 :
228 : CPL_C_START
229 : char CPL_DLL **CPLReadDir(const char *pszPath);
230 : CPL_C_END
231 :
232 0 : char **CPLReadDir(const char *pszPath)
233 : {
234 0 : return VSIReadDir(pszPath);
235 : }
236 :
237 : /************************************************************************/
238 : /* VSIOpenDir() */
239 : /************************************************************************/
240 :
241 : /**
242 : * \brief Open a directory to read its entries.
243 : *
244 : * This function is close to the POSIX opendir() function.
245 : *
246 : * For /vsis3/, /vsigs/, /vsioss/, /vsiaz/ and /vsiadls/, this function has an
247 : * efficient implementation, minimizing the number of network requests, when
248 : * invoked with nRecurseDepth <= 0.
249 : *
250 : * Entries are read by calling VSIGetNextDirEntry() on the handled returned by
251 : * that function, until it returns NULL. VSICloseDir() must be called once done
252 : * with the returned directory handle.
253 : *
254 : * @param pszPath the relative, or absolute path of a directory to read.
255 : * UTF-8 encoded.
256 : * @param nRecurseDepth 0 means do not recurse in subdirectories, 1 means
257 : * recurse only in the first level of subdirectories, etc. -1 means unlimited
258 : * recursion level
259 : * @param papszOptions NULL terminated list of options, or NULL. The following
260 : * options are implemented:
261 : * <ul>
262 : * <li>PREFIX=string: (GDAL >= 3.4) Filter to select filenames only starting
263 : * with the specified prefix. Implemented efficiently for /vsis3/, /vsigs/,
264 : * and /vsiaz/ (but not /vsiadls/)
265 : * </li>
266 : * <li>NAME_AND_TYPE_ONLY=YES/NO: (GDAL >= 3.4) Defaults to NO. If set to YES,
267 : * only the pszName and nMode members of VSIDIR are guaranteed to be set.
268 : * This is implemented efficiently for the Unix virtual file system.
269 : * </li>
270 : * </ul>
271 : *
272 : * @return a handle, or NULL in case of error
273 : * @since GDAL 2.4
274 : *
275 : */
276 :
277 1375 : VSIDIR *VSIOpenDir(const char *pszPath, int nRecurseDepth,
278 : const char *const *papszOptions)
279 : {
280 1375 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
281 :
282 1375 : return poFSHandler->OpenDir(pszPath, nRecurseDepth, papszOptions);
283 : }
284 :
285 : /************************************************************************/
286 : /* VSIGetNextDirEntry() */
287 : /************************************************************************/
288 :
289 : /**
290 : * \brief Return the next entry of the directory
291 : *
292 : * This function is close to the POSIX readdir() function. It actually returns
293 : * more information (file size, last modification time), which on 'real' file
294 : * systems involve one 'stat' call per file.
295 : *
296 : * For filesystems that can have both a regular file and a directory name of
297 : * the same name (typically /vsis3/), when this situation of duplicate happens,
298 : * the directory name will be suffixed by a slash character. Otherwise directory
299 : * names are not suffixed by slash.
300 : *
301 : * The returned entry remains valid until the next call to VSINextDirEntry()
302 : * or VSICloseDir() with the same handle.
303 : *
304 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
305 : * longer be always forward slash, but will be the one returned by
306 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
307 : * file systems.
308 : *
309 : * @param dir Directory handled returned by VSIOpenDir(). Must not be NULL.
310 : *
311 : * @return a entry, or NULL if there is no more entry in the directory. This
312 : * return value must not be freed.
313 : * @since GDAL 2.4
314 : *
315 : */
316 :
317 4317 : const VSIDIREntry *VSIGetNextDirEntry(VSIDIR *dir)
318 : {
319 4317 : return dir->NextDirEntry();
320 : }
321 :
322 : /************************************************************************/
323 : /* VSICloseDir() */
324 : /************************************************************************/
325 :
326 : /**
327 : * \brief Close a directory
328 : *
329 : * This function is close to the POSIX closedir() function.
330 : *
331 : * @param dir Directory handled returned by VSIOpenDir().
332 : *
333 : * @since GDAL 2.4
334 : */
335 :
336 1348 : void VSICloseDir(VSIDIR *dir)
337 : {
338 1348 : delete dir;
339 1348 : }
340 :
341 : /************************************************************************/
342 : /* VSIMkdir() */
343 : /************************************************************************/
344 :
345 : /**
346 : * \brief Create a directory.
347 : *
348 : * Create a new directory with the indicated mode. For POSIX-style systems,
349 : * the mode is modified by the file creation mask (umask). However, some
350 : * file systems and platforms may not use umask, or they may ignore the mode
351 : * completely. So a reasonable cross-platform default mode value is 0755.
352 : *
353 : * Analog of the POSIX mkdir() function.
354 : *
355 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
356 : * @param mode the permissions mode.
357 : *
358 : * @return 0 on success or -1 on an error.
359 : */
360 :
361 92155 : int VSIMkdir(const char *pszPathname, long mode)
362 :
363 : {
364 92155 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPathname);
365 :
366 92155 : return poFSHandler->Mkdir(pszPathname, mode);
367 : }
368 :
369 : /************************************************************************/
370 : /* VSIMkdirRecursive() */
371 : /************************************************************************/
372 :
373 : /**
374 : * \brief Create a directory and all its ancestors
375 : *
376 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
377 : * @param mode the permissions mode.
378 : *
379 : * @return 0 on success or -1 on an error.
380 : * @since GDAL 2.3
381 : */
382 :
383 131841 : int VSIMkdirRecursive(const char *pszPathname, long mode)
384 : {
385 131841 : if (pszPathname == nullptr || pszPathname[0] == '\0' ||
386 131841 : strncmp("/", pszPathname, 2) == 0)
387 : {
388 0 : return -1;
389 : }
390 :
391 263682 : const CPLString osPathname(pszPathname);
392 : VSIStatBufL sStat;
393 131841 : if (VSIStatL(osPathname, &sStat) == 0)
394 : {
395 43871 : return VSI_ISDIR(sStat.st_mode) ? 0 : -1;
396 : }
397 175940 : const CPLString osParentPath(CPLGetPath(osPathname));
398 :
399 : // Prevent crazy paths from recursing forever.
400 175940 : if (osParentPath == osPathname ||
401 87970 : osParentPath.length() >= osPathname.length())
402 : {
403 0 : return -1;
404 : }
405 :
406 87970 : if (VSIStatL(osParentPath, &sStat) != 0)
407 : {
408 44094 : if (VSIMkdirRecursive(osParentPath, mode) != 0)
409 1 : return -1;
410 : }
411 :
412 87969 : return VSIMkdir(osPathname, mode);
413 : }
414 :
415 : /************************************************************************/
416 : /* VSIUnlink() */
417 : /************************************************************************/
418 :
419 : /**
420 : * \brief Delete a file.
421 : *
422 : * Deletes a file object from the file system.
423 : *
424 : * This method goes through the VSIFileHandler virtualization and may
425 : * work on unusual filesystems such as in memory.
426 : *
427 : * Analog of the POSIX unlink() function.
428 : *
429 : * @param pszFilename the path of the file to be deleted. UTF-8 encoded.
430 : *
431 : * @return 0 on success or -1 on an error.
432 : */
433 :
434 103752 : int VSIUnlink(const char *pszFilename)
435 :
436 : {
437 103752 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
438 :
439 103751 : return poFSHandler->Unlink(pszFilename);
440 : }
441 :
442 : /************************************************************************/
443 : /* VSIUnlinkBatch() */
444 : /************************************************************************/
445 :
446 : /**
447 : * \brief Delete several files, possibly in a batch.
448 : *
449 : * All files should belong to the same file system handler.
450 : *
451 : * This is implemented efficiently for /vsis3/ and /vsigs/ (provided for /vsigs/
452 : * that OAuth2 authentication is used).
453 : *
454 : * @param papszFiles NULL terminated list of files. UTF-8 encoded.
455 : *
456 : * @return an array of size CSLCount(papszFiles), whose values are TRUE or FALSE
457 : * depending on the success of deletion of the corresponding file. The array
458 : * should be freed with VSIFree().
459 : * NULL might be return in case of a more general error (for example,
460 : * files belonging to different file system handlers)
461 : *
462 : * @since GDAL 3.1
463 : */
464 :
465 10 : int *VSIUnlinkBatch(CSLConstList papszFiles)
466 : {
467 10 : VSIFilesystemHandler *poFSHandler = nullptr;
468 28 : for (CSLConstList papszIter = papszFiles; papszIter && *papszIter;
469 : ++papszIter)
470 : {
471 19 : auto poFSHandlerThisFile = VSIFileManager::GetHandler(*papszIter);
472 19 : if (poFSHandler == nullptr)
473 9 : poFSHandler = poFSHandlerThisFile;
474 10 : else if (poFSHandler != poFSHandlerThisFile)
475 : {
476 1 : CPLError(CE_Failure, CPLE_AppDefined,
477 : "Files belong to different file system handlers");
478 1 : poFSHandler = nullptr;
479 1 : break;
480 : }
481 : }
482 10 : if (poFSHandler == nullptr)
483 2 : return nullptr;
484 8 : return poFSHandler->UnlinkBatch(papszFiles);
485 : }
486 :
487 : /************************************************************************/
488 : /* VSIRename() */
489 : /************************************************************************/
490 :
491 : /**
492 : * \brief Rename a file.
493 : *
494 : * Renames a file object in the file system. It should be possible
495 : * to rename a file onto a new filesystem, but it is safest if this
496 : * function is only used to rename files that remain in the same directory.
497 : *
498 : * This method goes through the VSIFileHandler virtualization and may
499 : * work on unusual filesystems such as in memory.
500 : *
501 : * Analog of the POSIX rename() function.
502 : *
503 : * @param oldpath the name of the file to be renamed. UTF-8 encoded.
504 : * @param newpath the name the file should be given. UTF-8 encoded.
505 : *
506 : * @return 0 on success or -1 on an error.
507 : */
508 :
509 640 : int VSIRename(const char *oldpath, const char *newpath)
510 :
511 : {
512 640 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(oldpath);
513 :
514 640 : return poFSHandler->Rename(oldpath, newpath);
515 : }
516 :
517 : /************************************************************************/
518 : /* VSICopyFile() */
519 : /************************************************************************/
520 :
521 : /**
522 : * \brief Copy a source file into a target file.
523 : *
524 : * For a /vsizip/foo.zip/bar target, the options available are those of
525 : * CPLAddFileInZip()
526 : *
527 : * The following copies are made fully on the target server, without local
528 : * download from source and upload to target:
529 : * - /vsis3/ -> /vsis3/
530 : * - /vsigs/ -> /vsigs/
531 : * - /vsiaz/ -> /vsiaz/
532 : * - /vsiadls/ -> /vsiadls/
533 : * - any of the above or /vsicurl/ -> /vsiaz/ (starting with GDAL 3.8)
534 : *
535 : * @param pszSource Source filename. UTF-8 encoded. May be NULL if fpSource is
536 : * not NULL.
537 : * @param pszTarget Target filename. UTF-8 encoded. Must not be NULL
538 : * @param fpSource File handle on the source file. May be NULL if pszSource is
539 : * not NULL.
540 : * @param nSourceSize Size of the source file. Pass -1 if unknown.
541 : * If set to -1, and progress callback is used, VSIStatL() will be used on
542 : * pszSource to retrieve the source size.
543 : * @param papszOptions Null terminated list of options, or NULL.
544 : * @param pProgressFunc Progress callback, or NULL.
545 : * @param pProgressData User data of progress callback, or NULL.
546 : *
547 : * @return 0 on success.
548 : * @since GDAL 3.7
549 : */
550 :
551 2229 : int VSICopyFile(const char *pszSource, const char *pszTarget,
552 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
553 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
554 : void *pProgressData)
555 :
556 : {
557 2229 : if (!pszSource && !fpSource)
558 : {
559 1 : CPLError(CE_Failure, CPLE_AppDefined,
560 : "pszSource == nullptr && fpSource == nullptr");
561 1 : return -1;
562 : }
563 2228 : if (!pszTarget || pszTarget[0] == '\0')
564 : {
565 0 : return -1;
566 : }
567 :
568 : VSIFilesystemHandler *poFSHandlerTarget =
569 2228 : VSIFileManager::GetHandler(pszTarget);
570 2228 : return poFSHandlerTarget->CopyFile(pszSource, pszTarget, fpSource,
571 : nSourceSize, papszOptions, pProgressFunc,
572 2228 : pProgressData);
573 : }
574 :
575 : /************************************************************************/
576 : /* VSICopyFileRestartable() */
577 : /************************************************************************/
578 :
579 : /**
580 : \brief Copy a source file into a target file in a way that can (potentially)
581 : be restarted.
582 :
583 : This function provides the possibility of efficiently restarting upload of
584 : large files to cloud storage that implements upload in a chunked way,
585 : such as /vsis3/ and /vsigs/.
586 : For other destination file systems, this function may fallback to
587 : VSICopyFile() and not provide any smart restartable implementation.
588 :
589 : Example of a potential workflow:
590 :
591 : @code{.cpp}
592 : char* pszOutputPayload = NULL;
593 : int ret = VSICopyFileRestartable(pszSource, pszTarget, NULL,
594 : &pszOutputPayload, NULL, NULL, NULL);
595 : while( ret == 1 ) // add also a limiting counter to avoid potentiall endless looping
596 : {
597 : // TODO: wait for some time
598 :
599 : char* pszOutputPayloadNew = NULL;
600 : const char* pszInputPayload = pszOutputPayload;
601 : ret = VSICopyFileRestartable(pszSource, pszTarget, pszInputPayload,
602 : &pszOutputPayloadNew, NULL, NULL, NULL);
603 : VSIFree(pszOutputPayload);
604 : pszOutputPayload = pszOutputPayloadNew;
605 : }
606 : VSIFree(pszOutputPayload);
607 : @endcode
608 :
609 : @param pszSource Source filename. UTF-8 encoded. Must not be NULL
610 : @param pszTarget Target filename. UTF-8 encoded. Must not be NULL
611 : @param pszInputPayload NULL at the first invocation. When doing a retry,
612 : should be the content of *ppszOutputPayload from a
613 : previous invocation.
614 : @param[out] ppszOutputPayload Pointer to an output string that will be set to
615 : a value that can be provided as pszInputPayload
616 : for a next call to VSICopyFileRestartable().
617 : ppszOutputPayload must not be NULL.
618 : The string set in *ppszOutputPayload, if not NULL,
619 : is JSON-encoded, and can be re-used in another
620 : process instance. It must be freed with VSIFree()
621 : when no longer needed.
622 : @param papszOptions Null terminated list of options, or NULL.
623 : Currently accepted options are:
624 : <ul>
625 : <li>NUM_THREADS=integer or ALL_CPUS. Number of threads to use for parallel
626 : file copying. Only use for when /vsis3/, /vsigs/, /vsiaz/ or /vsiadls/ is in
627 : source or target. The default is 10.
628 : </li>
629 : <li>CHUNK_SIZE=integer. Maximum size of chunk (in bytes) to use
630 : to split large objects. For upload to /vsis3/, this chunk size must be set at
631 : least to 5 MB. The default is 50 MB.
632 : </li>
633 : </ul>
634 : @param pProgressFunc Progress callback, or NULL.
635 : @param pProgressData User data of progress callback, or NULL.
636 : @return 0 on success,
637 : -1 on (non-restartable) failure,
638 : 1 if VSICopyFileRestartable() can be called again in a restartable way
639 : @since GDAL 3.10
640 :
641 : @see VSIAbortPendingUploads()
642 : */
643 :
644 20 : int VSICopyFileRestartable(const char *pszSource, const char *pszTarget,
645 : const char *pszInputPayload,
646 : char **ppszOutputPayload,
647 : const char *const *papszOptions,
648 : GDALProgressFunc pProgressFunc, void *pProgressData)
649 :
650 : {
651 20 : if (!pszSource)
652 : {
653 0 : return -1;
654 : }
655 20 : if (!pszTarget || pszTarget[0] == '\0')
656 : {
657 0 : return -1;
658 : }
659 20 : if (!ppszOutputPayload)
660 : {
661 0 : return -1;
662 : }
663 :
664 : VSIFilesystemHandler *poFSHandlerTarget =
665 20 : VSIFileManager::GetHandler(pszTarget);
666 20 : return poFSHandlerTarget->CopyFileRestartable(
667 : pszSource, pszTarget, pszInputPayload, ppszOutputPayload, papszOptions,
668 20 : pProgressFunc, pProgressData);
669 : }
670 :
671 : /************************************************************************/
672 : /* VSISync() */
673 : /************************************************************************/
674 :
675 : /**
676 : * \brief Synchronize a source file/directory with a target file/directory.
677 : *
678 : * This is a analog of the 'rsync' utility. In the current implementation,
679 : * rsync would be more efficient for local file copying, but VSISync() main
680 : * interest is when the source or target is a remote
681 : * file system like /vsis3/ or /vsigs/, in which case it can take into account
682 : * the timestamps of the files (or optionally the ETag/MD5Sum) to avoid
683 : * unneeded copy operations.
684 : *
685 : * This is only implemented efficiently for:
686 : * <ul>
687 : * <li> local filesystem <--> remote filesystem.</li>
688 : * <li> remote filesystem <--> remote filesystem (starting with GDAL 3.1).
689 : * Where the source and target remote filesystems are the same and one of
690 : * /vsis3/, /vsigs/ or /vsiaz/. Or when the target is /vsiaz/ and the source
691 : * is /vsis3/, /vsigs/, /vsiadls/ or /vsicurl/ (starting with GDAL 3.8)</li>
692 : * </ul>
693 : *
694 : * Similarly to rsync behavior, if the source filename ends with a slash,
695 : * it means that the content of the directory must be copied, but not the
696 : * directory name. For example, assuming "/home/even/foo" contains a file "bar",
697 : * VSISync("/home/even/foo/", "/mnt/media", ...) will create a "/mnt/media/bar"
698 : * file. Whereas VSISync("/home/even/foo", "/mnt/media", ...) will create a
699 : * "/mnt/media/foo" directory which contains a bar file.
700 : *
701 : * @param pszSource Source file or directory. UTF-8 encoded.
702 : * @param pszTarget Target file or directory. UTF-8 encoded.
703 : * @param papszOptions Null terminated list of options, or NULL.
704 : * Currently accepted options are:
705 : * <ul>
706 : * <li>RECURSIVE=NO (the default is YES)</li>
707 : * <li>SYNC_STRATEGY=TIMESTAMP/ETAG/OVERWRITE.
708 : *
709 : * Determines which criterion is used to determine if a target file must be
710 : * replaced when it already exists and has the same file size as the source.
711 : * Only applies for a source or target being a network filesystem.
712 : *
713 : * The default is TIMESTAMP (similarly to how 'aws s3 sync' works), that is
714 : * to say that for an upload operation, a remote file is
715 : * replaced if it has a different size or if it is older than the source.
716 : * For a download operation, a local file is replaced if it has a different
717 : * size or if it is newer than the remote file.
718 : *
719 : * The ETAG strategy assumes that the ETag metadata of the remote file is
720 : * the MD5Sum of the file content, which is only true in the case of /vsis3/
721 : * for files not using KMS server side encryption and uploaded in a single
722 : * PUT operation (so smaller than 50 MB given the default used by GDAL).
723 : * Only to be used for /vsis3/, /vsigs/ or other filesystems using a
724 : * MD5Sum as ETAG.
725 : *
726 : * The OVERWRITE strategy (GDAL >= 3.2) will always overwrite the target
727 : * file with the source one.
728 : * </li>
729 : * <li>NUM_THREADS=integer. (GDAL >= 3.1) Number of threads to use for parallel
730 : * file copying. Only use for when /vsis3/, /vsigs/, /vsiaz/ or /vsiadls/ is in
731 : * source or target. The default is 10 since GDAL 3.3</li>
732 : * <li>CHUNK_SIZE=integer. (GDAL >= 3.1) Maximum size of chunk (in bytes) to use
733 : * to split large objects when downloading them from /vsis3/, /vsigs/, /vsiaz/
734 : * or /vsiadls/ to local file system, or for upload to /vsis3/, /vsiaz/ or
735 : * /vsiadls/ from local file system. Only used if NUM_THREADS > 1. For upload to
736 : * /vsis3/, this chunk size must be set at least to 5 MB. The default is 8 MB
737 : * since GDAL 3.3</li> <li>x-amz-KEY=value. (GDAL >= 3.5) MIME header to pass
738 : * during creation of a /vsis3/ object.</li> <li>x-goog-KEY=value. (GDAL >= 3.5)
739 : * MIME header to pass during creation of a /vsigs/ object.</li>
740 : * <li>x-ms-KEY=value. (GDAL >= 3.5) MIME header to pass during creation of a
741 : * /vsiaz/ or /vsiadls/ object.</li>
742 : * </ul>
743 : * @param pProgressFunc Progress callback, or NULL.
744 : * @param pProgressData User data of progress callback, or NULL.
745 : * @param ppapszOutputs Unused. Should be set to NULL for now.
746 : *
747 : * @return TRUE on success or FALSE on an error.
748 : * @since GDAL 2.4
749 : */
750 :
751 45 : int VSISync(const char *pszSource, const char *pszTarget,
752 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
753 : void *pProgressData, char ***ppapszOutputs)
754 :
755 : {
756 45 : if (pszSource[0] == '\0' || pszTarget[0] == '\0')
757 : {
758 0 : return FALSE;
759 : }
760 :
761 : VSIFilesystemHandler *poFSHandlerSource =
762 45 : VSIFileManager::GetHandler(pszSource);
763 : VSIFilesystemHandler *poFSHandlerTarget =
764 45 : VSIFileManager::GetHandler(pszTarget);
765 45 : VSIFilesystemHandler *poFSHandlerLocal = VSIFileManager::GetHandler("");
766 : VSIFilesystemHandler *poFSHandlerMem =
767 45 : VSIFileManager::GetHandler("/vsimem/");
768 45 : VSIFilesystemHandler *poFSHandler = poFSHandlerSource;
769 45 : if (poFSHandlerTarget != poFSHandlerLocal &&
770 : poFSHandlerTarget != poFSHandlerMem)
771 : {
772 22 : poFSHandler = poFSHandlerTarget;
773 : }
774 :
775 90 : return poFSHandler->Sync(pszSource, pszTarget, papszOptions, pProgressFunc,
776 45 : pProgressData, ppapszOutputs)
777 45 : ? TRUE
778 45 : : FALSE;
779 : }
780 :
781 : /************************************************************************/
782 : /* VSIMultipartUploadGetCapabilities() */
783 : /************************************************************************/
784 :
785 : /**
786 : * \brief Return capabilities for multiple part file upload.
787 : *
788 : * @param pszFilename Filename, or virtual file system prefix, onto which
789 : * capabilities should apply.
790 : * @param[out] pbNonSequentialUploadSupported If not null,
791 : * the pointed value is set if parts can be uploaded in a non-sequential way.
792 : * @param[out] pbParallelUploadSupported If not null,
793 : * the pointed value is set if parts can be uploaded in a parallel way.
794 : * (implies *pbNonSequentialUploadSupported = true)
795 : * @param[out] pbAbortSupported If not null,
796 : * the pointed value is set if VSIMultipartUploadAbort() is implemented.
797 : * @param[out] pnMinPartSize If not null, the pointed value is set to the minimum
798 : * size of parts (but the last one), in MiB.
799 : * @param[out] pnMaxPartSize If not null, the pointed value is set to the maximum
800 : * size of parts, in MiB.
801 : * @param[out] pnMaxPartCount If not null, the pointed value is set to the
802 : * maximum number of parts that can be uploaded.
803 : *
804 : * @return TRUE in case of success, FALSE otherwise.
805 : *
806 : * @since 3.10
807 : */
808 7 : int VSIMultipartUploadGetCapabilities(
809 : const char *pszFilename, int *pbNonSequentialUploadSupported,
810 : int *pbParallelUploadSupported, int *pbAbortSupported,
811 : size_t *pnMinPartSize, size_t *pnMaxPartSize, int *pnMaxPartCount)
812 : {
813 7 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
814 :
815 14 : return poFSHandler->MultipartUploadGetCapabilities(
816 : pbNonSequentialUploadSupported, pbParallelUploadSupported,
817 7 : pbAbortSupported, pnMinPartSize, pnMaxPartSize, pnMaxPartCount);
818 : }
819 :
820 : /************************************************************************/
821 : /* VSIMultipartUploadStart() */
822 : /************************************************************************/
823 :
824 : /**
825 : * \brief Initiates the upload a (big) file in a piece-wise way.
826 : *
827 : * Using this API directly is generally not needed, but in very advanced cases,
828 : * as VSIFOpenL(..., "wb") + VSIFWriteL(), VSISync(), VSICopyFile() or
829 : * VSICopyFileRestartable() may be able to leverage it when needed.
830 : *
831 : * This is only implemented for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ and
832 : * /vsioss/ virtual file systems.
833 : *
834 : * The typical workflow is to do :
835 : * - VSIMultipartUploadStart()
836 : * - VSIMultipartUploadAddPart(): several times
837 : * - VSIMultipartUploadEnd()
838 : *
839 : * If VSIMultipartUploadAbort() is supported by the filesystem (VSIMultipartUploadGetCapabilities()
840 : * can be used to determine it), this function should be called to cancel an
841 : * upload. This can be needed to avoid extra billing for some cloud storage
842 : * providers.
843 : *
844 : * The following options are supported:
845 : * <ul>
846 : * <li>MIME headers such as Content-Type and Content-Encoding
847 : * are supported for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ file systems.</li>
848 : * </ul>
849 : *
850 : * @param pszFilename Filename to create
851 : * @param papszOptions NULL or null-terminated list of options.
852 : * @return an upload ID to pass to other VSIMultipartUploadXXXXX() functions,
853 : * and to free with CPLFree() once done, or nullptr in case of error.
854 : *
855 : * @since 3.10
856 : */
857 4 : char *VSIMultipartUploadStart(const char *pszFilename,
858 : CSLConstList papszOptions)
859 : {
860 4 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
861 :
862 4 : return poFSHandler->MultipartUploadStart(pszFilename, papszOptions);
863 : }
864 :
865 : /************************************************************************/
866 : /* VSIMultipartUploadAddPart() */
867 : /************************************************************************/
868 :
869 : /**
870 : * \brief Uploads a new part to a multi-part uploaded file.
871 : *
872 : * Cf VSIMultipartUploadStart().
873 : *
874 : * VSIMultipartUploadGetCapabilities() returns hints on the constraints that
875 : * apply to the upload, in terms of minimum/maximum size of each part, maximum
876 : * number of parts, and whether non-sequential or parallel uploads are
877 : * supported.
878 : *
879 : * @param pszFilename Filename to which to append the new part. Should be the
880 : * same as the one used for VSIMultipartUploadStart()
881 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
882 : * @param nPartNumber Part number, starting at 1.
883 : * @param nFileOffset Offset within the file at which (starts at 0) the passed
884 : * data starts.
885 : * @param pData Pointer to an array of nDataLength bytes.
886 : * @param nDataLength Size in bytes of pData.
887 : * @param papszOptions Unused. Should be nullptr.
888 : *
889 : * @return a part identifier that must be passed into the apszPartIds[] array of
890 : * VSIMultipartUploadEnd(), and to free with CPLFree() once done, or nullptr in
891 : * case of error.
892 : *
893 : * @since 3.10
894 : */
895 5 : char *VSIMultipartUploadAddPart(const char *pszFilename,
896 : const char *pszUploadId, int nPartNumber,
897 : vsi_l_offset nFileOffset, const void *pData,
898 : size_t nDataLength, CSLConstList papszOptions)
899 : {
900 5 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
901 :
902 5 : return poFSHandler->MultipartUploadAddPart(pszFilename, pszUploadId,
903 : nPartNumber, nFileOffset, pData,
904 5 : nDataLength, papszOptions);
905 : }
906 :
907 : /************************************************************************/
908 : /* VSIMultipartUploadEnd() */
909 : /************************************************************************/
910 :
911 : /**
912 : * \brief Completes a multi-part file upload.
913 : *
914 : * Cf VSIMultipartUploadStart().
915 : *
916 : * @param pszFilename Filename for which multipart upload should be completed.
917 : * Should be the same as the one used for
918 : * VSIMultipartUploadStart()
919 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
920 : * @param nPartIdsCount Number of parts, andsize of apszPartIds
921 : * @param apszPartIds Array of part identifiers (as returned by
922 : * VSIMultipartUploadAddPart()), that must be ordered in
923 : * the sequential order of parts, and of size nPartIdsCount.
924 : * @param nTotalSize Total size of the file in bytes (must be equal to the sum
925 : * of nDataLength passed to VSIMultipartUploadAddPart())
926 : * @param papszOptions Unused. Should be nullptr.
927 : *
928 : * @return TRUE in case of success, FALSE in case of failure.
929 : *
930 : * @since 3.10
931 : */
932 5 : int VSIMultipartUploadEnd(const char *pszFilename, const char *pszUploadId,
933 : size_t nPartIdsCount, const char *const *apszPartIds,
934 : vsi_l_offset nTotalSize, CSLConstList papszOptions)
935 : {
936 5 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
937 :
938 10 : return poFSHandler->MultipartUploadEnd(pszFilename, pszUploadId,
939 : nPartIdsCount, apszPartIds,
940 5 : nTotalSize, papszOptions);
941 : }
942 :
943 : /************************************************************************/
944 : /* VSIMultipartUploadAbort() */
945 : /************************************************************************/
946 :
947 : /**
948 : * \brief Aborts a multi-part file upload.
949 : *
950 : * Cf VSIMultipartUploadStart().
951 : *
952 : * This function is not implemented for all virtual file systems.
953 : * Use VSIMultipartUploadGetCapabilities() to determine if it is supported.
954 : *
955 : * This can be needed to avoid extra billing for some cloud storage providers.
956 : *
957 : * @param pszFilename Filename for which multipart upload should be completed.
958 : * Should be the same as the one used for
959 : * VSIMultipartUploadStart()
960 : * @param pszUploadId Value returned by VSIMultipartUploadStart()
961 : * @param papszOptions Unused. Should be nullptr.
962 : *
963 : * @return TRUE in case of success, FALSE in case of failure.
964 : *
965 : * @since 3.10
966 : */
967 6 : int VSIMultipartUploadAbort(const char *pszFilename, const char *pszUploadId,
968 : CSLConstList papszOptions)
969 : {
970 6 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
971 :
972 12 : return poFSHandler->MultipartUploadAbort(pszFilename, pszUploadId,
973 6 : papszOptions);
974 : }
975 :
976 : #ifndef DOXYGEN_SKIP
977 :
978 : /************************************************************************/
979 : /* MultipartUploadGetCapabilities() */
980 : /************************************************************************/
981 :
982 2 : bool VSIFilesystemHandler::MultipartUploadGetCapabilities(int *, int *, int *,
983 : size_t *, size_t *,
984 : int *)
985 : {
986 2 : CPLError(
987 : CE_Failure, CPLE_NotSupported,
988 : "MultipartUploadGetCapabilities() not supported by this file system");
989 2 : return false;
990 : }
991 :
992 : /************************************************************************/
993 : /* MultipartUploadStart() */
994 : /************************************************************************/
995 :
996 1 : char *VSIFilesystemHandler::MultipartUploadStart(const char *, CSLConstList)
997 : {
998 1 : CPLError(CE_Failure, CPLE_NotSupported,
999 : "MultipartUploadStart() not supported by this file system");
1000 1 : return nullptr;
1001 : }
1002 :
1003 : /************************************************************************/
1004 : /* MultipartUploadAddPart() */
1005 : /************************************************************************/
1006 :
1007 1 : char *VSIFilesystemHandler::MultipartUploadAddPart(const char *, const char *,
1008 : int, vsi_l_offset,
1009 : const void *, size_t,
1010 : CSLConstList)
1011 : {
1012 1 : CPLError(CE_Failure, CPLE_NotSupported,
1013 : "MultipartUploadAddPart() not supported by this file system");
1014 1 : return nullptr;
1015 : }
1016 :
1017 : /************************************************************************/
1018 : /* MultipartUploadEnd() */
1019 : /************************************************************************/
1020 :
1021 1 : bool VSIFilesystemHandler::MultipartUploadEnd(const char *, const char *,
1022 : size_t, const char *const *,
1023 : vsi_l_offset, CSLConstList)
1024 : {
1025 1 : CPLError(CE_Failure, CPLE_NotSupported,
1026 : "MultipartUploadEnd() not supported by this file system");
1027 1 : return FALSE;
1028 : }
1029 :
1030 : /************************************************************************/
1031 : /* MultipartUploadAbort() */
1032 : /************************************************************************/
1033 :
1034 1 : bool VSIFilesystemHandler::MultipartUploadAbort(const char *, const char *,
1035 : CSLConstList)
1036 : {
1037 1 : CPLError(CE_Failure, CPLE_NotSupported,
1038 : "MultipartUploadAbort() not supported by this file system");
1039 1 : return FALSE;
1040 : }
1041 :
1042 : #endif
1043 :
1044 : /************************************************************************/
1045 : /* VSIAbortPendingUploads() */
1046 : /************************************************************************/
1047 :
1048 : /**
1049 : * \brief Abort all ongoing multi-part uploads.
1050 : *
1051 : * Abort ongoing multi-part uploads on AWS S3 and Google Cloud Storage. This
1052 : * can be used in case a process doing such uploads was killed in a unclean way.
1053 : *
1054 : * This can be needed to avoid extra billing for some cloud storage providers.
1055 : *
1056 : * Without effect on other virtual file systems.
1057 : *
1058 : * VSIMultipartUploadAbort() can also be used to cancel a given upload, if the
1059 : * upload ID is known.
1060 : *
1061 : * @param pszFilename filename or prefix of a directory into which multipart
1062 : * uploads must be aborted. This can be the root directory of a bucket. UTF-8
1063 : * encoded.
1064 : *
1065 : * @return TRUE on success or FALSE on an error.
1066 : * @since GDAL 3.4
1067 : */
1068 :
1069 1 : int VSIAbortPendingUploads(const char *pszFilename)
1070 : {
1071 1 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1072 :
1073 1 : return poFSHandler->AbortPendingUploads(pszFilename);
1074 : }
1075 :
1076 : /************************************************************************/
1077 : /* VSIRmdir() */
1078 : /************************************************************************/
1079 :
1080 : /**
1081 : * \brief Delete a directory.
1082 : *
1083 : * Deletes a directory object from the file system. On some systems
1084 : * the directory must be empty before it can be deleted.
1085 : *
1086 : * This method goes through the VSIFileHandler virtualization and may
1087 : * work on unusual filesystems such as in memory.
1088 : *
1089 : * Analog of the POSIX rmdir() function.
1090 : *
1091 : * @param pszDirname the path of the directory to be deleted. UTF-8 encoded.
1092 : *
1093 : * @return 0 on success or -1 on an error.
1094 : */
1095 :
1096 319 : int VSIRmdir(const char *pszDirname)
1097 :
1098 : {
1099 319 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
1100 :
1101 319 : return poFSHandler->Rmdir(pszDirname);
1102 : }
1103 :
1104 : /************************************************************************/
1105 : /* VSIRmdirRecursive() */
1106 : /************************************************************************/
1107 :
1108 : /**
1109 : * \brief Delete a directory recursively
1110 : *
1111 : * Deletes a directory object and its content from the file system.
1112 : *
1113 : * Starting with GDAL 3.1, /vsis3/ has an efficient implementation of this
1114 : * function.
1115 : * Starting with GDAL 3.4, /vsigs/ has an efficient implementation of this
1116 : * function, provided that OAuth2 authentication is used.
1117 : *
1118 : * @return 0 on success or -1 on an error.
1119 : * @since GDAL 2.3
1120 : */
1121 :
1122 2728 : int VSIRmdirRecursive(const char *pszDirname)
1123 : {
1124 2728 : if (pszDirname == nullptr || pszDirname[0] == '\0' ||
1125 2728 : strncmp("/", pszDirname, 2) == 0)
1126 : {
1127 0 : return -1;
1128 : }
1129 2728 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
1130 2728 : return poFSHandler->RmdirRecursive(pszDirname);
1131 : }
1132 :
1133 : /************************************************************************/
1134 : /* VSIStatL() */
1135 : /************************************************************************/
1136 :
1137 : /**
1138 : * \brief Get filesystem object info.
1139 : *
1140 : * Fetches status information about a filesystem object (file, directory, etc).
1141 : * The returned information is placed in the VSIStatBufL structure. For
1142 : * portability, only use the st_size (size in bytes) and st_mode (file type).
1143 : * This method is similar to VSIStat(), but will work on large files on
1144 : * systems where this requires special calls.
1145 : *
1146 : * This method goes through the VSIFileHandler virtualization and may
1147 : * work on unusual filesystems such as in memory.
1148 : *
1149 : * Analog of the POSIX stat() function.
1150 : *
1151 : * @param pszFilename the path of the filesystem object to be queried.
1152 : * UTF-8 encoded.
1153 : * @param psStatBuf the structure to load with information.
1154 : *
1155 : * @return 0 on success or -1 on an error.
1156 : */
1157 :
1158 411474 : int VSIStatL(const char *pszFilename, VSIStatBufL *psStatBuf)
1159 :
1160 : {
1161 411474 : return VSIStatExL(pszFilename, psStatBuf, 0);
1162 : }
1163 :
1164 : /************************************************************************/
1165 : /* VSIStatExL() */
1166 : /************************************************************************/
1167 :
1168 : /**
1169 : * \brief Get filesystem object info.
1170 : *
1171 : * Fetches status information about a filesystem object (file, directory, etc).
1172 : * The returned information is placed in the VSIStatBufL structure. For
1173 : * portability, only use the st_size (size in bytes) and st_mode (file type).
1174 : * This method is similar to VSIStat(), but will work on large files on
1175 : * systems where this requires special calls.
1176 : *
1177 : * This method goes through the VSIFileHandler virtualization and may
1178 : * work on unusual filesystems such as in memory.
1179 : *
1180 : * Analog of the POSIX stat() function, with an extra parameter to
1181 : * specify which information is needed, which offers a potential for
1182 : * speed optimizations on specialized and potentially slow virtual
1183 : * filesystem objects (/vsigzip/, /vsicurl/)
1184 : *
1185 : * @param pszFilename the path of the filesystem object to be queried.
1186 : * UTF-8 encoded.
1187 : * @param psStatBuf the structure to load with information.
1188 : * @param nFlags 0 to get all information, or VSI_STAT_EXISTS_FLAG,
1189 : * VSI_STAT_NATURE_FLAG, VSI_STAT_SIZE_FLAG,
1190 : * VSI_STAT_SET_ERROR_FLAG, VSI_STAT_CACHE_ONLY or a combination of those to get
1191 : * partial info.
1192 : *
1193 : * @return 0 on success or -1 on an error.
1194 : *
1195 : * @since GDAL 1.8.0
1196 : */
1197 :
1198 763400 : int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags)
1199 :
1200 : {
1201 763400 : char szAltPath[4] = {'\0'};
1202 :
1203 : // Enable to work on "C:" as if it were "C:\".
1204 763400 : if (pszFilename[0] != '\0' && pszFilename[1] == ':' &&
1205 20 : pszFilename[2] == '\0')
1206 : {
1207 0 : szAltPath[0] = pszFilename[0];
1208 0 : szAltPath[1] = pszFilename[1];
1209 0 : szAltPath[2] = '\\';
1210 0 : szAltPath[3] = '\0';
1211 :
1212 0 : pszFilename = szAltPath;
1213 : }
1214 :
1215 763400 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1216 :
1217 763400 : if (nFlags == 0)
1218 415130 : nFlags =
1219 : VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG | VSI_STAT_SIZE_FLAG;
1220 :
1221 1526800 : return poFSHandler->Stat(pszFilename, psStatBuf, nFlags);
1222 : }
1223 :
1224 : /************************************************************************/
1225 : /* VSIGetFileMetadata() */
1226 : /************************************************************************/
1227 :
1228 : /**
1229 : * \brief Get metadata on files.
1230 : *
1231 : * Implemented currently only for network-like filesystems, or starting
1232 : * with GDAL 3.7 for /vsizip/
1233 : *
1234 : * Starting with GDAL 3.11, calling it with pszFilename being the root of a
1235 : * /vsigs/ bucket and pszDomain == nullptr, and when authenticated through
1236 : * OAuth2, will result in returning the result of a "Buckets: get"
1237 : * operation (https://cloud.google.com/storage/docs/json_api/v1/buckets/get),
1238 : * with the keys of the top-level JSON document as keys of the key=value pairs
1239 : * returned by this function.
1240 : *
1241 : * @param pszFilename the path of the filesystem object to be queried.
1242 : * UTF-8 encoded.
1243 : * @param pszDomain Metadata domain to query. Depends on the file system.
1244 : * The following ones are supported:
1245 : * <ul>
1246 : * <li>HEADERS: to get HTTP headers for network-like filesystems (/vsicurl/,
1247 : * /vsis3/, /vsgis/, etc)</li>
1248 : * <li>TAGS:
1249 : * <ul>
1250 : * <li>/vsis3/: to get S3 Object tagging information</li>
1251 : * <li>/vsiaz/: to get blob tags. Refer to
1252 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags
1253 : * </li>
1254 : * </ul>
1255 : * </li>
1256 : * <li>STATUS: specific to /vsiadls/: returns all system defined properties for
1257 : * a path (seems in practice to be a subset of HEADERS)</li> <li>ACL: specific
1258 : * to /vsiadls/ and /vsigs/: returns the access control list for a path. For
1259 : * /vsigs/, a single XML=xml_content string is returned. Refer to
1260 : * https://cloud.google.com/storage/docs/xml-api/get-object-acls
1261 : * </li>
1262 : * <li>METADATA: specific to /vsiaz/: to get blob metadata. Refer to
1263 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-metadata.
1264 : * Note: this will be a subset of what pszDomain=HEADERS returns</li>
1265 : * <li>ZIP: specific to /vsizip/: to obtain ZIP specific metadata, in particular
1266 : * if a file is SOZIP-enabled (SOZIP_VALID=YES)</li>
1267 : * </ul>
1268 : * @param papszOptions Unused. Should be set to NULL.
1269 : *
1270 : * @return a NULL-terminated list of key=value strings, to be freed with
1271 : * CSLDestroy() or NULL in case of error / empty list.
1272 : *
1273 : * @since GDAL 3.1.0
1274 : */
1275 :
1276 76 : char **VSIGetFileMetadata(const char *pszFilename, const char *pszDomain,
1277 : CSLConstList papszOptions)
1278 : {
1279 76 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1280 76 : return poFSHandler->GetFileMetadata(pszFilename, pszDomain, papszOptions);
1281 : }
1282 :
1283 : /************************************************************************/
1284 : /* VSISetFileMetadata() */
1285 : /************************************************************************/
1286 :
1287 : /**
1288 : * \brief Set metadata on files.
1289 : *
1290 : * Implemented currently only for /vsis3/, /vsigs/, /vsiaz/ and /vsiadls/
1291 : *
1292 : * @param pszFilename the path of the filesystem object to be set.
1293 : * UTF-8 encoded.
1294 : * @param papszMetadata NULL-terminated list of key=value strings.
1295 : * @param pszDomain Metadata domain to set. Depends on the file system.
1296 : * The following are supported:
1297 : * <ul>
1298 : * <li>HEADERS: specific to /vsis3/ and /vsigs/: to set HTTP headers, such as
1299 : * "Content-Type", or other file system specific header.
1300 : * For /vsigs/, this also includes: x-goog-meta-{key}={value}. Note that you
1301 : * should specify all metadata to be set, as existing metadata will be
1302 : * overridden.
1303 : * </li>
1304 : * <li>TAGS: Content of papszMetadata should be KEY=VALUE pairs.
1305 : * <ul>
1306 : * <li>/vsis3/: to set S3 Object tagging information</li>
1307 : * <li>/vsiaz/: to set blob tags. Refer to
1308 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-tags.
1309 : * Note: storageV2 must be enabled on the account</li>
1310 : * </ul>
1311 : * </li>
1312 : * <li>PROPERTIES:
1313 : * <ul>
1314 : * <li>to /vsiaz/: to set properties. Refer to
1315 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-properties.</li>
1316 : * <li>to /vsiadls/: to set properties. Refer to
1317 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
1318 : * for headers valid for action=setProperties.</li>
1319 : * </ul>
1320 : * </li>
1321 : * <li>ACL: specific to /vsiadls/ and /vsigs/: to set access control list.
1322 : * For /vsiadls/, refer to
1323 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
1324 : * for headers valid for action=setAccessControl or setAccessControlRecursive.
1325 : * In setAccessControlRecursive, x-ms-acl must be specified in papszMetadata.
1326 : * For /vsigs/, refer to
1327 : * https://cloud.google.com/storage/docs/xml-api/put-object-acls. A single
1328 : * XML=xml_content string should be specified as in papszMetadata.
1329 : * </li>
1330 : * <li>METADATA: specific to /vsiaz/: to set blob metadata. Refer to
1331 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-metadata.
1332 : * Content of papszMetadata should be strings in the form
1333 : * x-ms-meta-name=value</li>
1334 : * </ul>
1335 : * @param papszOptions NULL or NULL terminated list of options.
1336 : * For /vsiadls/ and pszDomain=ACL, "RECURSIVE=TRUE" can be
1337 : * set to set the access control list recursively. When
1338 : * RECURSIVE=TRUE is set, MODE should also be set to one of
1339 : * "set", "modify" or "remove".
1340 : *
1341 : * @return TRUE in case of success.
1342 : *
1343 : * @since GDAL 3.1.0
1344 : */
1345 :
1346 17 : int VSISetFileMetadata(const char *pszFilename, CSLConstList papszMetadata,
1347 : const char *pszDomain, CSLConstList papszOptions)
1348 : {
1349 17 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1350 34 : return poFSHandler->SetFileMetadata(pszFilename, papszMetadata, pszDomain,
1351 17 : papszOptions)
1352 17 : ? 1
1353 17 : : 0;
1354 : }
1355 :
1356 : /************************************************************************/
1357 : /* VSIIsCaseSensitiveFS() */
1358 : /************************************************************************/
1359 :
1360 : /**
1361 : * \brief Returns if the filenames of the filesystem are case sensitive.
1362 : *
1363 : * This method retrieves to which filesystem belongs the passed filename
1364 : * and return TRUE if the filenames of that filesystem are case sensitive.
1365 : *
1366 : * Currently, this will return FALSE only for Windows real filenames. Other
1367 : * VSI virtual filesystems are case sensitive.
1368 : *
1369 : * This methods avoid ugly \#ifndef _WIN32 / \#endif code, that is wrong when
1370 : * dealing with virtual filenames.
1371 : *
1372 : * @param pszFilename the path of the filesystem object to be tested.
1373 : * UTF-8 encoded.
1374 : *
1375 : * @return TRUE if the filenames of the filesystem are case sensitive.
1376 : *
1377 : * @since GDAL 1.8.0
1378 : */
1379 :
1380 25738 : int VSIIsCaseSensitiveFS(const char *pszFilename)
1381 : {
1382 25738 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1383 :
1384 25738 : return poFSHandler->IsCaseSensitive(pszFilename);
1385 : }
1386 :
1387 : /************************************************************************/
1388 : /* VSISupportsSparseFiles() */
1389 : /************************************************************************/
1390 :
1391 : /**
1392 : * \brief Returns if the filesystem supports sparse files.
1393 : *
1394 : * Only supported on Linux (and no other Unix derivatives) and
1395 : * Windows. On Linux, the answer depends on a few hardcoded
1396 : * signatures for common filesystems. Other filesystems will be
1397 : * considered as not supporting sparse files.
1398 : *
1399 : * @param pszPath the path of the filesystem object to be tested.
1400 : * UTF-8 encoded.
1401 : *
1402 : * @return TRUE if the file system is known to support sparse files. FALSE may
1403 : * be returned both in cases where it is known to not support them,
1404 : * or when it is unknown.
1405 : *
1406 : * @since GDAL 2.2
1407 : */
1408 :
1409 2 : int VSISupportsSparseFiles(const char *pszPath)
1410 : {
1411 2 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1412 :
1413 2 : return poFSHandler->SupportsSparseFiles(pszPath);
1414 : }
1415 :
1416 : /************************************************************************/
1417 : /* VSIIsLocal() */
1418 : /************************************************************************/
1419 :
1420 : /**
1421 : * \brief Returns if the file/filesystem is "local".
1422 : *
1423 : * The concept of local is mostly by opposition with a network / remote
1424 : * file system whose access time can be long.
1425 : *
1426 : * /vsimem/ is considered to be a local file system, although a non-persistent
1427 : * one.
1428 : *
1429 : * @param pszPath the path of the filesystem object to be tested.
1430 : * UTF-8 encoded.
1431 : *
1432 : * @return TRUE or FALSE
1433 : *
1434 : * @since GDAL 3.6
1435 : */
1436 :
1437 216 : bool VSIIsLocal(const char *pszPath)
1438 : {
1439 216 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1440 :
1441 216 : return poFSHandler->IsLocal(pszPath);
1442 : }
1443 :
1444 : /************************************************************************/
1445 : /* VSIGetCanonicalFilename() */
1446 : /************************************************************************/
1447 :
1448 : /**
1449 : * \brief Returns the canonical filename.
1450 : *
1451 : * May be implemented by case-insensitive filesystems
1452 : * (currently Win32 and MacOSX) to return the filename with its actual case
1453 : * (i.e. the one that would be used when listing the content of the directory).
1454 : *
1455 : * @param pszPath UTF-8 encoded path
1456 : *
1457 : * @return UTF-8 encoded string, to free with VSIFree()
1458 : *
1459 : * @since GDAL 3.8
1460 : */
1461 :
1462 248 : char *VSIGetCanonicalFilename(const char *pszPath)
1463 : {
1464 248 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1465 :
1466 248 : return CPLStrdup(poFSHandler->GetCanonicalFilename(pszPath).c_str());
1467 : }
1468 :
1469 : /************************************************************************/
1470 : /* VSISupportsSequentialWrite() */
1471 : /************************************************************************/
1472 :
1473 : /**
1474 : * \brief Returns if the filesystem supports sequential write.
1475 : *
1476 : * @param pszPath the path of the filesystem object to be tested.
1477 : * UTF-8 encoded.
1478 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1479 : * local temporary file before uploading to the target location.
1480 : *
1481 : * @return TRUE or FALSE
1482 : *
1483 : * @since GDAL 3.6
1484 : */
1485 :
1486 94 : bool VSISupportsSequentialWrite(const char *pszPath, bool bAllowLocalTempFile)
1487 : {
1488 94 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1489 :
1490 94 : return poFSHandler->SupportsSequentialWrite(pszPath, bAllowLocalTempFile);
1491 : }
1492 :
1493 : /************************************************************************/
1494 : /* VSISupportsRandomWrite() */
1495 : /************************************************************************/
1496 :
1497 : /**
1498 : * \brief Returns if the filesystem supports random write.
1499 : *
1500 : * @param pszPath the path of the filesystem object to be tested.
1501 : * UTF-8 encoded.
1502 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1503 : * local temporary file before uploading to the target location.
1504 : *
1505 : * @return TRUE or FALSE
1506 : *
1507 : * @since GDAL 3.6
1508 : */
1509 :
1510 316 : bool VSISupportsRandomWrite(const char *pszPath, bool bAllowLocalTempFile)
1511 : {
1512 316 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1513 :
1514 316 : return poFSHandler->SupportsRandomWrite(pszPath, bAllowLocalTempFile);
1515 : }
1516 :
1517 : /************************************************************************/
1518 : /* VSIHasOptimizedReadMultiRange() */
1519 : /************************************************************************/
1520 :
1521 : /**
1522 : * \brief Returns if the filesystem supports efficient multi-range reading.
1523 : *
1524 : * Currently only returns TRUE for /vsicurl/ and derived file systems.
1525 : *
1526 : * @param pszPath the path of the filesystem object to be tested.
1527 : * UTF-8 encoded.
1528 : *
1529 : * @return TRUE if the file system is known to have an efficient multi-range
1530 : * reading.
1531 : *
1532 : * @since GDAL 2.3
1533 : */
1534 :
1535 12385 : int VSIHasOptimizedReadMultiRange(const char *pszPath)
1536 : {
1537 12385 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1538 :
1539 12385 : return poFSHandler->HasOptimizedReadMultiRange(pszPath);
1540 : }
1541 :
1542 : /************************************************************************/
1543 : /* VSIGetActualURL() */
1544 : /************************************************************************/
1545 :
1546 : /**
1547 : * \brief Returns the actual URL of a supplied filename.
1548 : *
1549 : * Currently only returns a non-NULL value for network-based virtual file
1550 : * systems. For example "/vsis3/bucket/filename" will be expanded as
1551 : * "https://bucket.s3.amazon.com/filename"
1552 : *
1553 : * Note that the lifetime of the returned string, is short, and may be
1554 : * invalidated by any following GDAL functions.
1555 : *
1556 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1557 : *
1558 : * @return the actual URL corresponding to the supplied filename, or NULL.
1559 : * Should not be freed.
1560 : *
1561 : * @since GDAL 2.3
1562 : */
1563 :
1564 9 : const char *VSIGetActualURL(const char *pszFilename)
1565 : {
1566 9 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1567 :
1568 9 : return poFSHandler->GetActualURL(pszFilename);
1569 : }
1570 :
1571 : /************************************************************************/
1572 : /* VSIGetSignedURL() */
1573 : /************************************************************************/
1574 :
1575 : /**
1576 : * \brief Returns a signed URL of a supplied filename.
1577 : *
1578 : * Currently only returns a non-NULL value for /vsis3/, /vsigs/, /vsiaz/ and
1579 : * /vsioss/ For example "/vsis3/bucket/filename" will be expanded as
1580 : * "https://bucket.s3.amazon.com/filename?X-Amz-Algorithm=AWS4-HMAC-SHA256..."
1581 : * Configuration options that apply for file opening (typically to provide
1582 : * credentials), and are returned by VSIGetFileSystemOptions(), are also valid
1583 : * in that context.
1584 : *
1585 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1586 : * @param papszOptions list of options, or NULL. Depend on file system handler.
1587 : * For /vsis3/, /vsigs/, /vsiaz/ and /vsioss/, the following options are
1588 : * supported: <ul> <li>START_DATE=YYMMDDTHHMMSSZ: date and time in UTC following
1589 : * ISO 8601 standard, corresponding to the start of validity of the URL. If not
1590 : * specified, current date time.</li> <li>EXPIRATION_DELAY=number_of_seconds:
1591 : * number between 1 and 604800 (seven days) for the validity of the signed URL.
1592 : * Defaults to 3600 (one hour)</li> <li>VERB=GET/HEAD/DELETE/PUT/POST: HTTP VERB
1593 : * for which the request will be used. Default to GET.</li>
1594 : * </ul>
1595 : *
1596 : * /vsiaz/ supports additional options:
1597 : * <ul>
1598 : * <li>SIGNEDIDENTIFIER=value: to relate the given shared access signature
1599 : * to a corresponding stored access policy.</li>
1600 : * <li>SIGNEDPERMISSIONS=r|w: permissions associated with the shared access
1601 : * signature. Normally deduced from VERB.</li>
1602 : * </ul>
1603 : *
1604 : * @return a signed URL, or NULL. Should be freed with CPLFree().
1605 : * @since GDAL 2.3
1606 : */
1607 :
1608 25 : char *VSIGetSignedURL(const char *pszFilename, CSLConstList papszOptions)
1609 : {
1610 25 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1611 :
1612 25 : return poFSHandler->GetSignedURL(pszFilename, papszOptions);
1613 : }
1614 :
1615 : /************************************************************************/
1616 : /* VSIFOpenL() */
1617 : /************************************************************************/
1618 :
1619 : /**
1620 : * \brief Open file.
1621 : *
1622 : * This function opens a file with the desired access. Large files (larger
1623 : * than 2GB) should be supported. Binary access is always implied and
1624 : * the "b" does not need to be included in the pszAccess string.
1625 : *
1626 : * Note that the "VSILFILE *" returned since GDAL 1.8.0 by this function is
1627 : * *NOT* a standard C library FILE *, and cannot be used with any functions
1628 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
1629 : *
1630 : * On windows it is possible to define the configuration option
1631 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
1632 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
1633 : *
1634 : * This method goes through the VSIFileHandler virtualization and may
1635 : * work on unusual filesystems such as in memory.
1636 : *
1637 : * Analog of the POSIX fopen() function.
1638 : *
1639 : * @param pszFilename the file to open. UTF-8 encoded.
1640 : * @param pszAccess access requested (i.e. "r", "r+", "w")
1641 : *
1642 : * @return NULL on failure, or the file handle.
1643 : */
1644 :
1645 223221 : VSILFILE *VSIFOpenL(const char *pszFilename, const char *pszAccess)
1646 :
1647 : {
1648 223221 : return VSIFOpenExL(pszFilename, pszAccess, false);
1649 : }
1650 :
1651 : /************************************************************************/
1652 : /* Open() */
1653 : /************************************************************************/
1654 :
1655 : #ifndef DOXYGEN_SKIP
1656 :
1657 5701 : VSIVirtualHandle *VSIFilesystemHandler::Open(const char *pszFilename,
1658 : const char *pszAccess)
1659 : {
1660 5701 : return Open(pszFilename, pszAccess, false, nullptr);
1661 : }
1662 :
1663 : /************************************************************************/
1664 : /* CopyFile() */
1665 : /************************************************************************/
1666 :
1667 2208 : int VSIFilesystemHandler::CopyFile(const char *pszSource, const char *pszTarget,
1668 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
1669 : CSLConstList papszOptions,
1670 : GDALProgressFunc pProgressFunc,
1671 : void *pProgressData)
1672 : {
1673 2208 : VSIVirtualHandleUniquePtr poFileHandleAutoClose;
1674 2208 : if (!fpSource)
1675 : {
1676 2191 : CPLAssert(pszSource);
1677 2191 : fpSource = VSIFOpenExL(pszSource, "rb", TRUE);
1678 2191 : if (!fpSource)
1679 : {
1680 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSource);
1681 1 : return -1;
1682 : }
1683 2190 : poFileHandleAutoClose.reset(fpSource);
1684 : }
1685 2207 : if (nSourceSize == static_cast<vsi_l_offset>(-1) &&
1686 2 : pProgressFunc != nullptr && pszSource != nullptr)
1687 : {
1688 : VSIStatBufL sStat;
1689 2 : if (VSIStatL(pszSource, &sStat) == 0)
1690 : {
1691 2 : nSourceSize = sStat.st_size;
1692 : }
1693 : }
1694 :
1695 2207 : VSILFILE *fpOut = VSIFOpenEx2L(pszTarget, "wb", TRUE, papszOptions);
1696 2207 : if (!fpOut)
1697 : {
1698 2 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", pszTarget);
1699 2 : return -1;
1700 : }
1701 :
1702 4410 : CPLString osMsg;
1703 2205 : if (pszSource)
1704 2203 : osMsg.Printf("Copying of %s", pszSource);
1705 : else
1706 2 : pszSource = "(unknown filename)";
1707 :
1708 2205 : int ret = 0;
1709 2205 : constexpr size_t nBufferSize = 10 * 4096;
1710 2205 : std::vector<GByte> abyBuffer(nBufferSize, 0);
1711 2205 : GUIntBig nOffset = 0;
1712 : while (true)
1713 : {
1714 2505 : const size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpSource);
1715 2505 : if (nRead < nBufferSize && VSIFErrorL(fpSource))
1716 : {
1717 8 : CPLError(
1718 : CE_Failure, CPLE_FileIO,
1719 : "Copying of %s to %s failed: error while reading source file",
1720 : pszSource, pszTarget);
1721 8 : ret = -1;
1722 8 : break;
1723 : }
1724 2497 : if (nRead > 0)
1725 : {
1726 2496 : const size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
1727 2496 : if (nWritten != nRead)
1728 : {
1729 20 : CPLError(CE_Failure, CPLE_FileIO,
1730 : "Copying of %s to %s failed: error while writing into "
1731 : "target file",
1732 : pszSource, pszTarget);
1733 20 : ret = -1;
1734 20 : break;
1735 : }
1736 2476 : nOffset += nRead;
1737 2520 : if (pProgressFunc &&
1738 88 : !pProgressFunc(
1739 : nSourceSize == 0 ? 1.0
1740 44 : : nSourceSize > 0 &&
1741 : nSourceSize != static_cast<vsi_l_offset>(-1)
1742 88 : ? double(nOffset) / nSourceSize
1743 : : 0.0,
1744 44 : !osMsg.empty() ? osMsg.c_str() : nullptr, pProgressData))
1745 : {
1746 1 : ret = -1;
1747 1 : break;
1748 : }
1749 : }
1750 2476 : if (nRead < nBufferSize)
1751 : {
1752 2176 : break;
1753 : }
1754 300 : }
1755 :
1756 2205 : if (nSourceSize != static_cast<vsi_l_offset>(-1) && nOffset != nSourceSize)
1757 : {
1758 2 : CPLError(CE_Failure, CPLE_FileIO,
1759 : "Copying of %s to %s failed: %" PRIu64 " bytes were copied "
1760 : "whereas %" PRIu64 " were expected",
1761 : pszSource, pszTarget, static_cast<uint64_t>(nOffset),
1762 : static_cast<uint64_t>(nSourceSize));
1763 2 : ret = -1;
1764 : }
1765 :
1766 2205 : if (VSIFCloseL(fpOut) != 0)
1767 : {
1768 1 : ret = -1;
1769 : }
1770 :
1771 2205 : if (ret != 0)
1772 30 : VSIUnlink(pszTarget);
1773 :
1774 2205 : return ret;
1775 : }
1776 :
1777 : /************************************************************************/
1778 : /* CopyFileRestartable() */
1779 : /************************************************************************/
1780 :
1781 2 : int VSIFilesystemHandler::CopyFileRestartable(
1782 : const char *pszSource, const char *pszTarget,
1783 : const char * /* pszInputPayload */, char **ppszOutputPayload,
1784 : CSLConstList papszOptions, GDALProgressFunc pProgressFunc,
1785 : void *pProgressData)
1786 : {
1787 2 : *ppszOutputPayload = nullptr;
1788 2 : return CopyFile(pszSource, pszTarget, nullptr,
1789 : static_cast<vsi_l_offset>(-1), papszOptions, pProgressFunc,
1790 2 : pProgressData);
1791 : }
1792 :
1793 : /************************************************************************/
1794 : /* Sync() */
1795 : /************************************************************************/
1796 :
1797 28 : bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget,
1798 : const char *const *papszOptions,
1799 : GDALProgressFunc pProgressFunc,
1800 : void *pProgressData, char ***ppapszOutputs)
1801 : {
1802 28 : const char SOURCE_SEP = VSIGetDirectorySeparator(pszSource)[0];
1803 :
1804 28 : if (ppapszOutputs)
1805 : {
1806 0 : *ppapszOutputs = nullptr;
1807 : }
1808 :
1809 : VSIStatBufL sSource;
1810 56 : CPLString osSource(pszSource);
1811 56 : CPLString osSourceWithoutSlash(pszSource);
1812 34 : if (osSourceWithoutSlash.back() == '/' ||
1813 6 : osSourceWithoutSlash.back() == '\\')
1814 : {
1815 22 : osSourceWithoutSlash.pop_back();
1816 : }
1817 28 : if (VSIStatL(osSourceWithoutSlash, &sSource) < 0)
1818 : {
1819 1 : CPLError(CE_Failure, CPLE_FileIO, "%s does not exist", pszSource);
1820 1 : return false;
1821 : }
1822 :
1823 27 : if (VSI_ISDIR(sSource.st_mode))
1824 : {
1825 22 : CPLString osTargetDir(pszTarget);
1826 11 : if (osSource.back() != '/' && osSource.back() != '\\')
1827 : {
1828 : osTargetDir = CPLFormFilename(osTargetDir,
1829 1 : CPLGetFilename(pszSource), nullptr);
1830 : }
1831 :
1832 : VSIStatBufL sTarget;
1833 11 : bool ret = true;
1834 11 : if (VSIStatL(osTargetDir, &sTarget) < 0)
1835 : {
1836 9 : if (VSIMkdirRecursive(osTargetDir, 0755) < 0)
1837 : {
1838 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create directory %s",
1839 : osTargetDir.c_str());
1840 1 : return false;
1841 : }
1842 : }
1843 :
1844 10 : if (!CPLFetchBool(papszOptions, "STOP_ON_DIR", false))
1845 : {
1846 20 : CPLStringList aosChildOptions(CSLDuplicate(papszOptions));
1847 10 : if (!CPLFetchBool(papszOptions, "RECURSIVE", true))
1848 : {
1849 0 : aosChildOptions.SetNameValue("RECURSIVE", nullptr);
1850 0 : aosChildOptions.AddString("STOP_ON_DIR=TRUE");
1851 : }
1852 :
1853 10 : char **papszSrcFiles = VSIReadDir(osSourceWithoutSlash);
1854 10 : int nFileCount = 0;
1855 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter)
1856 : {
1857 17 : if (strcmp(*iter, ".") != 0 && strcmp(*iter, "..") != 0)
1858 : {
1859 17 : nFileCount++;
1860 : }
1861 : }
1862 10 : int iFile = 0;
1863 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter, ++iFile)
1864 : {
1865 17 : if (strcmp(*iter, ".") == 0 || strcmp(*iter, "..") == 0)
1866 : {
1867 0 : continue;
1868 : }
1869 : CPLString osSubSource(
1870 17 : CPLFormFilename(osSourceWithoutSlash, *iter, nullptr));
1871 : CPLString osSubTarget(
1872 17 : CPLFormFilename(osTargetDir, *iter, nullptr));
1873 : // coverity[divide_by_zero]
1874 34 : void *pScaledProgress = GDALCreateScaledProgress(
1875 17 : double(iFile) / nFileCount, double(iFile + 1) / nFileCount,
1876 : pProgressFunc, pProgressData);
1877 17 : ret = Sync((osSubSource + SOURCE_SEP).c_str(), osSubTarget,
1878 17 : aosChildOptions.List(), GDALScaledProgress,
1879 17 : pScaledProgress, nullptr);
1880 17 : GDALDestroyScaledProgress(pScaledProgress);
1881 17 : if (!ret)
1882 : {
1883 0 : break;
1884 : }
1885 : }
1886 10 : CSLDestroy(papszSrcFiles);
1887 : }
1888 10 : return ret;
1889 : }
1890 :
1891 : VSIStatBufL sTarget;
1892 32 : CPLString osTarget(pszTarget);
1893 16 : if (VSIStatL(osTarget, &sTarget) == 0)
1894 : {
1895 4 : bool bTargetIsFile = true;
1896 4 : if (VSI_ISDIR(sTarget.st_mode))
1897 : {
1898 : osTarget =
1899 2 : CPLFormFilename(osTarget, CPLGetFilename(pszSource), nullptr);
1900 3 : bTargetIsFile = VSIStatL(osTarget, &sTarget) == 0 &&
1901 1 : !CPL_TO_BOOL(VSI_ISDIR(sTarget.st_mode));
1902 : }
1903 4 : if (bTargetIsFile)
1904 : {
1905 3 : if (sSource.st_size == sTarget.st_size &&
1906 3 : sSource.st_mtime == sTarget.st_mtime && sSource.st_mtime != 0)
1907 : {
1908 2 : CPLDebug("VSI",
1909 : "%s and %s have same size and modification "
1910 : "date. Skipping copying",
1911 : osSourceWithoutSlash.c_str(), osTarget.c_str());
1912 2 : return true;
1913 : }
1914 : }
1915 : }
1916 :
1917 14 : VSILFILE *fpIn = VSIFOpenExL(osSourceWithoutSlash, "rb", TRUE);
1918 14 : if (fpIn == nullptr)
1919 : {
1920 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s",
1921 : osSourceWithoutSlash.c_str());
1922 0 : return false;
1923 : }
1924 :
1925 14 : VSILFILE *fpOut = VSIFOpenExL(osTarget.c_str(), "wb", TRUE);
1926 14 : if (fpOut == nullptr)
1927 : {
1928 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", osTarget.c_str());
1929 1 : VSIFCloseL(fpIn);
1930 1 : return false;
1931 : }
1932 :
1933 13 : bool ret = true;
1934 13 : constexpr size_t nBufferSize = 10 * 4096;
1935 26 : std::vector<GByte> abyBuffer(nBufferSize, 0);
1936 13 : GUIntBig nOffset = 0;
1937 13 : CPLString osMsg;
1938 13 : osMsg.Printf("Copying of %s", osSourceWithoutSlash.c_str());
1939 : while (true)
1940 : {
1941 16 : size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpIn);
1942 16 : size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
1943 16 : if (nWritten != nRead)
1944 : {
1945 0 : CPLError(CE_Failure, CPLE_FileIO, "Copying of %s to %s failed",
1946 : osSourceWithoutSlash.c_str(), osTarget.c_str());
1947 0 : ret = false;
1948 0 : break;
1949 : }
1950 16 : nOffset += nRead;
1951 16 : if (pProgressFunc && !pProgressFunc(double(nOffset) / sSource.st_size,
1952 : osMsg.c_str(), pProgressData))
1953 : {
1954 0 : ret = false;
1955 0 : break;
1956 : }
1957 16 : if (nRead < nBufferSize)
1958 : {
1959 13 : break;
1960 : }
1961 3 : }
1962 :
1963 13 : VSIFCloseL(fpIn);
1964 13 : if (VSIFCloseL(fpOut) != 0)
1965 : {
1966 0 : ret = false;
1967 : }
1968 13 : return ret;
1969 : }
1970 :
1971 : /************************************************************************/
1972 : /* VSIDIREntry() */
1973 : /************************************************************************/
1974 :
1975 3192 : VSIDIREntry::VSIDIREntry()
1976 : : pszName(nullptr), nMode(0), nSize(0), nMTime(0), bModeKnown(false),
1977 3192 : bSizeKnown(false), bMTimeKnown(false), papszExtra(nullptr)
1978 : {
1979 3192 : }
1980 :
1981 : /************************************************************************/
1982 : /* VSIDIREntry() */
1983 : /************************************************************************/
1984 :
1985 6 : VSIDIREntry::VSIDIREntry(const VSIDIREntry &other)
1986 6 : : pszName(VSIStrdup(other.pszName)), nMode(other.nMode), nSize(other.nSize),
1987 6 : nMTime(other.nMTime), bModeKnown(other.bModeKnown),
1988 6 : bSizeKnown(other.bSizeKnown), bMTimeKnown(other.bMTimeKnown),
1989 6 : papszExtra(CSLDuplicate(other.papszExtra))
1990 : {
1991 6 : }
1992 :
1993 : /************************************************************************/
1994 : /* ~VSIDIREntry() */
1995 : /************************************************************************/
1996 :
1997 6396 : VSIDIREntry::~VSIDIREntry()
1998 : {
1999 3198 : CPLFree(pszName);
2000 3198 : CSLDestroy(papszExtra);
2001 3198 : }
2002 :
2003 : /************************************************************************/
2004 : /* ~VSIDIR() */
2005 : /************************************************************************/
2006 :
2007 2804 : VSIDIR::~VSIDIR()
2008 : {
2009 2804 : }
2010 :
2011 : /************************************************************************/
2012 : /* VSIDIRGeneric */
2013 : /************************************************************************/
2014 :
2015 : namespace
2016 : {
2017 : struct VSIDIRGeneric : public VSIDIR
2018 : {
2019 : CPLString osRootPath{};
2020 : CPLString osBasePath{};
2021 : char **papszContent = nullptr;
2022 : int nRecurseDepth = 0;
2023 : int nPos = 0;
2024 : VSIDIREntry entry{};
2025 : std::vector<VSIDIRGeneric *> aoStackSubDir{};
2026 : VSIFilesystemHandler *poFS = nullptr;
2027 : std::string m_osFilterPrefix{};
2028 :
2029 2444 : explicit VSIDIRGeneric(VSIFilesystemHandler *poFSIn) : poFS(poFSIn)
2030 : {
2031 2444 : }
2032 :
2033 : ~VSIDIRGeneric();
2034 :
2035 : const VSIDIREntry *NextDirEntry() override;
2036 :
2037 : VSIDIRGeneric(const VSIDIRGeneric &) = delete;
2038 : VSIDIRGeneric &operator=(const VSIDIRGeneric &) = delete;
2039 : };
2040 :
2041 : /************************************************************************/
2042 : /* ~VSIDIRGeneric() */
2043 : /************************************************************************/
2044 :
2045 7332 : VSIDIRGeneric::~VSIDIRGeneric()
2046 : {
2047 2449 : while (!aoStackSubDir.empty())
2048 : {
2049 5 : delete aoStackSubDir.back();
2050 5 : aoStackSubDir.pop_back();
2051 : }
2052 2444 : CSLDestroy(papszContent);
2053 4888 : }
2054 :
2055 : } // namespace
2056 :
2057 : /************************************************************************/
2058 : /* OpenDir() */
2059 : /************************************************************************/
2060 :
2061 2450 : VSIDIR *VSIFilesystemHandler::OpenDir(const char *pszPath, int nRecurseDepth,
2062 : const char *const *papszOptions)
2063 : {
2064 2450 : char **papszContent = VSIReadDir(pszPath);
2065 : VSIStatBufL sStatL;
2066 2987 : if (papszContent == nullptr &&
2067 537 : (VSIStatL(pszPath, &sStatL) != 0 || !VSI_ISDIR(sStatL.st_mode)))
2068 : {
2069 6 : return nullptr;
2070 : }
2071 2444 : VSIDIRGeneric *dir = new VSIDIRGeneric(this);
2072 2444 : dir->osRootPath = pszPath;
2073 4888 : if (!dir->osRootPath.empty() &&
2074 2444 : (dir->osRootPath.back() == '/' || dir->osRootPath.back() == '\\'))
2075 1 : dir->osRootPath.pop_back();
2076 2444 : dir->nRecurseDepth = nRecurseDepth;
2077 2444 : dir->papszContent = papszContent;
2078 2444 : dir->m_osFilterPrefix = CSLFetchNameValueDef(papszOptions, "PREFIX", "");
2079 2444 : return dir;
2080 : }
2081 :
2082 : /************************************************************************/
2083 : /* NextDirEntry() */
2084 : /************************************************************************/
2085 :
2086 506624 : const VSIDIREntry *VSIDIRGeneric::NextDirEntry()
2087 : {
2088 506624 : const char SEP = VSIGetDirectorySeparator(osRootPath.c_str())[0];
2089 :
2090 506625 : begin:
2091 506625 : if (VSI_ISDIR(entry.nMode) && nRecurseDepth != 0)
2092 : {
2093 1136 : CPLString osCurFile(osRootPath);
2094 1136 : if (!osCurFile.empty())
2095 1136 : osCurFile += SEP;
2096 1136 : osCurFile += entry.pszName;
2097 : auto subdir =
2098 1136 : static_cast<VSIDIRGeneric *>(poFS->VSIFilesystemHandler::OpenDir(
2099 1136 : osCurFile, nRecurseDepth - 1, nullptr));
2100 1136 : if (subdir)
2101 : {
2102 1136 : subdir->osRootPath = osRootPath;
2103 1136 : subdir->osBasePath = entry.pszName;
2104 1136 : subdir->m_osFilterPrefix = m_osFilterPrefix;
2105 1136 : aoStackSubDir.push_back(subdir);
2106 : }
2107 1136 : entry.nMode = 0;
2108 : }
2109 :
2110 507756 : while (!aoStackSubDir.empty())
2111 : {
2112 502726 : auto l_entry = aoStackSubDir.back()->NextDirEntry();
2113 502726 : if (l_entry)
2114 : {
2115 501595 : return l_entry;
2116 : }
2117 1131 : delete aoStackSubDir.back();
2118 1131 : aoStackSubDir.pop_back();
2119 : }
2120 :
2121 5030 : if (papszContent == nullptr)
2122 : {
2123 531 : return nullptr;
2124 : }
2125 :
2126 : while (true)
2127 : {
2128 4526 : if (!papszContent[nPos])
2129 : {
2130 1903 : return nullptr;
2131 : }
2132 : // Skip . and ..entries
2133 2623 : if (papszContent[nPos][0] == '.' &&
2134 98 : (papszContent[nPos][1] == '\0' ||
2135 98 : (papszContent[nPos][1] == '.' && papszContent[nPos][2] == '\0')))
2136 : {
2137 22 : nPos++;
2138 : }
2139 : else
2140 : {
2141 2601 : CPLFree(entry.pszName);
2142 2601 : CPLString osName(osBasePath);
2143 2601 : if (!osName.empty())
2144 1519 : osName += SEP;
2145 2601 : osName += papszContent[nPos];
2146 2601 : nPos++;
2147 :
2148 2601 : entry.pszName = CPLStrdup(osName);
2149 2601 : entry.nMode = 0;
2150 2601 : CPLString osCurFile(osRootPath);
2151 2601 : if (!osCurFile.empty())
2152 2601 : osCurFile += SEP;
2153 2601 : osCurFile += entry.pszName;
2154 :
2155 5192 : const auto StatFile = [&osCurFile, this]()
2156 : {
2157 : VSIStatBufL sStatL;
2158 2596 : if (VSIStatL(osCurFile, &sStatL) == 0)
2159 : {
2160 2592 : entry.nMode = sStatL.st_mode;
2161 2592 : entry.nSize = sStatL.st_size;
2162 2592 : entry.nMTime = sStatL.st_mtime;
2163 2592 : entry.bModeKnown = true;
2164 2592 : entry.bSizeKnown = true;
2165 2592 : entry.bMTimeKnown = true;
2166 : }
2167 : else
2168 : {
2169 4 : entry.nMode = 0;
2170 4 : entry.nSize = 0;
2171 4 : entry.nMTime = 0;
2172 4 : entry.bModeKnown = false;
2173 4 : entry.bSizeKnown = false;
2174 4 : entry.bMTimeKnown = false;
2175 : }
2176 2596 : };
2177 :
2178 2613 : if (!m_osFilterPrefix.empty() &&
2179 12 : m_osFilterPrefix.size() > osName.size())
2180 : {
2181 6 : if (STARTS_WITH(m_osFilterPrefix.c_str(), osName.c_str()) &&
2182 2 : m_osFilterPrefix[osName.size()] == SEP)
2183 : {
2184 1 : StatFile();
2185 1 : if (VSI_ISDIR(entry.nMode))
2186 : {
2187 1 : goto begin;
2188 : }
2189 : }
2190 3 : continue;
2191 : }
2192 2605 : if (!m_osFilterPrefix.empty() &&
2193 8 : !STARTS_WITH(osName.c_str(), m_osFilterPrefix.c_str()))
2194 : {
2195 2 : continue;
2196 : }
2197 :
2198 2595 : StatFile();
2199 :
2200 2595 : break;
2201 : }
2202 27 : }
2203 :
2204 2595 : return &(entry);
2205 : }
2206 :
2207 : /************************************************************************/
2208 : /* UnlinkBatch() */
2209 : /************************************************************************/
2210 :
2211 1 : int *VSIFilesystemHandler::UnlinkBatch(CSLConstList papszFiles)
2212 : {
2213 : int *panRet =
2214 1 : static_cast<int *>(CPLMalloc(sizeof(int) * CSLCount(papszFiles)));
2215 3 : for (int i = 0; papszFiles && papszFiles[i]; ++i)
2216 : {
2217 2 : panRet[i] = VSIUnlink(papszFiles[i]) == 0;
2218 : }
2219 1 : return panRet;
2220 : }
2221 :
2222 : /************************************************************************/
2223 : /* RmdirRecursive() */
2224 : /************************************************************************/
2225 :
2226 37 : int VSIFilesystemHandler::RmdirRecursive(const char *pszDirname)
2227 : {
2228 74 : CPLString osDirnameWithoutEndSlash(pszDirname);
2229 74 : if (!osDirnameWithoutEndSlash.empty() &&
2230 37 : (osDirnameWithoutEndSlash.back() == '/' ||
2231 37 : osDirnameWithoutEndSlash.back() == '\\'))
2232 : {
2233 0 : osDirnameWithoutEndSlash.pop_back();
2234 : }
2235 :
2236 37 : const char SEP = VSIGetDirectorySeparator(pszDirname)[0];
2237 :
2238 74 : CPLStringList aosOptions;
2239 : auto poDir =
2240 74 : std::unique_ptr<VSIDIR>(OpenDir(pszDirname, -1, aosOptions.List()));
2241 37 : if (!poDir)
2242 8 : return -1;
2243 58 : std::vector<std::string> aosDirs;
2244 : while (true)
2245 : {
2246 21637 : auto entry = poDir->NextDirEntry();
2247 21637 : if (!entry)
2248 29 : break;
2249 :
2250 43216 : const CPLString osFilename(osDirnameWithoutEndSlash + SEP +
2251 43216 : entry->pszName);
2252 21608 : if ((entry->nMode & S_IFDIR))
2253 : {
2254 150 : aosDirs.push_back(osFilename);
2255 : }
2256 : else
2257 : {
2258 21458 : if (VSIUnlink(osFilename) != 0)
2259 0 : return -1;
2260 : }
2261 21608 : }
2262 :
2263 : // Sort in reverse order, so that inner-most directories are deleted first
2264 29 : std::sort(aosDirs.begin(), aosDirs.end(),
2265 888 : [](const std::string &a, const std::string &b) { return a > b; });
2266 :
2267 179 : for (const auto &osDir : aosDirs)
2268 : {
2269 150 : if (VSIRmdir(osDir.c_str()) != 0)
2270 0 : return -1;
2271 : }
2272 :
2273 29 : return VSIRmdir(pszDirname);
2274 : }
2275 :
2276 : /************************************************************************/
2277 : /* GetFileMetadata() */
2278 : /************************************************************************/
2279 :
2280 0 : char **VSIFilesystemHandler::GetFileMetadata(const char * /* pszFilename*/,
2281 : const char * /*pszDomain*/,
2282 : CSLConstList /*papszOptions*/)
2283 : {
2284 0 : return nullptr;
2285 : }
2286 :
2287 : /************************************************************************/
2288 : /* SetFileMetadata() */
2289 : /************************************************************************/
2290 :
2291 0 : bool VSIFilesystemHandler::SetFileMetadata(const char * /* pszFilename*/,
2292 : CSLConstList /* papszMetadata */,
2293 : const char * /* pszDomain */,
2294 : CSLConstList /* papszOptions */)
2295 : {
2296 0 : CPLError(CE_Failure, CPLE_NotSupported, "SetFileMetadata() not supported");
2297 0 : return false;
2298 : }
2299 :
2300 : #endif
2301 :
2302 : /************************************************************************/
2303 : /* VSIFOpenExL() */
2304 : /************************************************************************/
2305 :
2306 : /**
2307 : * \brief Open/create file.
2308 : *
2309 : * This function opens (or creates) a file with the desired access.
2310 : * Binary access is always implied and
2311 : * the "b" does not need to be included in the pszAccess string.
2312 : *
2313 : * Note that the "VSILFILE *" returned by this function is
2314 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2315 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2316 : *
2317 : * On windows it is possible to define the configuration option
2318 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2319 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2320 : *
2321 : * This method goes through the VSIFileHandler virtualization and may
2322 : * work on unusual filesystems such as in memory.
2323 : *
2324 : * Analog of the POSIX fopen() function.
2325 : *
2326 : * @param pszFilename the file to open. UTF-8 encoded.
2327 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2328 : * @param bSetError flag determining whether or not this open call
2329 : * should set VSIErrors on failure.
2330 : *
2331 : * @return NULL on failure, or the file handle.
2332 : *
2333 : * @since GDAL 2.1
2334 : */
2335 :
2336 356402 : VSILFILE *VSIFOpenExL(const char *pszFilename, const char *pszAccess,
2337 : int bSetError)
2338 :
2339 : {
2340 356402 : return VSIFOpenEx2L(pszFilename, pszAccess, bSetError, nullptr);
2341 : }
2342 :
2343 : /************************************************************************/
2344 : /* VSIFOpenEx2L() */
2345 : /************************************************************************/
2346 :
2347 : /**
2348 : * \brief Open/create file.
2349 : *
2350 : * This function opens (or creates) a file with the desired access.
2351 : * Binary access is always implied and
2352 : * the "b" does not need to be included in the pszAccess string.
2353 : *
2354 : * Note that the "VSILFILE *" returned by this function is
2355 : * *NOT* a standard C library FILE *, and cannot be used with any functions
2356 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
2357 : *
2358 : * On windows it is possible to define the configuration option
2359 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
2360 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
2361 : *
2362 : * This method goes through the VSIFileHandler virtualization and may
2363 : * work on unusual filesystems such as in memory.
2364 : *
2365 : * The following options are supported:
2366 : * <ul>
2367 : * <li>MIME headers such as Content-Type and Content-Encoding
2368 : * are supported for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ file systems.</li>
2369 : * <li>DISABLE_READDIR_ON_OPEN=YES/NO (GDAL >= 3.6) for /vsicurl/ and other
2370 : * network-based file systems. By default, directory file listing is done,
2371 : * unless YES is specified.</li>
2372 : * <li>WRITE_THROUGH=YES (GDAL >= 3.8) for the Windows regular files to
2373 : * set the FILE_FLAG_WRITE_THROUGH flag to the CreateFile() function. In that
2374 : * mode, the data is written to the system cache but is flushed to disk without
2375 : * delay.</li>
2376 : * </ul>
2377 : *
2378 : * Options specifics to /vsis3/, /vsigs/, /vsioss/ and /vsiaz/ in "w" mode:
2379 : * <ul>
2380 : * <li>CHUNK_SIZE=val in MiB. (GDAL >= 3.10) Size of a block. Default is 50 MiB.
2381 : * For /vsis3/, /vsigz/, /vsioss/, it can be up to 5000 MiB.
2382 : * For /vsiaz/, only taken into account when BLOB_TYPE=BLOCK. It can be up to 4000 MiB.
2383 : * </li>
2384 : * </ul>
2385 : *
2386 : * Options specifics to /vsiaz/ in "w" mode:
2387 : * <ul>
2388 : * <li>BLOB_TYPE=APPEND/BLOCK. (GDAL >= 3.10) Type of blob. Defaults to APPEND.
2389 : * Append blocks are limited to 195 GiB
2390 : * (however if the file size is below 4 MiB, a block blob will be created in a
2391 : * single PUT operation)
2392 : * </li>
2393 : * </ul>
2394 : *
2395 : * Analog of the POSIX fopen() function.
2396 : *
2397 : * @param pszFilename the file to open. UTF-8 encoded.
2398 : * @param pszAccess access requested (i.e. "r", "r+", "w")
2399 : * @param bSetError flag determining whether or not this open call
2400 : * should set VSIErrors on failure.
2401 : * @param papszOptions NULL or NULL-terminated list of strings. The content is
2402 : * highly file system dependent.
2403 : *
2404 : *
2405 : * @return NULL on failure, or the file handle.
2406 : *
2407 : * @since GDAL 3.3
2408 : */
2409 :
2410 385368 : VSILFILE *VSIFOpenEx2L(const char *pszFilename, const char *pszAccess,
2411 : int bSetError, CSLConstList papszOptions)
2412 :
2413 : {
2414 : // Too long filenames can cause excessive memory allocation due to
2415 : // recursion in some filesystem handlers
2416 385368 : constexpr size_t knMaxPath = 8192;
2417 385368 : if (CPLStrnlen(pszFilename, knMaxPath) == knMaxPath)
2418 0 : return nullptr;
2419 :
2420 385403 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2421 :
2422 385421 : VSILFILE *fp = poFSHandler->Open(pszFilename, pszAccess,
2423 385433 : CPL_TO_BOOL(bSetError), papszOptions);
2424 :
2425 : VSIDebug4("VSIFOpenEx2L(%s,%s,%d) = %p", pszFilename, pszAccess, bSetError,
2426 : fp);
2427 :
2428 385410 : return fp;
2429 : }
2430 :
2431 : /************************************************************************/
2432 : /* VSIFCloseL() */
2433 : /************************************************************************/
2434 :
2435 : /**
2436 : * \fn VSIVirtualHandle::Close()
2437 : * \brief Close file.
2438 : *
2439 : * This function closes the indicated file.
2440 : *
2441 : * This method goes through the VSIFileHandler virtualization and may
2442 : * work on unusual filesystems such as in memory.
2443 : *
2444 : * Analog of the POSIX fclose() function.
2445 : *
2446 : * @return 0 on success or -1 on failure.
2447 : */
2448 :
2449 : /**
2450 : * \brief Close file.
2451 : *
2452 : * This function closes the indicated file.
2453 : *
2454 : * This method goes through the VSIFileHandler virtualization and may
2455 : * work on unusual filesystems such as in memory.
2456 : *
2457 : * Analog of the POSIX fclose() function.
2458 : *
2459 : * @param fp file handle opened with VSIFOpenL(). Passing a nullptr produces
2460 : * undefined behavior.
2461 : *
2462 : * @return 0 on success or -1 on failure.
2463 : */
2464 :
2465 278418 : int VSIFCloseL(VSILFILE *fp)
2466 :
2467 : {
2468 : VSIDebug1("VSIFCloseL(%p)", fp);
2469 :
2470 278418 : const int nResult = fp->Close();
2471 :
2472 278215 : delete fp;
2473 :
2474 278040 : return nResult;
2475 : }
2476 :
2477 : /************************************************************************/
2478 : /* VSIFSeekL() */
2479 : /************************************************************************/
2480 :
2481 : /**
2482 : * \fn int VSIVirtualHandle::Seek( vsi_l_offset nOffset, int nWhence )
2483 : * \brief Seek to requested offset.
2484 : *
2485 : * Seek to the desired offset (nOffset) in the indicated file.
2486 : *
2487 : * This method goes through the VSIFileHandler virtualization and may
2488 : * work on unusual filesystems such as in memory.
2489 : *
2490 : * Analog of the POSIX fseek() call.
2491 : *
2492 : * Caution: vsi_l_offset is a unsigned type, so SEEK_CUR can only be used
2493 : * for positive seek. If negative seek is needed, use
2494 : * handle->Seek( handle->Tell() + negative_offset, SEEK_SET ).
2495 : *
2496 : * @param nOffset offset in bytes.
2497 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
2498 : *
2499 : * @return 0 on success or -1 one failure.
2500 : */
2501 :
2502 : /**
2503 : * \brief Seek to requested offset.
2504 : *
2505 : * Seek to the desired offset (nOffset) in the indicated file.
2506 : *
2507 : * This method goes through the VSIFileHandler virtualization and may
2508 : * work on unusual filesystems such as in memory.
2509 : *
2510 : * Analog of the POSIX fseek() call.
2511 : *
2512 : * Caution: vsi_l_offset is a unsigned type, so SEEK_CUR can only be used
2513 : * for positive seek. If negative seek is needed, use
2514 : * VSIFSeekL( fp, VSIFTellL(fp) + negative_offset, SEEK_SET ).
2515 : *
2516 : * @param fp file handle opened with VSIFOpenL().
2517 : * @param nOffset offset in bytes.
2518 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
2519 : *
2520 : * @return 0 on success or -1 one failure.
2521 : */
2522 :
2523 8134630 : int VSIFSeekL(VSILFILE *fp, vsi_l_offset nOffset, int nWhence)
2524 :
2525 : {
2526 8134630 : return fp->Seek(nOffset, nWhence);
2527 : }
2528 :
2529 : /************************************************************************/
2530 : /* VSIFTellL() */
2531 : /************************************************************************/
2532 :
2533 : /**
2534 : * \fn VSIVirtualHandle::Tell()
2535 : * \brief Tell current file offset.
2536 : *
2537 : * Returns the current file read/write offset in bytes from the beginning of
2538 : * the file.
2539 : *
2540 : * This method goes through the VSIFileHandler virtualization and may
2541 : * work on unusual filesystems such as in memory.
2542 : *
2543 : * Analog of the POSIX ftell() call.
2544 : *
2545 : * @return file offset in bytes.
2546 : */
2547 :
2548 : /**
2549 : * \brief Tell current file offset.
2550 : *
2551 : * Returns the current file read/write offset in bytes from the beginning of
2552 : * the file.
2553 : *
2554 : * This method goes through the VSIFileHandler virtualization and may
2555 : * work on unusual filesystems such as in memory.
2556 : *
2557 : * Analog of the POSIX ftell() call.
2558 : *
2559 : * @param fp file handle opened with VSIFOpenL().
2560 : *
2561 : * @return file offset in bytes.
2562 : */
2563 :
2564 5474180 : vsi_l_offset VSIFTellL(VSILFILE *fp)
2565 :
2566 : {
2567 5474180 : return fp->Tell();
2568 : }
2569 :
2570 : /************************************************************************/
2571 : /* VSIRewindL() */
2572 : /************************************************************************/
2573 :
2574 : /**
2575 : * \brief Rewind the file pointer to the beginning of the file.
2576 : *
2577 : * This is equivalent to VSIFSeekL( fp, 0, SEEK_SET )
2578 : *
2579 : * Analog of the POSIX rewind() call.
2580 : *
2581 : * @param fp file handle opened with VSIFOpenL().
2582 : */
2583 :
2584 77269 : void VSIRewindL(VSILFILE *fp)
2585 :
2586 : {
2587 77269 : CPL_IGNORE_RET_VAL(VSIFSeekL(fp, 0, SEEK_SET));
2588 77272 : }
2589 :
2590 : /************************************************************************/
2591 : /* VSIFFlushL() */
2592 : /************************************************************************/
2593 :
2594 : /**
2595 : * \fn VSIVirtualHandle::Flush()
2596 : * \brief Flush pending writes to disk.
2597 : *
2598 : * For files in write or update mode and on filesystem types where it is
2599 : * applicable, all pending output on the file is flushed to the physical disk.
2600 : *
2601 : * This method goes through the VSIFileHandler virtualization and may
2602 : * work on unusual filesystems such as in memory.
2603 : *
2604 : * Analog of the POSIX fflush() call.
2605 : *
2606 : * On Windows regular files, this method does nothing, unless the
2607 : * VSI_FLUSH configuration option is set to YES (and only when the file has
2608 : * *not* been opened with the WRITE_THROUGH option).
2609 : *
2610 : * @return 0 on success or -1 on error.
2611 : */
2612 :
2613 : /**
2614 : * \brief Flush pending writes to disk.
2615 : *
2616 : * For files in write or update mode and on filesystem types where it is
2617 : * applicable, all pending output on the file is flushed to the physical disk.
2618 : *
2619 : * This method goes through the VSIFileHandler virtualization and may
2620 : * work on unusual filesystems such as in memory.
2621 : *
2622 : * Analog of the POSIX fflush() call.
2623 : *
2624 : * On Windows regular files, this method does nothing, unless the
2625 : * VSI_FLUSH configuration option is set to YES (and only when the file has
2626 : * *not* been opened with the WRITE_THROUGH option).
2627 : *
2628 : * @param fp file handle opened with VSIFOpenL().
2629 : *
2630 : * @return 0 on success or -1 on error.
2631 : */
2632 :
2633 67988 : int VSIFFlushL(VSILFILE *fp)
2634 :
2635 : {
2636 67988 : return fp->Flush();
2637 : }
2638 :
2639 : /************************************************************************/
2640 : /* VSIFReadL() */
2641 : /************************************************************************/
2642 :
2643 : /**
2644 : * \fn VSIVirtualHandle::Read( void *pBuffer, size_t nSize, size_t nCount )
2645 : * \brief Read bytes from file.
2646 : *
2647 : * Reads nCount objects of nSize bytes from the indicated file at the
2648 : * current offset into the indicated buffer.
2649 : *
2650 : * This method goes through the VSIFileHandler virtualization and may
2651 : * work on unusual filesystems such as in memory.
2652 : *
2653 : * Analog of the POSIX fread() call.
2654 : *
2655 : * @param pBuffer the buffer into which the data should be read (at least
2656 : * nCount * nSize bytes in size.
2657 : * @param nSize size of objects to read in bytes.
2658 : * @param nCount number of objects to read.
2659 : *
2660 : * @return number of objects successfully read. If that number is less than
2661 : * nCount, VSIFEofL() or VSIFErrorL() can be used to determine the reason for
2662 : * the short read.
2663 : */
2664 :
2665 : /**
2666 : * \brief Read bytes from file.
2667 : *
2668 : * Reads nCount objects of nSize bytes from the indicated file at the
2669 : * current offset into the indicated buffer.
2670 : *
2671 : * This method goes through the VSIFileHandler virtualization and may
2672 : * work on unusual filesystems such as in memory.
2673 : *
2674 : * Analog of the POSIX fread() call.
2675 : *
2676 : * @param pBuffer the buffer into which the data should be read (at least
2677 : * nCount * nSize bytes in size.
2678 : * @param nSize size of objects to read in bytes.
2679 : * @param nCount number of objects to read.
2680 : * @param fp file handle opened with VSIFOpenL().
2681 : *
2682 : * @return number of objects successfully read. If that number is less than
2683 : * nCount, VSIFEofL() or VSIFErrorL() can be used to determine the reason for
2684 : * the short read.
2685 : */
2686 :
2687 14897600 : size_t VSIFReadL(void *pBuffer, size_t nSize, size_t nCount, VSILFILE *fp)
2688 :
2689 : {
2690 14897600 : return fp->Read(pBuffer, nSize, nCount);
2691 : }
2692 :
2693 : /************************************************************************/
2694 : /* VSIFReadMultiRangeL() */
2695 : /************************************************************************/
2696 :
2697 : /**
2698 : * \fn VSIVirtualHandle::ReadMultiRange( int nRanges, void ** ppData,
2699 : * const vsi_l_offset* panOffsets,
2700 : * const size_t* panSizes )
2701 : * \brief Read several ranges of bytes from file.
2702 : *
2703 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
2704 : * offset panOffsets[i] into the buffer ppData[i].
2705 : *
2706 : * Ranges must be sorted in ascending start offset, and must not overlap each
2707 : * other.
2708 : *
2709 : * This method goes through the VSIFileHandler virtualization and may
2710 : * work on unusual filesystems such as in memory or /vsicurl/.
2711 : *
2712 : * @param nRanges number of ranges to read.
2713 : * @param ppData array of nRanges buffer into which the data should be read
2714 : * (ppData[i] must be at list panSizes[i] bytes).
2715 : * @param panOffsets array of nRanges offsets at which the data should be read.
2716 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
2717 : *
2718 : * @return 0 in case of success, -1 otherwise.
2719 : * @since GDAL 1.9.0
2720 : */
2721 :
2722 : /**
2723 : * \brief Read several ranges of bytes from file.
2724 : *
2725 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
2726 : * offset panOffsets[i] into the buffer ppData[i].
2727 : *
2728 : * Ranges must be sorted in ascending start offset, and must not overlap each
2729 : * other.
2730 : *
2731 : * This method goes through the VSIFileHandler virtualization and may
2732 : * work on unusual filesystems such as in memory or /vsicurl/.
2733 : *
2734 : * @param nRanges number of ranges to read.
2735 : * @param ppData array of nRanges buffer into which the data should be read
2736 : * (ppData[i] must be at list panSizes[i] bytes).
2737 : * @param panOffsets array of nRanges offsets at which the data should be read.
2738 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
2739 : * @param fp file handle opened with VSIFOpenL().
2740 : *
2741 : * @return 0 in case of success, -1 otherwise.
2742 : * @since GDAL 1.9.0
2743 : */
2744 :
2745 604 : int VSIFReadMultiRangeL(int nRanges, void **ppData,
2746 : const vsi_l_offset *panOffsets, const size_t *panSizes,
2747 : VSILFILE *fp)
2748 : {
2749 604 : return fp->ReadMultiRange(nRanges, ppData, panOffsets, panSizes);
2750 : }
2751 :
2752 : /************************************************************************/
2753 : /* VSIFWriteL() */
2754 : /************************************************************************/
2755 :
2756 : /**
2757 : * \fn VSIVirtualHandle::Write( const void *pBuffer,
2758 : * size_t nSize, size_t nCount )
2759 : * \brief Write bytes to file.
2760 : *
2761 : * Writes nCount objects of nSize bytes to the indicated file at the
2762 : * current offset into the indicated buffer.
2763 : *
2764 : * This method goes through the VSIFileHandler virtualization and may
2765 : * work on unusual filesystems such as in memory.
2766 : *
2767 : * Analog of the POSIX fwrite() call.
2768 : *
2769 : * @param pBuffer the buffer from which the data should be written (at least
2770 : * nCount * nSize bytes in size.
2771 : * @param nSize size of objects to write in bytes.
2772 : * @param nCount number of objects to write.
2773 : *
2774 : * @return number of objects successfully written.
2775 : */
2776 :
2777 : /**
2778 : * \brief Write bytes to file.
2779 : *
2780 : * Writes nCount objects of nSize bytes to the indicated file at the
2781 : * current offset into the indicated buffer.
2782 : *
2783 : * This method goes through the VSIFileHandler virtualization and may
2784 : * work on unusual filesystems such as in memory.
2785 : *
2786 : * Analog of the POSIX fwrite() call.
2787 : *
2788 : * @param pBuffer the buffer from which the data should be written (at least
2789 : * nCount * nSize bytes in size.
2790 : * @param nSize size of objects to write in bytes.
2791 : * @param nCount number of objects to write.
2792 : * @param fp file handle opened with VSIFOpenL().
2793 : *
2794 : * @return number of objects successfully written.
2795 : */
2796 :
2797 4188780 : size_t VSIFWriteL(const void *pBuffer, size_t nSize, size_t nCount,
2798 : VSILFILE *fp)
2799 :
2800 : {
2801 4188780 : return fp->Write(pBuffer, nSize, nCount);
2802 : }
2803 :
2804 : /************************************************************************/
2805 : /* VSIFEofL() */
2806 : /************************************************************************/
2807 :
2808 : /**
2809 : * \fn VSIVirtualHandle::Eof()
2810 : * \brief Test for end of file.
2811 : *
2812 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
2813 : * previous read operation. The end-of-file flag is cleared by a successful
2814 : * VSIFSeekL() call, or a call to VSIFClearErrL().
2815 : *
2816 : * This method goes through the VSIFileHandler virtualization and may
2817 : * work on unusual filesystems such as in memory.
2818 : *
2819 : * Analog of the POSIX feof() call.
2820 : *
2821 : * @return TRUE if at EOF, else FALSE.
2822 : */
2823 :
2824 : /**
2825 : * \brief Test for end of file.
2826 : *
2827 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
2828 : * previous read operation. The end-of-file flag is cleared by a successful
2829 : * VSIFSeekL() call, or a call to VSIFClearErrL().
2830 : *
2831 : * This method goes through the VSIFileHandler virtualization and may
2832 : * work on unusual filesystems such as in memory.
2833 : *
2834 : * Analog of the POSIX feof() call.
2835 : *
2836 : * @param fp file handle opened with VSIFOpenL().
2837 : *
2838 : * @return TRUE if at EOF, else FALSE.
2839 : */
2840 :
2841 278632 : int VSIFEofL(VSILFILE *fp)
2842 :
2843 : {
2844 278632 : return fp->Eof();
2845 : }
2846 :
2847 : /************************************************************************/
2848 : /* VSIFErrorL() */
2849 : /************************************************************************/
2850 :
2851 : /**
2852 : * \fn VSIVirtualHandle::Error()
2853 : * \brief Test the error indicator.
2854 : *
2855 : * Returns TRUE (non-zero) if an error condition occurred during the
2856 : * previous read operation. The error indicator is cleared by a call to
2857 : * VSIFClearErrL(). Note that a end-of-file situation, reported by VSIFEofL(),
2858 : * is *not* an error reported by VSIFErrorL().
2859 : *
2860 : * This method goes through the VSIFileHandler virtualization and may
2861 : * work on unusual filesystems such as in memory.
2862 : *
2863 : * Analog of the POSIX ferror() call.
2864 : *
2865 : * @return TRUE if the error indicator is set, else FALSE.
2866 : * @since 3.10
2867 : */
2868 :
2869 : /**
2870 : * \brief Test the error indicator.
2871 : *
2872 : * Returns TRUE (non-zero) if an error condition occurred during the
2873 : * previous read operation. The error indicator is cleared by a call to
2874 : * VSIFClearErrL(). Note that a end-of-file situation, reported by VSIFEofL(),
2875 : * is *not* an error reported by VSIFErrorL().
2876 : *
2877 : * This method goes through the VSIFileHandler virtualization and may
2878 : * work on unusual filesystems such as in memory.
2879 : *
2880 : * Analog of the POSIX feof() call.
2881 : *
2882 : * @param fp file handle opened with VSIFOpenL().
2883 : *
2884 : * @return TRUE if the error indicator is set, else FALSE.
2885 : * @since 3.10
2886 : */
2887 :
2888 85496 : int VSIFErrorL(VSILFILE *fp)
2889 :
2890 : {
2891 85496 : return fp->Error();
2892 : }
2893 :
2894 : /************************************************************************/
2895 : /* VSIFClearErrL() */
2896 : /************************************************************************/
2897 :
2898 : /**
2899 : * \fn VSIVirtualHandle::ClearErr()
2900 : * \brief Reset the error and end-of-file indicators.
2901 : *
2902 : * This method goes through the VSIFileHandler virtualization and may
2903 : * work on unusual filesystems such as in memory.
2904 : *
2905 : * Analog of the POSIX clearerr() call.
2906 : *
2907 : * @since 3.10
2908 : */
2909 :
2910 : /**
2911 : * \brief Reset the error and end-of-file indicators.
2912 : *
2913 : * This method goes through the VSIFileHandler virtualization and may
2914 : * work on unusual filesystems such as in memory.
2915 : *
2916 : * Analog of the POSIX clearerr() call.
2917 : *
2918 : * @param fp file handle opened with VSIFOpenL().
2919 : *
2920 : * @since 3.10
2921 : */
2922 :
2923 29935 : void VSIFClearErrL(VSILFILE *fp)
2924 :
2925 : {
2926 29935 : fp->ClearErr();
2927 29935 : }
2928 :
2929 : /************************************************************************/
2930 : /* VSIFTruncateL() */
2931 : /************************************************************************/
2932 :
2933 : /**
2934 : * \fn VSIVirtualHandle::Truncate( vsi_l_offset nNewSize )
2935 : * \brief Truncate/expand the file to the specified size
2936 :
2937 : * This method goes through the VSIFileHandler virtualization and may
2938 : * work on unusual filesystems such as in memory.
2939 : *
2940 : * Analog of the POSIX ftruncate() call.
2941 : *
2942 : * @param nNewSize new size in bytes.
2943 : *
2944 : * @return 0 on success
2945 : * @since GDAL 1.9.0
2946 : */
2947 :
2948 : /**
2949 : * \brief Truncate/expand the file to the specified size
2950 :
2951 : * This method goes through the VSIFileHandler virtualization and may
2952 : * work on unusual filesystems such as in memory.
2953 : *
2954 : * Analog of the POSIX ftruncate() call.
2955 : *
2956 : * @param fp file handle opened with VSIFOpenL().
2957 : * @param nNewSize new size in bytes.
2958 : *
2959 : * @return 0 on success
2960 : * @since GDAL 1.9.0
2961 : */
2962 :
2963 1213 : int VSIFTruncateL(VSILFILE *fp, vsi_l_offset nNewSize)
2964 :
2965 : {
2966 1213 : return fp->Truncate(nNewSize);
2967 : }
2968 :
2969 : /************************************************************************/
2970 : /* VSIFPrintfL() */
2971 : /************************************************************************/
2972 :
2973 : /**
2974 : * \brief Formatted write to file.
2975 : *
2976 : * Provides fprintf() style formatted output to a VSI*L file. This formats
2977 : * an internal buffer which is written using VSIFWriteL().
2978 : *
2979 : * Analog of the POSIX fprintf() call.
2980 : *
2981 : * @param fp file handle opened with VSIFOpenL().
2982 : * @param pszFormat the printf() style format string.
2983 : *
2984 : * @return the number of bytes written or -1 on an error.
2985 : */
2986 :
2987 96064 : int VSIFPrintfL(VSILFILE *fp, CPL_FORMAT_STRING(const char *pszFormat), ...)
2988 :
2989 : {
2990 : va_list args;
2991 :
2992 96064 : va_start(args, pszFormat);
2993 96064 : CPLString osResult;
2994 96064 : osResult.vPrintf(pszFormat, args);
2995 96064 : va_end(args);
2996 :
2997 : return static_cast<int>(
2998 192128 : VSIFWriteL(osResult.c_str(), 1, osResult.length(), fp));
2999 : }
3000 :
3001 : /************************************************************************/
3002 : /* VSIVirtualHandle::Printf() */
3003 : /************************************************************************/
3004 :
3005 : /**
3006 : * \brief Formatted write to file.
3007 : *
3008 : * Provides fprintf() style formatted output to a VSI*L file. This formats
3009 : * an internal buffer which is written using VSIFWriteL().
3010 : *
3011 : * Analog of the POSIX fprintf() call.
3012 : *
3013 : * @param pszFormat the printf() style format string.
3014 : *
3015 : * @return the number of bytes written or -1 on an error.
3016 : */
3017 :
3018 770 : int VSIVirtualHandle::Printf(CPL_FORMAT_STRING(const char *pszFormat), ...)
3019 : {
3020 : va_list args;
3021 :
3022 770 : va_start(args, pszFormat);
3023 770 : CPLString osResult;
3024 770 : osResult.vPrintf(pszFormat, args);
3025 770 : va_end(args);
3026 :
3027 1540 : return static_cast<int>(Write(osResult.c_str(), 1, osResult.length()));
3028 : }
3029 :
3030 : /************************************************************************/
3031 : /* VSIFPutcL() */
3032 : /************************************************************************/
3033 :
3034 : // TODO: should we put in conformance with POSIX regarding the return
3035 : // value. As of today (2015-08-29), no code in GDAL sources actually
3036 : // check the return value.
3037 :
3038 : /**
3039 : * \brief Write a single byte to the file
3040 : *
3041 : * Writes the character nChar, cast to an unsigned char, to file.
3042 : *
3043 : * Almost an analog of the POSIX fputc() call, except that it returns
3044 : * the number of character written (1 or 0), and not the (cast)
3045 : * character itself or EOF.
3046 : *
3047 : * @param nChar character to write.
3048 : * @param fp file handle opened with VSIFOpenL().
3049 : *
3050 : * @return 1 in case of success, 0 on error.
3051 : */
3052 :
3053 2082 : int VSIFPutcL(int nChar, VSILFILE *fp)
3054 :
3055 : {
3056 2082 : const unsigned char cChar = static_cast<unsigned char>(nChar);
3057 2082 : return static_cast<int>(VSIFWriteL(&cChar, 1, 1, fp));
3058 : }
3059 :
3060 : /************************************************************************/
3061 : /* VSIFGetRangeStatusL() */
3062 : /************************************************************************/
3063 :
3064 : /**
3065 : * \fn VSIVirtualHandle::GetRangeStatus( vsi_l_offset nOffset,
3066 : * vsi_l_offset nLength )
3067 : * \brief Return if a given file range contains data or holes filled with zeroes
3068 : *
3069 : * This uses the filesystem capabilities of querying which regions of
3070 : * a sparse file are allocated or not. This is currently only
3071 : * implemented for Linux (and no other Unix derivatives) and Windows.
3072 : *
3073 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
3074 : * extent is filled with zeroes! It must be interpreted as "may
3075 : * contain non-zero data".
3076 : *
3077 : * @param nOffset offset of the start of the extent.
3078 : * @param nLength extent length.
3079 : *
3080 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
3081 : * VSI_RANGE_STATUS_HOLE
3082 : * @since GDAL 2.2
3083 : */
3084 :
3085 : /**
3086 : * \brief Return if a given file range contains data or holes filled with zeroes
3087 : *
3088 : * This uses the filesystem capabilities of querying which regions of
3089 : * a sparse file are allocated or not. This is currently only
3090 : * implemented for Linux (and no other Unix derivatives) and Windows.
3091 : *
3092 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
3093 : * extent is filled with zeroes! It must be interpreted as "may
3094 : * contain non-zero data".
3095 : *
3096 : * @param fp file handle opened with VSIFOpenL().
3097 : * @param nOffset offset of the start of the extent.
3098 : * @param nLength extent length.
3099 : *
3100 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
3101 : * VSI_RANGE_STATUS_HOLE
3102 : * @since GDAL 2.2
3103 : */
3104 :
3105 602 : VSIRangeStatus VSIFGetRangeStatusL(VSILFILE *fp, vsi_l_offset nOffset,
3106 : vsi_l_offset nLength)
3107 : {
3108 602 : return fp->GetRangeStatus(nOffset, nLength);
3109 : }
3110 :
3111 : /************************************************************************/
3112 : /* VSIIngestFile() */
3113 : /************************************************************************/
3114 :
3115 : /**
3116 : * \brief Ingest a file into memory.
3117 : *
3118 : * Read the whole content of a file into a memory buffer.
3119 : *
3120 : * Either fp or pszFilename can be NULL, but not both at the same time.
3121 : *
3122 : * If fp is passed non-NULL, it is the responsibility of the caller to
3123 : * close it.
3124 : *
3125 : * If non-NULL, the returned buffer is guaranteed to be NUL-terminated.
3126 : *
3127 : * @param fp file handle opened with VSIFOpenL().
3128 : * @param pszFilename filename.
3129 : * @param ppabyRet pointer to the target buffer. *ppabyRet must be freed with
3130 : * VSIFree()
3131 : * @param pnSize pointer to variable to store the file size. May be NULL.
3132 : * @param nMaxSize maximum size of file allowed. If no limit, set to a negative
3133 : * value.
3134 : *
3135 : * @return TRUE in case of success.
3136 : *
3137 : * @since GDAL 1.11
3138 : */
3139 :
3140 9375 : int VSIIngestFile(VSILFILE *fp, const char *pszFilename, GByte **ppabyRet,
3141 : vsi_l_offset *pnSize, GIntBig nMaxSize)
3142 : {
3143 9375 : if (fp == nullptr && pszFilename == nullptr)
3144 0 : return FALSE;
3145 9375 : if (ppabyRet == nullptr)
3146 0 : return FALSE;
3147 :
3148 9375 : *ppabyRet = nullptr;
3149 9375 : if (pnSize != nullptr)
3150 2780 : *pnSize = 0;
3151 :
3152 9375 : bool bFreeFP = false;
3153 9375 : if (nullptr == fp)
3154 : {
3155 8225 : fp = VSIFOpenL(pszFilename, "rb");
3156 8225 : if (nullptr == fp)
3157 : {
3158 327 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open file '%s'",
3159 : pszFilename);
3160 327 : return FALSE;
3161 : }
3162 7898 : bFreeFP = true;
3163 : }
3164 : else
3165 : {
3166 1150 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3167 0 : return FALSE;
3168 : }
3169 :
3170 9048 : vsi_l_offset nDataLen = 0;
3171 :
3172 9048 : if (pszFilename == nullptr || strcmp(pszFilename, "/vsistdin/") == 0)
3173 : {
3174 100 : vsi_l_offset nDataAlloc = 0;
3175 100 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3176 : {
3177 0 : if (bFreeFP)
3178 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3179 0 : return FALSE;
3180 : }
3181 : while (true)
3182 : {
3183 507 : if (nDataLen + 8192 + 1 > nDataAlloc)
3184 : {
3185 233 : nDataAlloc = (nDataAlloc * 4) / 3 + 8192 + 1;
3186 : if (nDataAlloc >
3187 : static_cast<vsi_l_offset>(static_cast<size_t>(nDataAlloc)))
3188 : {
3189 : CPLError(CE_Failure, CPLE_AppDefined,
3190 : "Input file too large to be opened");
3191 : VSIFree(*ppabyRet);
3192 : *ppabyRet = nullptr;
3193 : if (bFreeFP)
3194 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3195 : return FALSE;
3196 : }
3197 : GByte *pabyNew = static_cast<GByte *>(
3198 233 : VSIRealloc(*ppabyRet, static_cast<size_t>(nDataAlloc)));
3199 233 : if (pabyNew == nullptr)
3200 : {
3201 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
3202 : "Cannot allocate " CPL_FRMT_GIB " bytes",
3203 : nDataAlloc);
3204 0 : VSIFree(*ppabyRet);
3205 0 : *ppabyRet = nullptr;
3206 0 : if (bFreeFP)
3207 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3208 0 : return FALSE;
3209 : }
3210 233 : *ppabyRet = pabyNew;
3211 : }
3212 : const int nRead =
3213 507 : static_cast<int>(VSIFReadL(*ppabyRet + nDataLen, 1, 8192, fp));
3214 507 : nDataLen += nRead;
3215 :
3216 507 : if (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize))
3217 : {
3218 0 : CPLError(CE_Failure, CPLE_AppDefined,
3219 : "Input file too large to be opened");
3220 0 : VSIFree(*ppabyRet);
3221 0 : *ppabyRet = nullptr;
3222 0 : if (pnSize != nullptr)
3223 0 : *pnSize = 0;
3224 0 : if (bFreeFP)
3225 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3226 0 : return FALSE;
3227 : }
3228 :
3229 507 : if (pnSize != nullptr)
3230 245 : *pnSize += nRead;
3231 507 : (*ppabyRet)[nDataLen] = '\0';
3232 507 : if (nRead == 0)
3233 100 : break;
3234 507 : }
3235 : }
3236 : else
3237 : {
3238 8948 : if (VSIFSeekL(fp, 0, SEEK_END) != 0)
3239 : {
3240 0 : if (bFreeFP)
3241 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3242 0 : return FALSE;
3243 : }
3244 8948 : nDataLen = VSIFTellL(fp);
3245 :
3246 : // With "large" VSI I/O API we can read data chunks larger than
3247 : // VSIMalloc could allocate. Catch it here.
3248 8948 : if (nDataLen !=
3249 : static_cast<vsi_l_offset>(static_cast<size_t>(nDataLen)) ||
3250 : nDataLen + 1 < nDataLen
3251 : // opening a directory returns nDataLen = INT_MAX (on 32bit) or
3252 : // INT64_MAX (on 64bit)
3253 12697 : || nDataLen + 1 > std::numeric_limits<size_t>::max() / 2 ||
3254 3749 : (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize)))
3255 : {
3256 0 : CPLError(CE_Failure, CPLE_AppDefined,
3257 : "Input file too large to be opened");
3258 0 : if (bFreeFP)
3259 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3260 0 : return FALSE;
3261 : }
3262 :
3263 8948 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
3264 : {
3265 0 : if (bFreeFP)
3266 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3267 0 : return FALSE;
3268 : }
3269 :
3270 8948 : *ppabyRet =
3271 8948 : static_cast<GByte *>(VSIMalloc(static_cast<size_t>(nDataLen + 1)));
3272 8948 : if (nullptr == *ppabyRet)
3273 : {
3274 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
3275 : "Cannot allocate " CPL_FRMT_GIB " bytes", nDataLen + 1);
3276 0 : if (bFreeFP)
3277 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3278 0 : return FALSE;
3279 : }
3280 :
3281 8948 : (*ppabyRet)[nDataLen] = '\0';
3282 8948 : if (nDataLen !=
3283 8948 : VSIFReadL(*ppabyRet, 1, static_cast<size_t>(nDataLen), fp))
3284 : {
3285 0 : CPLError(CE_Failure, CPLE_FileIO,
3286 : "Cannot read " CPL_FRMT_GIB " bytes", nDataLen);
3287 0 : VSIFree(*ppabyRet);
3288 0 : *ppabyRet = nullptr;
3289 0 : if (bFreeFP)
3290 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3291 0 : return FALSE;
3292 : }
3293 8948 : if (pnSize != nullptr)
3294 2424 : *pnSize = nDataLen;
3295 : }
3296 9048 : if (bFreeFP)
3297 7898 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
3298 9048 : return TRUE;
3299 : }
3300 :
3301 : /************************************************************************/
3302 : /* VSIOverwriteFile() */
3303 : /************************************************************************/
3304 :
3305 : /**
3306 : * \brief Overwrite an existing file with content from another one
3307 : *
3308 : * @param fpTarget file handle opened with VSIFOpenL() with "rb+" flag.
3309 : * @param pszSourceFilename source filename
3310 : *
3311 : * @return TRUE in case of success.
3312 : *
3313 : * @since GDAL 3.1
3314 : */
3315 :
3316 4 : int VSIOverwriteFile(VSILFILE *fpTarget, const char *pszSourceFilename)
3317 : {
3318 4 : VSILFILE *fpSource = VSIFOpenL(pszSourceFilename, "rb");
3319 4 : if (fpSource == nullptr)
3320 : {
3321 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSourceFilename);
3322 0 : return false;
3323 : }
3324 :
3325 4 : const size_t nBufferSize = 4096;
3326 4 : void *pBuffer = CPLMalloc(nBufferSize);
3327 4 : VSIFSeekL(fpTarget, 0, SEEK_SET);
3328 4 : bool bRet = true;
3329 : while (true)
3330 : {
3331 4 : size_t nRead = VSIFReadL(pBuffer, 1, nBufferSize, fpSource);
3332 4 : size_t nWritten = VSIFWriteL(pBuffer, 1, nRead, fpTarget);
3333 4 : if (nWritten != nRead)
3334 : {
3335 0 : bRet = false;
3336 0 : break;
3337 : }
3338 4 : if (nRead < nBufferSize)
3339 4 : break;
3340 0 : }
3341 :
3342 4 : if (bRet)
3343 : {
3344 4 : bRet = VSIFTruncateL(fpTarget, VSIFTellL(fpTarget)) == 0;
3345 4 : if (!bRet)
3346 : {
3347 0 : CPLError(CE_Failure, CPLE_FileIO, "Truncation failed");
3348 : }
3349 : }
3350 :
3351 4 : CPLFree(pBuffer);
3352 4 : VSIFCloseL(fpSource);
3353 4 : return bRet;
3354 : }
3355 :
3356 : /************************************************************************/
3357 : /* VSIFGetNativeFileDescriptorL() */
3358 : /************************************************************************/
3359 :
3360 : /**
3361 : * \fn VSIVirtualHandle::GetNativeFileDescriptor()
3362 : * \brief Returns the "native" file descriptor for the virtual handle.
3363 : *
3364 : * This will only return a non-NULL value for "real" files handled by the
3365 : * operating system (to be opposed to GDAL virtual file systems).
3366 : *
3367 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
3368 : * On Windows systems, this will be the HANDLE.
3369 : *
3370 : * @return the native file descriptor, or NULL.
3371 : */
3372 :
3373 : /**
3374 : * \brief Returns the "native" file descriptor for the virtual handle.
3375 : *
3376 : * This will only return a non-NULL value for "real" files handled by the
3377 : * operating system (to be opposed to GDAL virtual file systems).
3378 : *
3379 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
3380 : * On Windows systems, this will be the HANDLE.
3381 : *
3382 : * @param fp file handle opened with VSIFOpenL().
3383 : *
3384 : * @return the native file descriptor, or NULL.
3385 : */
3386 :
3387 64 : void *VSIFGetNativeFileDescriptorL(VSILFILE *fp)
3388 : {
3389 64 : return fp->GetNativeFileDescriptor();
3390 : }
3391 :
3392 : /************************************************************************/
3393 : /* VSIGetDiskFreeSpace() */
3394 : /************************************************************************/
3395 :
3396 : /**
3397 : * \brief Return free disk space available on the filesystem.
3398 : *
3399 : * This function returns the free disk space available on the filesystem.
3400 : *
3401 : * @param pszDirname a directory of the filesystem to query.
3402 : * @return The free space in bytes. Or -1 in case of error.
3403 : * @since GDAL 2.1
3404 : */
3405 :
3406 73 : GIntBig VSIGetDiskFreeSpace(const char *pszDirname)
3407 : {
3408 73 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
3409 :
3410 73 : return poFSHandler->GetDiskFreeSpace(pszDirname);
3411 : }
3412 :
3413 : /************************************************************************/
3414 : /* VSIGetFileSystemsPrefixes() */
3415 : /************************************************************************/
3416 :
3417 : /**
3418 : * \brief Return the list of prefixes for virtual file system handlers
3419 : * currently registered.
3420 : *
3421 : * Typically: "", "/vsimem/", "/vsicurl/", etc
3422 : *
3423 : * @return a NULL terminated list of prefixes. Must be freed with CSLDestroy()
3424 : * @since GDAL 2.3
3425 : */
3426 :
3427 2 : char **VSIGetFileSystemsPrefixes(void)
3428 : {
3429 2 : return VSIFileManager::GetPrefixes();
3430 : }
3431 :
3432 : /************************************************************************/
3433 : /* VSIGetFileSystemOptions() */
3434 : /************************************************************************/
3435 :
3436 : /**
3437 : * \brief Return the list of options associated with a virtual file system
3438 : * handler as a serialized XML string.
3439 : *
3440 : * Those options may be set as configuration options with CPLSetConfigOption().
3441 : *
3442 : * @param pszFilename a filename, or prefix of a virtual file system handler.
3443 : * @return a string, which must not be freed, or NULL if no options is declared.
3444 : * @since GDAL 2.3
3445 : */
3446 :
3447 33 : const char *VSIGetFileSystemOptions(const char *pszFilename)
3448 : {
3449 33 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
3450 :
3451 33 : return poFSHandler->GetOptions();
3452 : }
3453 :
3454 : /************************************************************************/
3455 : /* VSISetPathSpecificOption() */
3456 : /************************************************************************/
3457 :
3458 : static std::mutex oMutexPathSpecificOptions;
3459 :
3460 : // key is a path prefix
3461 : // value is a map of key, value pair
3462 : static std::map<std::string, std::map<std::string, std::string>>
3463 : oMapPathSpecificOptions;
3464 :
3465 : /**
3466 : * \brief Set a credential (or more generally an option related to a
3467 : * virtual file system) for a given path prefix.
3468 : * @deprecated in GDAL 3.6 for the better named VSISetPathSpecificOption()
3469 : * @see VSISetPathSpecificOption()
3470 : */
3471 0 : void VSISetCredential(const char *pszPathPrefix, const char *pszKey,
3472 : const char *pszValue)
3473 : {
3474 0 : VSISetPathSpecificOption(pszPathPrefix, pszKey, pszValue);
3475 0 : }
3476 :
3477 : /**
3478 : * \brief Set a path specific option for a given path prefix.
3479 : *
3480 : * Such option is typically, but not limited to, a credential setting for a
3481 : * virtual file system.
3482 : *
3483 : * That option may also be set as a configuration option with
3484 : * CPLSetConfigOption(), but this function allows to specify them with a
3485 : * granularity at the level of a file path, which makes it easier if using the
3486 : * same virtual file system but with different credentials (e.g. different
3487 : * credentials for bucket "/vsis3/foo" and "/vsis3/bar")
3488 : *
3489 : * This is supported for the following virtual file systems:
3490 : * /vsis3/, /vsigs/, /vsiaz/, /vsioss/, /vsiwebhdfs, /vsiswift.
3491 : * Note: setting them for a path starting with /vsiXXX/ will also apply for
3492 : * /vsiXXX_streaming/ requests.
3493 : *
3494 : * Note that no particular care is taken to store them in RAM in a secure way.
3495 : * So they might accidentally hit persistent storage if swapping occurs, or
3496 : * someone with access to the memory allocated by the process may be able to
3497 : * read them.
3498 : *
3499 : * @param pszPathPrefix a path prefix of a virtual file system handler.
3500 : * Typically of the form "/vsiXXX/bucket". Must NOT be
3501 : * NULL. Should not include trailing slashes.
3502 : * @param pszKey Option name. Must NOT be NULL.
3503 : * @param pszValue Option value. May be NULL to erase it.
3504 : *
3505 : * @since GDAL 3.6
3506 : */
3507 :
3508 82 : void VSISetPathSpecificOption(const char *pszPathPrefix, const char *pszKey,
3509 : const char *pszValue)
3510 : {
3511 164 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
3512 82 : auto oIter = oMapPathSpecificOptions.find(pszPathPrefix);
3513 164 : CPLString osKey(pszKey);
3514 82 : osKey.toupper();
3515 82 : if (oIter == oMapPathSpecificOptions.end())
3516 : {
3517 21 : if (pszValue != nullptr)
3518 21 : oMapPathSpecificOptions[pszPathPrefix][osKey] = pszValue;
3519 : }
3520 61 : else if (pszValue != nullptr)
3521 57 : oIter->second[osKey] = pszValue;
3522 : else
3523 4 : oIter->second.erase(osKey);
3524 82 : }
3525 :
3526 : /************************************************************************/
3527 : /* VSIClearPathSpecificOptions() */
3528 : /************************************************************************/
3529 :
3530 : /**
3531 : * \brief Clear path specific options set with VSISetPathSpecificOption()
3532 : * @deprecated in GDAL 3.6 for the better named VSIClearPathSpecificOptions()
3533 : * @see VSIClearPathSpecificOptions()
3534 : */
3535 0 : void VSIClearCredentials(const char *pszPathPrefix)
3536 : {
3537 0 : return VSIClearPathSpecificOptions(pszPathPrefix);
3538 : }
3539 :
3540 : /**
3541 : * \brief Clear path specific options set with VSISetPathSpecificOption()
3542 : *
3543 : * Note that no particular care is taken to remove them from RAM in a secure
3544 : * way.
3545 : *
3546 : * @param pszPathPrefix If set to NULL, all path specific options are cleared.
3547 : * If set to not-NULL, only those set with
3548 : * VSISetPathSpecificOption(pszPathPrefix, ...) will be
3549 : * cleared.
3550 : *
3551 : * @since GDAL 3.6
3552 : */
3553 18 : void VSIClearPathSpecificOptions(const char *pszPathPrefix)
3554 : {
3555 36 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
3556 18 : if (pszPathPrefix == nullptr)
3557 : {
3558 2 : oMapPathSpecificOptions.clear();
3559 : }
3560 : else
3561 : {
3562 16 : oMapPathSpecificOptions.erase(pszPathPrefix);
3563 : }
3564 18 : }
3565 :
3566 : /************************************************************************/
3567 : /* VSIGetPathSpecificOption() */
3568 : /************************************************************************/
3569 :
3570 : /**
3571 : * \brief Get the value of a credential (or more generally an option related to
3572 : * a virtual file system) for a given path.
3573 : * @deprecated in GDAL 3.6 for the better named VSIGetPathSpecificOption()
3574 : * @see VSIGetPathSpecificOption()
3575 : */
3576 0 : const char *VSIGetCredential(const char *pszPath, const char *pszKey,
3577 : const char *pszDefault)
3578 : {
3579 0 : return VSIGetPathSpecificOption(pszPath, pszKey, pszDefault);
3580 : }
3581 :
3582 : /**
3583 : * \brief Get the value a path specific option.
3584 : *
3585 : * Such option is typically, but not limited to, a credential setting for a
3586 : * virtual file system.
3587 : *
3588 : * If no match occurs, CPLGetConfigOption(pszKey, pszDefault) is returned.
3589 : *
3590 : * Mostly to be used by virtual file system implementations.
3591 : *
3592 : * @since GDAL 3.6
3593 : * @see VSISetPathSpecificOption()
3594 : */
3595 126491 : const char *VSIGetPathSpecificOption(const char *pszPath, const char *pszKey,
3596 : const char *pszDefault)
3597 : {
3598 : {
3599 126491 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
3600 145957 : for (auto it = oMapPathSpecificOptions.rbegin();
3601 165421 : it != oMapPathSpecificOptions.rend(); ++it)
3602 : {
3603 19725 : if (STARTS_WITH(pszPath, it->first.c_str()))
3604 : {
3605 2990 : auto oIter = it->second.find(CPLString(pszKey).toupper());
3606 2990 : if (oIter != it->second.end())
3607 261 : return oIter->second.c_str();
3608 : }
3609 : }
3610 : }
3611 126232 : return CPLGetConfigOption(pszKey, pszDefault);
3612 : }
3613 :
3614 : /************************************************************************/
3615 : /* VSIDuplicateFileSystemHandler() */
3616 : /************************************************************************/
3617 :
3618 : /**
3619 : * \brief Duplicate an existing file system handler.
3620 : *
3621 : * A number of virtual file system for remote object stores use protocols
3622 : * identical or close to popular ones (typically AWS S3), but with slightly
3623 : * different settings (at the very least the endpoint).
3624 : *
3625 : * This functions allows to duplicate the source virtual file system handler
3626 : * as a new one with a different prefix (when the source virtual file system
3627 : * handler supports the duplication operation).
3628 : *
3629 : * VSISetPathSpecificOption() will typically be called afterwards to change
3630 : * configurable settings on the cloned file system handler (e.g. AWS_S3_ENDPOINT
3631 : * for a clone of /vsis3/).
3632 : *
3633 : * @since GDAL 3.7
3634 : */
3635 4 : bool VSIDuplicateFileSystemHandler(const char *pszSourceFSName,
3636 : const char *pszNewFSName)
3637 : {
3638 : VSIFilesystemHandler *poTargetFSHandler =
3639 4 : VSIFileManager::GetHandler(pszNewFSName);
3640 4 : if (poTargetFSHandler != VSIFileManager::GetHandler("/"))
3641 : {
3642 1 : CPLError(CE_Failure, CPLE_AppDefined,
3643 : "%s is already a known virtual file system", pszNewFSName);
3644 1 : return false;
3645 : }
3646 :
3647 : VSIFilesystemHandler *poSourceFSHandler =
3648 3 : VSIFileManager::GetHandler(pszSourceFSName);
3649 3 : if (!poSourceFSHandler)
3650 : {
3651 0 : CPLError(CE_Failure, CPLE_AppDefined,
3652 : "%s is not a known virtual file system", pszSourceFSName);
3653 0 : return false;
3654 : }
3655 :
3656 3 : poTargetFSHandler = poSourceFSHandler->Duplicate(pszNewFSName);
3657 3 : if (!poTargetFSHandler)
3658 2 : return false;
3659 :
3660 1 : VSIFileManager::InstallHandler(pszNewFSName, poTargetFSHandler);
3661 1 : return true;
3662 : }
3663 :
3664 : /************************************************************************/
3665 : /* ==================================================================== */
3666 : /* VSIFileManager() */
3667 : /* ==================================================================== */
3668 : /************************************************************************/
3669 :
3670 : #ifndef DOXYGEN_SKIP
3671 :
3672 : /*
3673 : ** Notes on Multithreading:
3674 : **
3675 : ** The VSIFileManager maintains a list of file type handlers (mem, large
3676 : ** file, etc). It should be thread safe.
3677 : **/
3678 :
3679 : /************************************************************************/
3680 : /* VSIFileManager() */
3681 : /************************************************************************/
3682 :
3683 1304 : VSIFileManager::VSIFileManager() : poDefaultHandler(nullptr)
3684 : {
3685 1304 : }
3686 :
3687 : /************************************************************************/
3688 : /* ~VSIFileManager() */
3689 : /************************************************************************/
3690 :
3691 933 : VSIFileManager::~VSIFileManager()
3692 : {
3693 1866 : std::set<VSIFilesystemHandler *> oSetAlreadyDeleted;
3694 26126 : for (std::map<std::string, VSIFilesystemHandler *>::const_iterator iter =
3695 933 : oHandlers.begin();
3696 53185 : iter != oHandlers.end(); ++iter)
3697 : {
3698 26126 : if (oSetAlreadyDeleted.find(iter->second) == oSetAlreadyDeleted.end())
3699 : {
3700 24260 : oSetAlreadyDeleted.insert(iter->second);
3701 24260 : delete iter->second;
3702 : }
3703 : }
3704 :
3705 933 : delete poDefaultHandler;
3706 933 : }
3707 :
3708 : /************************************************************************/
3709 : /* Get() */
3710 : /************************************************************************/
3711 :
3712 : static VSIFileManager *poManager = nullptr;
3713 : static CPLMutex *hVSIFileManagerMutex = nullptr;
3714 :
3715 2530860 : VSIFileManager *VSIFileManager::Get()
3716 : {
3717 5061880 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
3718 2531020 : if (poManager != nullptr)
3719 : {
3720 2529710 : return poManager;
3721 : }
3722 :
3723 1304 : poManager = new VSIFileManager;
3724 1304 : VSIInstallLargeFileHandler();
3725 1304 : VSIInstallSubFileHandler();
3726 1304 : VSIInstallMemFileHandler();
3727 : #ifdef HAVE_LIBZ
3728 1304 : VSIInstallGZipFileHandler();
3729 1304 : VSIInstallZipFileHandler();
3730 : #endif
3731 : #ifdef HAVE_LIBARCHIVE
3732 : VSIInstall7zFileHandler();
3733 : VSIInstallRarFileHandler();
3734 : #endif
3735 : #ifdef HAVE_CURL
3736 1304 : VSIInstallCurlFileHandler();
3737 1304 : VSIInstallCurlStreamingFileHandler();
3738 1304 : VSIInstallS3FileHandler();
3739 1304 : VSIInstallS3StreamingFileHandler();
3740 1304 : VSIInstallGSFileHandler();
3741 1304 : VSIInstallGSStreamingFileHandler();
3742 1304 : VSIInstallAzureFileHandler();
3743 1304 : VSIInstallAzureStreamingFileHandler();
3744 1304 : VSIInstallADLSFileHandler();
3745 1304 : VSIInstallOSSFileHandler();
3746 1304 : VSIInstallOSSStreamingFileHandler();
3747 1304 : VSIInstallSwiftFileHandler();
3748 1304 : VSIInstallSwiftStreamingFileHandler();
3749 1304 : VSIInstallWebHdfsHandler();
3750 : #endif
3751 1304 : VSIInstallStdinHandler();
3752 1304 : VSIInstallHdfsHandler();
3753 1304 : VSIInstallStdoutHandler();
3754 1304 : VSIInstallSparseFileHandler();
3755 1304 : VSIInstallTarFileHandler();
3756 1304 : VSIInstallCachedFileHandler();
3757 1304 : VSIInstallCryptFileHandler();
3758 :
3759 1304 : return poManager;
3760 : }
3761 :
3762 : /************************************************************************/
3763 : /* GetPrefixes() */
3764 : /************************************************************************/
3765 :
3766 600 : char **VSIFileManager::GetPrefixes()
3767 : {
3768 1200 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
3769 1200 : CPLStringList aosList;
3770 17400 : for (const auto &oIter : Get()->oHandlers)
3771 : {
3772 16800 : if (oIter.first != "/vsicurl?")
3773 : {
3774 16200 : aosList.AddString(oIter.first.c_str());
3775 : }
3776 : }
3777 1200 : return aosList.StealList();
3778 : }
3779 :
3780 : /************************************************************************/
3781 : /* GetHandler() */
3782 : /************************************************************************/
3783 :
3784 2492520 : VSIFilesystemHandler *VSIFileManager::GetHandler(const char *pszPath)
3785 :
3786 : {
3787 2492520 : VSIFileManager *poThis = Get();
3788 2492610 : const size_t nPathLen = strlen(pszPath);
3789 :
3790 48832300 : for (std::map<std::string, VSIFilesystemHandler *>::const_iterator iter =
3791 2492610 : poThis->oHandlers.begin();
3792 100153000 : iter != poThis->oHandlers.end(); ++iter)
3793 : {
3794 50537300 : const char *pszIterKey = iter->first.c_str();
3795 50540700 : const size_t nIterKeyLen = iter->first.size();
3796 50546600 : if (strncmp(pszPath, pszIterKey, nIterKeyLen) == 0)
3797 1718550 : return iter->second;
3798 :
3799 : // "/vsimem\foo" should be handled as "/vsimem/foo".
3800 48940600 : if (nIterKeyLen && nPathLen > nIterKeyLen &&
3801 44162600 : pszIterKey[nIterKeyLen - 1] == '/' &&
3802 38450900 : pszPath[nIterKeyLen - 1] == '\\' &&
3803 27 : strncmp(pszPath, pszIterKey, nIterKeyLen - 1) == 0)
3804 0 : return iter->second;
3805 :
3806 : // /vsimem should be treated as a match for /vsimem/.
3807 48940600 : if (nPathLen + 1 == nIterKeyLen &&
3808 495382 : strncmp(pszPath, pszIterKey, nPathLen) == 0)
3809 112530 : return iter->second;
3810 : }
3811 :
3812 773969 : return poThis->poDefaultHandler;
3813 : }
3814 :
3815 : /************************************************************************/
3816 : /* InstallHandler() */
3817 : /************************************************************************/
3818 :
3819 37808 : void VSIFileManager::InstallHandler(const std::string &osPrefix,
3820 : VSIFilesystemHandler *poHandler)
3821 :
3822 : {
3823 37808 : if (osPrefix == "")
3824 1304 : Get()->poDefaultHandler = poHandler;
3825 : else
3826 36504 : Get()->oHandlers[osPrefix] = poHandler;
3827 37808 : }
3828 :
3829 : /************************************************************************/
3830 : /* RemoveHandler() */
3831 : /************************************************************************/
3832 :
3833 3 : void VSIFileManager::RemoveHandler(const std::string &osPrefix)
3834 : {
3835 3 : if (osPrefix == "")
3836 0 : Get()->poDefaultHandler = nullptr;
3837 : else
3838 3 : Get()->oHandlers.erase(osPrefix);
3839 3 : }
3840 :
3841 : /************************************************************************/
3842 : /* VSICleanupFileManager() */
3843 : /************************************************************************/
3844 :
3845 933 : void VSICleanupFileManager()
3846 :
3847 : {
3848 933 : if (poManager)
3849 : {
3850 933 : delete poManager;
3851 933 : poManager = nullptr;
3852 : }
3853 :
3854 933 : if (hVSIFileManagerMutex != nullptr)
3855 : {
3856 933 : CPLDestroyMutex(hVSIFileManagerMutex);
3857 933 : hVSIFileManagerMutex = nullptr;
3858 : }
3859 :
3860 : #ifdef HAVE_CURL
3861 933 : VSICURLDestroyCacheFileProp();
3862 : #endif
3863 933 : }
3864 :
3865 : /************************************************************************/
3866 : /* Truncate() */
3867 : /************************************************************************/
3868 :
3869 2 : int VSIVirtualHandle::Truncate(vsi_l_offset nNewSize)
3870 : {
3871 2 : const vsi_l_offset nOriginalPos = Tell();
3872 2 : if (Seek(0, SEEK_END) == 0 && nNewSize >= Tell())
3873 : {
3874 : // Fill with zeroes
3875 2 : std::vector<GByte> aoBytes(4096, 0);
3876 1 : vsi_l_offset nCurOffset = nOriginalPos;
3877 3 : while (nCurOffset < nNewSize)
3878 : {
3879 2 : constexpr vsi_l_offset nMaxOffset = 4096;
3880 : const int nSize =
3881 2 : static_cast<int>(std::min(nMaxOffset, nNewSize - nCurOffset));
3882 2 : if (Write(&aoBytes[0], nSize, 1) != 1)
3883 : {
3884 0 : Seek(nOriginalPos, SEEK_SET);
3885 0 : return -1;
3886 : }
3887 2 : nCurOffset += nSize;
3888 : }
3889 1 : return Seek(nOriginalPos, SEEK_SET) == 0 ? 0 : -1;
3890 : }
3891 :
3892 1 : CPLDebug("VSI", "Truncation is not supported in generic implementation "
3893 : "of Truncate()");
3894 1 : Seek(nOriginalPos, SEEK_SET);
3895 1 : return -1;
3896 : }
3897 :
3898 : /************************************************************************/
3899 : /* ReadMultiRange() */
3900 : /************************************************************************/
3901 :
3902 603 : int VSIVirtualHandle::ReadMultiRange(int nRanges, void **ppData,
3903 : const vsi_l_offset *panOffsets,
3904 : const size_t *panSizes)
3905 : {
3906 603 : int nRet = 0;
3907 603 : const vsi_l_offset nCurOffset = Tell();
3908 46075 : for (int i = 0; i < nRanges; i++)
3909 : {
3910 45493 : if (Seek(panOffsets[i], SEEK_SET) < 0)
3911 : {
3912 0 : nRet = -1;
3913 0 : break;
3914 : }
3915 :
3916 45493 : const size_t nRead = Read(ppData[i], 1, panSizes[i]);
3917 45493 : if (panSizes[i] != nRead)
3918 : {
3919 21 : nRet = -1;
3920 21 : break;
3921 : }
3922 : }
3923 :
3924 603 : Seek(nCurOffset, SEEK_SET);
3925 :
3926 603 : return nRet;
3927 : }
3928 :
3929 : #endif // #ifndef DOXYGEN_SKIP
3930 :
3931 : /************************************************************************/
3932 : /* HasPRead() */
3933 : /************************************************************************/
3934 :
3935 : /** Returns whether this file handle supports the PRead() method.
3936 : *
3937 : * @since GDAL 3.6
3938 : */
3939 0 : bool VSIVirtualHandle::HasPRead() const
3940 : {
3941 0 : return false;
3942 : }
3943 :
3944 : /************************************************************************/
3945 : /* PRead() */
3946 : /************************************************************************/
3947 :
3948 : /** Do a parallel-compatible read operation.
3949 : *
3950 : * This methods reads into pBuffer up to nSize bytes starting at offset nOffset
3951 : * in the file. The current file offset is not affected by this method.
3952 : *
3953 : * The implementation is thread-safe: several threads can issue PRead()
3954 : * concurrently on the same VSIVirtualHandle object.
3955 : *
3956 : * This method has the same semantics as pread() Linux operation. It is only
3957 : * available if HasPRead() returns true.
3958 : *
3959 : * @param pBuffer output buffer (must be at least nSize bytes large).
3960 : * @param nSize number of bytes to read in the file.
3961 : * @param nOffset file offset from which to read.
3962 : * @return number of bytes read.
3963 : * @since GDAL 3.6
3964 : */
3965 0 : size_t VSIVirtualHandle::PRead(CPL_UNUSED void *pBuffer,
3966 : CPL_UNUSED size_t nSize,
3967 : CPL_UNUSED vsi_l_offset nOffset) const
3968 : {
3969 0 : return 0;
3970 : }
|
