Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: VSI Virtual File System
4 : * Purpose: Implementation VSI*L File API and other file system access
5 : * methods going through file virtualization.
6 : * Author: Frank Warmerdam, warmerdam@pobox.com
7 : *
8 : ******************************************************************************
9 : * Copyright (c) 2005, Frank Warmerdam <warmerdam@pobox.com>
10 : * Copyright (c) 2008-2014, Even Rouault <even dot rouault at spatialys.com>
11 : *
12 : * Permission is hereby granted, free of charge, to any person obtaining a
13 : * copy of this software and associated documentation files (the "Software"),
14 : * to deal in the Software without restriction, including without limitation
15 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
16 : * and/or sell copies of the Software, and to permit persons to whom the
17 : * Software is furnished to do so, subject to the following conditions:
18 : *
19 : * The above copyright notice and this permission notice shall be included
20 : * in all copies or substantial portions of the Software.
21 : *
22 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
24 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
25 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
26 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
28 : * DEALINGS IN THE SOFTWARE.
29 : ****************************************************************************/
30 :
31 : #include "cpl_port.h"
32 : #include "cpl_vsi.h"
33 :
34 : #include <cassert>
35 : #include <cinttypes>
36 : #include <cstdarg>
37 : #include <cstddef>
38 : #include <cstring>
39 : #if HAVE_FCNTL_H
40 : #include <fcntl.h>
41 : #endif
42 :
43 : #include <algorithm>
44 : #include <limits>
45 : #include <map>
46 : #include <memory>
47 : #include <mutex>
48 : #include <set>
49 : #include <string>
50 : #include <utility>
51 : #include <vector>
52 :
53 : #include "cpl_conv.h"
54 : #include "cpl_error.h"
55 : #include "cpl_multiproc.h"
56 : #include "cpl_string.h"
57 : #include "cpl_vsi_virtual.h"
58 : #include "cpl_vsil_curl_class.h"
59 :
60 : // To avoid aliasing to GetDiskFreeSpace to GetDiskFreeSpaceA on Windows
61 : #ifdef GetDiskFreeSpace
62 : #undef GetDiskFreeSpace
63 : #endif
64 :
65 : /************************************************************************/
66 : /* VSIReadDir() */
67 : /************************************************************************/
68 :
69 : /**
70 : * \brief Read names in a directory.
71 : *
72 : * This function abstracts access to directory contains. It returns a
73 : * list of strings containing the names of files, and directories in this
74 : * directory. The resulting string list becomes the responsibility of the
75 : * application and should be freed with CSLDestroy() when no longer needed.
76 : *
77 : * Note that no error is issued via CPLError() if the directory path is
78 : * invalid, though NULL is returned.
79 : *
80 : * This function used to be known as CPLReadDir(), but the old name is now
81 : * deprecated.
82 : *
83 : * @param pszPath the relative, or absolute path of a directory to read.
84 : * UTF-8 encoded.
85 : * @return The list of entries in the directory, or NULL if the directory
86 : * doesn't exist. Filenames are returned in UTF-8 encoding.
87 : */
88 :
89 16947 : char **VSIReadDir(const char *pszPath)
90 : {
91 16947 : return VSIReadDirEx(pszPath, 0);
92 : }
93 :
94 : /************************************************************************/
95 : /* VSIReadDirEx() */
96 : /************************************************************************/
97 :
98 : /**
99 : * \brief Read names in a directory.
100 : *
101 : * This function abstracts access to directory contains. It returns a
102 : * list of strings containing the names of files, and directories in this
103 : * directory. The resulting string list becomes the responsibility of the
104 : * application and should be freed with CSLDestroy() when no longer needed.
105 : *
106 : * Note that no error is issued via CPLError() if the directory path is
107 : * invalid, though NULL is returned.
108 : *
109 : * If nMaxFiles is set to a positive number, directory listing will stop after
110 : * that limit has been reached. Note that to indicate truncate, at least one
111 : * element more than the nMaxFiles limit will be returned. If CSLCount() on the
112 : * result is lesser or equal to nMaxFiles, then no truncation occurred.
113 : *
114 : * @param pszPath the relative, or absolute path of a directory to read.
115 : * UTF-8 encoded.
116 : * @param nMaxFiles maximum number of files after which to stop, or 0 for no
117 : * limit.
118 : * @return The list of entries in the directory, or NULL if the directory
119 : * doesn't exist. Filenames are returned in UTF-8 encoding.
120 : * @since GDAL 2.1
121 : */
122 :
123 46589 : char **VSIReadDirEx(const char *pszPath, int nMaxFiles)
124 : {
125 46589 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
126 :
127 46589 : return poFSHandler->ReadDirEx(pszPath, nMaxFiles);
128 : }
129 :
130 : /************************************************************************/
131 : /* VSISiblingFiles() */
132 : /************************************************************************/
133 :
134 : /**
135 : * \brief Return related filenames
136 : *
137 : * This function is essentially meant at being used by GDAL internals.
138 : *
139 : * @param pszFilename the path of a filename to inspect
140 : * UTF-8 encoded.
141 : * @return The list of entries, relative to the directory, of all sidecar
142 : * files available or NULL if the list is not known.
143 : * Filenames are returned in UTF-8 encoding.
144 : * Most implementations will return NULL, and a subsequent ReadDir will
145 : * list all files available in the file's directory. This function will be
146 : * overridden by VSI FilesystemHandlers that wish to force e.g. an empty list
147 : * to avoid opening non-existent files on slow filesystems. The return value
148 : * shall be destroyed with CSLDestroy()
149 : * @since GDAL 3.2
150 : */
151 63371 : char **VSISiblingFiles(const char *pszFilename)
152 : {
153 63371 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
154 :
155 63371 : return poFSHandler->SiblingFiles(pszFilename);
156 : }
157 :
158 : /************************************************************************/
159 : /* VSIGetDirectorySeparator() */
160 : /************************************************************************/
161 :
162 : /** Return the directory separator for the specified path.
163 : *
164 : * Default is forward slash. The only exception currently is the Windows
165 : * file system which returns anti-slash, unless the specified path is of the
166 : * form "{drive_letter}:/{rest_of_the_path}".
167 : *
168 : * @since 3.9
169 : */
170 877713 : const char *VSIGetDirectorySeparator(const char *pszPath)
171 : {
172 877713 : if (STARTS_WITH(pszPath, "http://") || STARTS_WITH(pszPath, "https://"))
173 281 : return "/";
174 :
175 877432 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
176 877488 : return poFSHandler->GetDirectorySeparator(pszPath);
177 : }
178 :
179 : /************************************************************************/
180 : /* VSIReadRecursive() */
181 : /************************************************************************/
182 :
183 : /**
184 : * \brief Read names in a directory recursively.
185 : *
186 : * This function abstracts access to directory contents and subdirectories.
187 : * It returns a list of strings containing the names of files and directories
188 : * in this directory and all subdirectories. The resulting string list becomes
189 : * the responsibility of the application and should be freed with CSLDestroy()
190 : * when no longer needed.
191 : *
192 : * Note that no error is issued via CPLError() if the directory path is
193 : * invalid, though NULL is returned.
194 : *
195 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
196 : * longer be always forward slash, but will be the one returned by
197 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
198 : * file systems.
199 : *
200 : * @param pszPathIn the relative, or absolute path of a directory to read.
201 : * UTF-8 encoded.
202 : *
203 : * @return The list of entries in the directory and subdirectories
204 : * or NULL if the directory doesn't exist. Filenames are returned in UTF-8
205 : * encoding.
206 : * @since GDAL 1.10.0
207 : *
208 : */
209 :
210 1270 : char **VSIReadDirRecursive(const char *pszPathIn)
211 : {
212 1270 : const char SEP = VSIGetDirectorySeparator(pszPathIn)[0];
213 :
214 1270 : const char *const apszOptions[] = {"NAME_AND_TYPE_ONLY=YES", nullptr};
215 1270 : VSIDIR *psDir = VSIOpenDir(pszPathIn, -1, apszOptions);
216 1270 : if (!psDir)
217 1 : return nullptr;
218 2538 : CPLStringList oFiles;
219 3794 : while (auto psEntry = VSIGetNextDirEntry(psDir))
220 : {
221 2525 : if (VSI_ISDIR(psEntry->nMode) && psEntry->pszName[0] &&
222 1113 : psEntry->pszName[strlen(psEntry->pszName) - 1] != SEP)
223 : {
224 1113 : oFiles.AddString((std::string(psEntry->pszName) + SEP).c_str());
225 : }
226 : else
227 : {
228 1412 : oFiles.AddString(psEntry->pszName);
229 : }
230 2525 : }
231 1269 : VSICloseDir(psDir);
232 :
233 1269 : return oFiles.StealList();
234 : }
235 :
236 : /************************************************************************/
237 : /* CPLReadDir() */
238 : /* */
239 : /* This is present only to provide ABI compatibility with older */
240 : /* versions. */
241 : /************************************************************************/
242 : #undef CPLReadDir
243 :
244 : CPL_C_START
245 : char CPL_DLL **CPLReadDir(const char *pszPath);
246 : CPL_C_END
247 :
248 0 : char **CPLReadDir(const char *pszPath)
249 : {
250 0 : return VSIReadDir(pszPath);
251 : }
252 :
253 : /************************************************************************/
254 : /* VSIOpenDir() */
255 : /************************************************************************/
256 :
257 : /**
258 : * \brief Open a directory to read its entries.
259 : *
260 : * This function is close to the POSIX opendir() function.
261 : *
262 : * For /vsis3/, /vsigs/, /vsioss/, /vsiaz/ and /vsiadls/, this function has an
263 : * efficient implementation, minimizing the number of network requests, when
264 : * invoked with nRecurseDepth <= 0.
265 : *
266 : * Entries are read by calling VSIGetNextDirEntry() on the handled returned by
267 : * that function, until it returns NULL. VSICloseDir() must be called once done
268 : * with the returned directory handle.
269 : *
270 : * @param pszPath the relative, or absolute path of a directory to read.
271 : * UTF-8 encoded.
272 : * @param nRecurseDepth 0 means do not recurse in subdirectories, 1 means
273 : * recurse only in the first level of subdirectories, etc. -1 means unlimited
274 : * recursion level
275 : * @param papszOptions NULL terminated list of options, or NULL. The following
276 : * options are implemented:
277 : * <ul>
278 : * <li>PREFIX=string: (GDAL >= 3.4) Filter to select filenames only starting
279 : * with the specified prefix. Implemented efficiently for /vsis3/, /vsigs/,
280 : * and /vsiaz/ (but not /vsiadls/)
281 : * </li>
282 : * <li>NAME_AND_TYPE_ONLY=YES/NO: (GDAL >= 3.4) Defaults to NO. If set to YES,
283 : * only the pszName and nMode members of VSIDIR are guaranteed to be set.
284 : * This is implemented efficiently for the Unix virtual file system.
285 : * </li>
286 : * </ul>
287 : *
288 : * @return a handle, or NULL in case of error
289 : * @since GDAL 2.4
290 : *
291 : */
292 :
293 1400 : VSIDIR *VSIOpenDir(const char *pszPath, int nRecurseDepth,
294 : const char *const *papszOptions)
295 : {
296 1400 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
297 :
298 1400 : return poFSHandler->OpenDir(pszPath, nRecurseDepth, papszOptions);
299 : }
300 :
301 : /************************************************************************/
302 : /* VSIGetNextDirEntry() */
303 : /************************************************************************/
304 :
305 : /**
306 : * \brief Return the next entry of the directory
307 : *
308 : * This function is close to the POSIX readdir() function. It actually returns
309 : * more information (file size, last modification time), which on 'real' file
310 : * systems involve one 'stat' call per file.
311 : *
312 : * For filesystems that can have both a regular file and a directory name of
313 : * the same name (typically /vsis3/), when this situation of duplicate happens,
314 : * the directory name will be suffixed by a slash character. Otherwise directory
315 : * names are not suffixed by slash.
316 : *
317 : * The returned entry remains valid until the next call to VSINextDirEntry()
318 : * or VSICloseDir() with the same handle.
319 : *
320 : * Note: since GDAL 3.9, for recursive mode, the directory separator will no
321 : * longer be always forward slash, but will be the one returned by
322 : * VSIGetDirectorySeparator(pszPathIn), so potentially backslash on Windows
323 : * file systems.
324 : *
325 : * @param dir Directory handled returned by VSIOpenDir(). Must not be NULL.
326 : *
327 : * @return a entry, or NULL if there is no more entry in the directory. This
328 : * return value must not be freed.
329 : * @since GDAL 2.4
330 : *
331 : */
332 :
333 4272 : const VSIDIREntry *VSIGetNextDirEntry(VSIDIR *dir)
334 : {
335 4272 : return dir->NextDirEntry();
336 : }
337 :
338 : /************************************************************************/
339 : /* VSICloseDir() */
340 : /************************************************************************/
341 :
342 : /**
343 : * \brief Close a directory
344 : *
345 : * This function is close to the POSIX closedir() function.
346 : *
347 : * @param dir Directory handled returned by VSIOpenDir().
348 : *
349 : * @since GDAL 2.4
350 : */
351 :
352 1375 : void VSICloseDir(VSIDIR *dir)
353 : {
354 1375 : delete dir;
355 1375 : }
356 :
357 : /************************************************************************/
358 : /* VSIMkdir() */
359 : /************************************************************************/
360 :
361 : /**
362 : * \brief Create a directory.
363 : *
364 : * Create a new directory with the indicated mode. For POSIX-style systems,
365 : * the mode is modified by the file creation mask (umask). However, some
366 : * file systems and platforms may not use umask, or they may ignore the mode
367 : * completely. So a reasonable cross-platform default mode value is 0755.
368 : *
369 : * Analog of the POSIX mkdir() function.
370 : *
371 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
372 : * @param mode the permissions mode.
373 : *
374 : * @return 0 on success or -1 on an error.
375 : */
376 :
377 5111 : int VSIMkdir(const char *pszPathname, long mode)
378 :
379 : {
380 5111 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPathname);
381 :
382 5111 : return poFSHandler->Mkdir(pszPathname, mode);
383 : }
384 :
385 : /************************************************************************/
386 : /* VSIMkdirRecursive() */
387 : /************************************************************************/
388 :
389 : /**
390 : * \brief Create a directory and all its ancestors
391 : *
392 : * @param pszPathname the path to the directory to create. UTF-8 encoded.
393 : * @param mode the permissions mode.
394 : *
395 : * @return 0 on success or -1 on an error.
396 : * @since GDAL 2.3
397 : */
398 :
399 83541 : int VSIMkdirRecursive(const char *pszPathname, long mode)
400 : {
401 83541 : if (pszPathname == nullptr || pszPathname[0] == '\0' ||
402 83541 : strncmp("/", pszPathname, 2) == 0)
403 : {
404 0 : return -1;
405 : }
406 :
407 167082 : const CPLString osPathname(pszPathname);
408 : VSIStatBufL sStat;
409 83541 : if (VSIStatL(osPathname, &sStat) == 0)
410 : {
411 82360 : return VSI_ISDIR(sStat.st_mode) ? 0 : -1;
412 : }
413 2362 : const CPLString osParentPath(CPLGetPath(osPathname));
414 :
415 : // Prevent crazy paths from recursing forever.
416 2362 : if (osParentPath == osPathname ||
417 1181 : osParentPath.length() >= osPathname.length())
418 : {
419 0 : return -1;
420 : }
421 :
422 1181 : if (VSIStatL(osParentPath, &sStat) != 0)
423 : {
424 480 : if (VSIMkdirRecursive(osParentPath, mode) != 0)
425 1 : return -1;
426 : }
427 :
428 1180 : return VSIMkdir(osPathname, mode);
429 : }
430 :
431 : /************************************************************************/
432 : /* VSIUnlink() */
433 : /************************************************************************/
434 :
435 : /**
436 : * \brief Delete a file.
437 : *
438 : * Deletes a file object from the file system.
439 : *
440 : * This method goes through the VSIFileHandler virtualization and may
441 : * work on unusual filesystems such as in memory.
442 : *
443 : * Analog of the POSIX unlink() function.
444 : *
445 : * @param pszFilename the path of the file to be deleted. UTF-8 encoded.
446 : *
447 : * @return 0 on success or -1 on an error.
448 : */
449 :
450 111669 : int VSIUnlink(const char *pszFilename)
451 :
452 : {
453 111669 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
454 :
455 111668 : return poFSHandler->Unlink(pszFilename);
456 : }
457 :
458 : /************************************************************************/
459 : /* VSIUnlinkBatch() */
460 : /************************************************************************/
461 :
462 : /**
463 : * \brief Delete several files, possibly in a batch.
464 : *
465 : * All files should belong to the same file system handler.
466 : *
467 : * This is implemented efficiently for /vsis3/ and /vsigs/ (provided for /vsigs/
468 : * that OAuth2 authentication is used).
469 : *
470 : * @param papszFiles NULL terminated list of files. UTF-8 encoded.
471 : *
472 : * @return an array of size CSLCount(papszFiles), whose values are TRUE or FALSE
473 : * depending on the success of deletion of the corresponding file. The array
474 : * should be freed with VSIFree().
475 : * NULL might be return in case of a more general error (for example,
476 : * files belonging to different file system handlers)
477 : *
478 : * @since GDAL 3.1
479 : */
480 :
481 9 : int *VSIUnlinkBatch(CSLConstList papszFiles)
482 : {
483 9 : VSIFilesystemHandler *poFSHandler = nullptr;
484 26 : for (CSLConstList papszIter = papszFiles; papszIter && *papszIter;
485 : ++papszIter)
486 : {
487 18 : auto poFSHandlerThisFile = VSIFileManager::GetHandler(*papszIter);
488 18 : if (poFSHandler == nullptr)
489 8 : poFSHandler = poFSHandlerThisFile;
490 10 : else if (poFSHandler != poFSHandlerThisFile)
491 : {
492 1 : CPLError(CE_Failure, CPLE_AppDefined,
493 : "Files belong to different file system handlers");
494 1 : poFSHandler = nullptr;
495 1 : break;
496 : }
497 : }
498 9 : if (poFSHandler == nullptr)
499 2 : return nullptr;
500 7 : return poFSHandler->UnlinkBatch(papszFiles);
501 : }
502 :
503 : /************************************************************************/
504 : /* VSIRename() */
505 : /************************************************************************/
506 :
507 : /**
508 : * \brief Rename a file.
509 : *
510 : * Renames a file object in the file system. It should be possible
511 : * to rename a file onto a new filesystem, but it is safest if this
512 : * function is only used to rename files that remain in the same directory.
513 : *
514 : * This method goes through the VSIFileHandler virtualization and may
515 : * work on unusual filesystems such as in memory.
516 : *
517 : * Analog of the POSIX rename() function.
518 : *
519 : * @param oldpath the name of the file to be renamed. UTF-8 encoded.
520 : * @param newpath the name the file should be given. UTF-8 encoded.
521 : *
522 : * @return 0 on success or -1 on an error.
523 : */
524 :
525 624 : int VSIRename(const char *oldpath, const char *newpath)
526 :
527 : {
528 624 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(oldpath);
529 :
530 624 : return poFSHandler->Rename(oldpath, newpath);
531 : }
532 :
533 : /************************************************************************/
534 : /* VSICopyFile() */
535 : /************************************************************************/
536 :
537 : /**
538 : * \brief Copy a source file into a target file.
539 : *
540 : * For a /vsizip/foo.zip/bar target, the options available are those of
541 : * CPLAddFileInZip()
542 : *
543 : * The following copies are made fully on the target server, without local
544 : * download from source and upload to target:
545 : * - /vsis3/ -> /vsis3/
546 : * - /vsigs/ -> /vsigs/
547 : * - /vsiaz/ -> /vsiaz/
548 : * - /vsiadls/ -> /vsiadls/
549 : * - any of the above or /vsicurl/ -> /vsiaz/ (starting with GDAL 3.8)
550 : *
551 : * @param pszSource Source filename. UTF-8 encoded. May be NULL if fpSource is
552 : * not NULL.
553 : * @param pszTarget Target filename. UTF-8 encoded. Must not be NULL
554 : * @param fpSource File handle on the source file. May be NULL if pszSource is
555 : * not NULL.
556 : * @param nSourceSize Size of the source file. Pass -1 if unknown.
557 : * If set to -1, and progress callback is used, VSIStatL() will be used on
558 : * pszSource to retrieve the source size.
559 : * @param papszOptions Null terminated list of options, or NULL.
560 : * @param pProgressFunc Progress callback, or NULL.
561 : * @param pProgressData User data of progress callback, or NULL.
562 : *
563 : * @return 0 on success.
564 : * @since GDAL 3.7
565 : */
566 :
567 2059 : int VSICopyFile(const char *pszSource, const char *pszTarget,
568 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
569 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
570 : void *pProgressData)
571 :
572 : {
573 2059 : if (!pszSource && !fpSource)
574 : {
575 0 : CPLError(CE_Failure, CPLE_AppDefined,
576 : "pszSource == nullptr && fpSource == nullptr");
577 0 : return -1;
578 : }
579 2059 : if (!pszTarget || pszTarget[0] == '\0')
580 : {
581 0 : return -1;
582 : }
583 :
584 : VSIFilesystemHandler *poFSHandlerTarget =
585 2059 : VSIFileManager::GetHandler(pszTarget);
586 2059 : return poFSHandlerTarget->CopyFile(pszSource, pszTarget, fpSource,
587 : nSourceSize, papszOptions, pProgressFunc,
588 2059 : pProgressData);
589 : }
590 :
591 : /************************************************************************/
592 : /* VSISync() */
593 : /************************************************************************/
594 :
595 : /**
596 : * \brief Synchronize a source file/directory with a target file/directory.
597 : *
598 : * This is a analog of the 'rsync' utility. In the current implementation,
599 : * rsync would be more efficient for local file copying, but VSISync() main
600 : * interest is when the source or target is a remote
601 : * file system like /vsis3/ or /vsigs/, in which case it can take into account
602 : * the timestamps of the files (or optionally the ETag/MD5Sum) to avoid
603 : * unneeded copy operations.
604 : *
605 : * This is only implemented efficiently for:
606 : * <ul>
607 : * <li> local filesystem <--> remote filesystem.</li>
608 : * <li> remote filesystem <--> remote filesystem (starting with GDAL 3.1).
609 : * Where the source and target remote filesystems are the same and one of
610 : * /vsis3/, /vsigs/ or /vsiaz/. Or when the target is /vsiaz/ and the source
611 : * is /vsis3/, /vsigs/, /vsiadls/ or /vsicurl/ (starting with GDAL 3.8)</li>
612 : * </ul>
613 : *
614 : * Similarly to rsync behavior, if the source filename ends with a slash,
615 : * it means that the content of the directory must be copied, but not the
616 : * directory name. For example, assuming "/home/even/foo" contains a file "bar",
617 : * VSISync("/home/even/foo/", "/mnt/media", ...) will create a "/mnt/media/bar"
618 : * file. Whereas VSISync("/home/even/foo", "/mnt/media", ...) will create a
619 : * "/mnt/media/foo" directory which contains a bar file.
620 : *
621 : * @param pszSource Source file or directory. UTF-8 encoded.
622 : * @param pszTarget Target file or directory. UTF-8 encoded.
623 : * @param papszOptions Null terminated list of options, or NULL.
624 : * Currently accepted options are:
625 : * <ul>
626 : * <li>RECURSIVE=NO (the default is YES)</li>
627 : * <li>SYNC_STRATEGY=TIMESTAMP/ETAG/OVERWRITE.
628 : *
629 : * Determines which criterion is used to determine if a target file must be
630 : * replaced when it already exists and has the same file size as the source.
631 : * Only applies for a source or target being a network filesystem.
632 : *
633 : * The default is TIMESTAMP (similarly to how 'aws s3 sync' works), that is
634 : * to say that for an upload operation, a remote file is
635 : * replaced if it has a different size or if it is older than the source.
636 : * For a download operation, a local file is replaced if it has a different
637 : * size or if it is newer than the remote file.
638 : *
639 : * The ETAG strategy assumes that the ETag metadata of the remote file is
640 : * the MD5Sum of the file content, which is only true in the case of /vsis3/
641 : * for files not using KMS server side encryption and uploaded in a single
642 : * PUT operation (so smaller than 50 MB given the default used by GDAL).
643 : * Only to be used for /vsis3/, /vsigs/ or other filesystems using a
644 : * MD5Sum as ETAG.
645 : *
646 : * The OVERWRITE strategy (GDAL >= 3.2) will always overwrite the target
647 : * file with the source one.
648 : * </li>
649 : * <li>NUM_THREADS=integer. (GDAL >= 3.1) Number of threads to use for parallel
650 : * file copying. Only use for when /vsis3/, /vsigs/, /vsiaz/ or /vsiadls/ is in
651 : * source or target. The default is 10 since GDAL 3.3</li>
652 : * <li>CHUNK_SIZE=integer. (GDAL >= 3.1) Maximum size of chunk (in bytes) to use
653 : * to split large objects when downloading them from /vsis3/, /vsigs/, /vsiaz/
654 : * or /vsiadls/ to local file system, or for upload to /vsis3/, /vsiaz/ or
655 : * /vsiadls/ from local file system. Only used if NUM_THREADS > 1. For upload to
656 : * /vsis3/, this chunk size must be set at least to 5 MB. The default is 8 MB
657 : * since GDAL 3.3</li> <li>x-amz-KEY=value. (GDAL >= 3.5) MIME header to pass
658 : * during creation of a /vsis3/ object.</li> <li>x-goog-KEY=value. (GDAL >= 3.5)
659 : * MIME header to pass during creation of a /vsigs/ object.</li>
660 : * <li>x-ms-KEY=value. (GDAL >= 3.5) MIME header to pass during creation of a
661 : * /vsiaz/ or /vsiadls/ object.</li>
662 : * </ul>
663 : * @param pProgressFunc Progress callback, or NULL.
664 : * @param pProgressData User data of progress callback, or NULL.
665 : * @param ppapszOutputs Unused. Should be set to NULL for now.
666 : *
667 : * @return TRUE on success or FALSE on an error.
668 : * @since GDAL 2.4
669 : */
670 :
671 44 : int VSISync(const char *pszSource, const char *pszTarget,
672 : const char *const *papszOptions, GDALProgressFunc pProgressFunc,
673 : void *pProgressData, char ***ppapszOutputs)
674 :
675 : {
676 44 : if (pszSource[0] == '\0' || pszTarget[0] == '\0')
677 : {
678 0 : return FALSE;
679 : }
680 :
681 : VSIFilesystemHandler *poFSHandlerSource =
682 44 : VSIFileManager::GetHandler(pszSource);
683 : VSIFilesystemHandler *poFSHandlerTarget =
684 44 : VSIFileManager::GetHandler(pszTarget);
685 44 : VSIFilesystemHandler *poFSHandlerLocal = VSIFileManager::GetHandler("");
686 : VSIFilesystemHandler *poFSHandlerMem =
687 44 : VSIFileManager::GetHandler("/vsimem/");
688 44 : VSIFilesystemHandler *poFSHandler = poFSHandlerSource;
689 44 : if (poFSHandlerTarget != poFSHandlerLocal &&
690 : poFSHandlerTarget != poFSHandlerMem)
691 : {
692 22 : poFSHandler = poFSHandlerTarget;
693 : }
694 :
695 88 : return poFSHandler->Sync(pszSource, pszTarget, papszOptions, pProgressFunc,
696 44 : pProgressData, ppapszOutputs)
697 44 : ? TRUE
698 44 : : FALSE;
699 : }
700 :
701 : /************************************************************************/
702 : /* VSIAbortOngoingUploads() */
703 : /************************************************************************/
704 :
705 : /**
706 : * \brief Abort ongoing multi-part uploads.
707 : *
708 : * Abort ongoing multi-part uploads on AWS S3 and Google Cloud Storage. This
709 : * can be used in case a process doing such uploads was killed in a unclean way.
710 : *
711 : * Without effect on other virtual file systems.
712 : *
713 : * @param pszFilename filename or prefix of a directory into which multipart
714 : * uploads must be aborted. This can be the root directory of a bucket. UTF-8
715 : * encoded.
716 : *
717 : * @return TRUE on success or FALSE on an error.
718 : * @since GDAL 3.4
719 : */
720 :
721 1 : int VSIAbortPendingUploads(const char *pszFilename)
722 : {
723 1 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
724 :
725 1 : return poFSHandler->AbortPendingUploads(pszFilename);
726 : }
727 :
728 : /************************************************************************/
729 : /* VSIRmdir() */
730 : /************************************************************************/
731 :
732 : /**
733 : * \brief Delete a directory.
734 : *
735 : * Deletes a directory object from the file system. On some systems
736 : * the directory must be empty before it can be deleted.
737 : *
738 : * This method goes through the VSIFileHandler virtualization and may
739 : * work on unusual filesystems such as in memory.
740 : *
741 : * Analog of the POSIX rmdir() function.
742 : *
743 : * @param pszDirname the path of the directory to be deleted. UTF-8 encoded.
744 : *
745 : * @return 0 on success or -1 on an error.
746 : */
747 :
748 3930 : int VSIRmdir(const char *pszDirname)
749 :
750 : {
751 3930 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
752 :
753 3930 : return poFSHandler->Rmdir(pszDirname);
754 : }
755 :
756 : /************************************************************************/
757 : /* VSIRmdirRecursive() */
758 : /************************************************************************/
759 :
760 : /**
761 : * \brief Delete a directory recursively
762 : *
763 : * Deletes a directory object and its content from the file system.
764 : *
765 : * Starting with GDAL 3.1, /vsis3/ has an efficient implementation of this
766 : * function.
767 : * Starting with GDAL 3.4, /vsigs/ has an efficient implementation of this
768 : * function, provided that OAuth2 authentication is used.
769 : *
770 : * @return 0 on success or -1 on an error.
771 : * @since GDAL 2.3
772 : */
773 :
774 2241 : int VSIRmdirRecursive(const char *pszDirname)
775 : {
776 2241 : if (pszDirname == nullptr || pszDirname[0] == '\0' ||
777 2241 : strncmp("/", pszDirname, 2) == 0)
778 : {
779 0 : return -1;
780 : }
781 2241 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
782 2241 : return poFSHandler->RmdirRecursive(pszDirname);
783 : }
784 :
785 : /************************************************************************/
786 : /* VSIStatL() */
787 : /************************************************************************/
788 :
789 : /**
790 : * \brief Get filesystem object info.
791 : *
792 : * Fetches status information about a filesystem object (file, directory, etc).
793 : * The returned information is placed in the VSIStatBufL structure. For
794 : * portability, only use the st_size (size in bytes) and st_mode (file type).
795 : * This method is similar to VSIStat(), but will work on large files on
796 : * systems where this requires special calls.
797 : *
798 : * This method goes through the VSIFileHandler virtualization and may
799 : * work on unusual filesystems such as in memory.
800 : *
801 : * Analog of the POSIX stat() function.
802 : *
803 : * @param pszFilename the path of the filesystem object to be queried.
804 : * UTF-8 encoded.
805 : * @param psStatBuf the structure to load with information.
806 : *
807 : * @return 0 on success or -1 on an error.
808 : */
809 :
810 275918 : int VSIStatL(const char *pszFilename, VSIStatBufL *psStatBuf)
811 :
812 : {
813 275918 : return VSIStatExL(pszFilename, psStatBuf, 0);
814 : }
815 :
816 : /************************************************************************/
817 : /* VSIStatExL() */
818 : /************************************************************************/
819 :
820 : /**
821 : * \brief Get filesystem object info.
822 : *
823 : * Fetches status information about a filesystem object (file, directory, etc).
824 : * The returned information is placed in the VSIStatBufL structure. For
825 : * portability, only use the st_size (size in bytes) and st_mode (file type).
826 : * This method is similar to VSIStat(), but will work on large files on
827 : * systems where this requires special calls.
828 : *
829 : * This method goes through the VSIFileHandler virtualization and may
830 : * work on unusual filesystems such as in memory.
831 : *
832 : * Analog of the POSIX stat() function, with an extra parameter to
833 : * specify which information is needed, which offers a potential for
834 : * speed optimizations on specialized and potentially slow virtual
835 : * filesystem objects (/vsigzip/, /vsicurl/)
836 : *
837 : * @param pszFilename the path of the filesystem object to be queried.
838 : * UTF-8 encoded.
839 : * @param psStatBuf the structure to load with information.
840 : * @param nFlags 0 to get all information, or VSI_STAT_EXISTS_FLAG,
841 : * VSI_STAT_NATURE_FLAG, VSI_STAT_SIZE_FLAG,
842 : * VSI_STAT_SET_ERROR_FLAG, VSI_STAT_CACHE_ONLY or a combination of those to get
843 : * partial info.
844 : *
845 : * @return 0 on success or -1 on an error.
846 : *
847 : * @since GDAL 1.8.0
848 : */
849 :
850 605945 : int VSIStatExL(const char *pszFilename, VSIStatBufL *psStatBuf, int nFlags)
851 :
852 : {
853 605945 : char szAltPath[4] = {'\0'};
854 :
855 : // Enable to work on "C:" as if it were "C:\".
856 605945 : if (pszFilename[0] != '\0' && pszFilename[1] == ':' &&
857 20 : pszFilename[2] == '\0')
858 : {
859 0 : szAltPath[0] = pszFilename[0];
860 0 : szAltPath[1] = pszFilename[1];
861 0 : szAltPath[2] = '\\';
862 0 : szAltPath[3] = '\0';
863 :
864 0 : pszFilename = szAltPath;
865 : }
866 :
867 605945 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
868 :
869 605945 : if (nFlags == 0)
870 279524 : nFlags =
871 : VSI_STAT_EXISTS_FLAG | VSI_STAT_NATURE_FLAG | VSI_STAT_SIZE_FLAG;
872 :
873 1211890 : return poFSHandler->Stat(pszFilename, psStatBuf, nFlags);
874 : }
875 :
876 : /************************************************************************/
877 : /* VSIGetFileMetadata() */
878 : /************************************************************************/
879 :
880 : /**
881 : * \brief Get metadata on files.
882 : *
883 : * Implemented currently only for network-like filesystems, or starting
884 : * with GDAL 3.7 for /vsizip/
885 : *
886 : * @param pszFilename the path of the filesystem object to be queried.
887 : * UTF-8 encoded.
888 : * @param pszDomain Metadata domain to query. Depends on the file system.
889 : * The following are supported:
890 : * <ul>
891 : * <li>HEADERS: to get HTTP headers for network-like filesystems (/vsicurl/,
892 : * /vsis3/, /vsgis/, etc)</li> <li>TAGS: <ul> <li>/vsis3/: to get S3 Object
893 : * tagging information</li> <li>/vsiaz/: to get blob tags. Refer to
894 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-tags</li>
895 : * </ul>
896 : * </li>
897 : * <li>STATUS: specific to /vsiadls/: returns all system defined properties for
898 : * a path (seems in practice to be a subset of HEADERS)</li> <li>ACL: specific
899 : * to /vsiadls/ and /vsigs/: returns the access control list for a path. For
900 : * /vsigs/, a single XML=xml_content string is returned. Refer to
901 : * https://cloud.google.com/storage/docs/xml-api/get-object-acls
902 : * </li>
903 : * <li>METADATA: specific to /vsiaz/: to set blob metadata. Refer to
904 : * https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-metadata.
905 : * Note: this will be a subset of what pszDomain=HEADERS returns</li>
906 : * <li>ZIP: specific to /vsizip/: to obtain ZIP specific metadata, in particular
907 : * if a file is SOZIP-enabled (SOZIP_VALID=YES)</li>
908 : * </ul>
909 : * @param papszOptions Unused. Should be set to NULL.
910 : *
911 : * @return a NULL-terminated list of key=value strings, to be freed with
912 : * CSLDestroy() or NULL in case of error / empty list.
913 : *
914 : * @since GDAL 3.1.0
915 : */
916 :
917 74 : char **VSIGetFileMetadata(const char *pszFilename, const char *pszDomain,
918 : CSLConstList papszOptions)
919 : {
920 74 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
921 74 : return poFSHandler->GetFileMetadata(pszFilename, pszDomain, papszOptions);
922 : }
923 :
924 : /************************************************************************/
925 : /* VSISetFileMetadata() */
926 : /************************************************************************/
927 :
928 : /**
929 : * \brief Set metadata on files.
930 : *
931 : * Implemented currently only for /vsis3/, /vsigs/, /vsiaz/ and /vsiadls/
932 : *
933 : * @param pszFilename the path of the filesystem object to be set.
934 : * UTF-8 encoded.
935 : * @param papszMetadata NULL-terminated list of key=value strings.
936 : * @param pszDomain Metadata domain to set. Depends on the file system.
937 : * The following are supported:
938 : * <ul>
939 : * <li>HEADERS: specific to /vsis3/ and /vsigs/: to set HTTP headers, such as
940 : * "Content-Type", or other file system specific header.
941 : * For /vsigs/, this also includes: x-goog-meta-{key}={value}. Note that you
942 : * should specify all metadata to be set, as existing metadata will be
943 : * overridden.
944 : * </li>
945 : * <li>TAGS: Content of papszMetadata should be KEY=VALUE pairs.
946 : * <ul>
947 : * <li>/vsis3/: to set S3 Object tagging information</li>
948 : * <li>/vsiaz/: to set blob tags. Refer to
949 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-tags.
950 : * Note: storageV2 must be enabled on the account</li>
951 : * </ul>
952 : * </li>
953 : * <li>PROPERTIES:
954 : * <ul>
955 : * <li>to /vsiaz/: to set properties. Refer to
956 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-properties.</li>
957 : * <li>to /vsiadls/: to set properties. Refer to
958 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
959 : * for headers valid for action=setProperties.</li>
960 : * </ul>
961 : * </li>
962 : * <li>ACL: specific to /vsiadls/ and /vsigs/: to set access control list.
963 : * For /vsiadls/, refer to
964 : * https://docs.microsoft.com/en-us/rest/api/storageservices/datalakestoragegen2/path/update
965 : * for headers valid for action=setAccessControl or setAccessControlRecursive.
966 : * In setAccessControlRecursive, x-ms-acl must be specified in papszMetadata.
967 : * For /vsigs/, refer to
968 : * https://cloud.google.com/storage/docs/xml-api/put-object-acls. A single
969 : * XML=xml_content string should be specified as in papszMetadata.
970 : * </li>
971 : * <li>METADATA: specific to /vsiaz/: to set blob metadata. Refer to
972 : * https://docs.microsoft.com/en-us/rest/api/storageservices/set-blob-metadata.
973 : * Content of papszMetadata should be strings in the form
974 : * x-ms-meta-name=value</li>
975 : * </ul>
976 : * @param papszOptions NULL or NULL terminated list of options.
977 : * For /vsiadls/ and pszDomain=ACL, "RECURSIVE=TRUE" can be
978 : * set to set the access control list recursively. When
979 : * RECURSIVE=TRUE is set, MODE should also be set to one of
980 : * "set", "modify" or "remove".
981 : *
982 : * @return TRUE in case of success.
983 : *
984 : * @since GDAL 3.1.0
985 : */
986 :
987 17 : int VSISetFileMetadata(const char *pszFilename, CSLConstList papszMetadata,
988 : const char *pszDomain, CSLConstList papszOptions)
989 : {
990 17 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
991 34 : return poFSHandler->SetFileMetadata(pszFilename, papszMetadata, pszDomain,
992 17 : papszOptions)
993 17 : ? 1
994 17 : : 0;
995 : }
996 :
997 : /************************************************************************/
998 : /* VSIIsCaseSensitiveFS() */
999 : /************************************************************************/
1000 :
1001 : /**
1002 : * \brief Returns if the filenames of the filesystem are case sensitive.
1003 : *
1004 : * This method retrieves to which filesystem belongs the passed filename
1005 : * and return TRUE if the filenames of that filesystem are case sensitive.
1006 : *
1007 : * Currently, this will return FALSE only for Windows real filenames. Other
1008 : * VSI virtual filesystems are case sensitive.
1009 : *
1010 : * This methods avoid ugly \#ifndef _WIN32 / \#endif code, that is wrong when
1011 : * dealing with virtual filenames.
1012 : *
1013 : * @param pszFilename the path of the filesystem object to be tested.
1014 : * UTF-8 encoded.
1015 : *
1016 : * @return TRUE if the filenames of the filesystem are case sensitive.
1017 : *
1018 : * @since GDAL 1.8.0
1019 : */
1020 :
1021 26983 : int VSIIsCaseSensitiveFS(const char *pszFilename)
1022 : {
1023 26983 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1024 :
1025 26983 : return poFSHandler->IsCaseSensitive(pszFilename);
1026 : }
1027 :
1028 : /************************************************************************/
1029 : /* VSISupportsSparseFiles() */
1030 : /************************************************************************/
1031 :
1032 : /**
1033 : * \brief Returns if the filesystem supports sparse files.
1034 : *
1035 : * Only supported on Linux (and no other Unix derivatives) and
1036 : * Windows. On Linux, the answer depends on a few hardcoded
1037 : * signatures for common filesystems. Other filesystems will be
1038 : * considered as not supporting sparse files.
1039 : *
1040 : * @param pszPath the path of the filesystem object to be tested.
1041 : * UTF-8 encoded.
1042 : *
1043 : * @return TRUE if the file system is known to support sparse files. FALSE may
1044 : * be returned both in cases where it is known to not support them,
1045 : * or when it is unknown.
1046 : *
1047 : * @since GDAL 2.2
1048 : */
1049 :
1050 2 : int VSISupportsSparseFiles(const char *pszPath)
1051 : {
1052 2 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1053 :
1054 2 : return poFSHandler->SupportsSparseFiles(pszPath);
1055 : }
1056 :
1057 : /************************************************************************/
1058 : /* VSIIsLocal() */
1059 : /************************************************************************/
1060 :
1061 : /**
1062 : * \brief Returns if the file/filesystem is "local".
1063 : *
1064 : * The concept of local is mostly by opposition with a network / remote
1065 : * file system whose access time can be long.
1066 : *
1067 : * /vsimem/ is considered to be a local file system, although a non-persistent
1068 : * one.
1069 : *
1070 : * @param pszPath the path of the filesystem object to be tested.
1071 : * UTF-8 encoded.
1072 : *
1073 : * @return TRUE or FALSE
1074 : *
1075 : * @since GDAL 3.6
1076 : */
1077 :
1078 204 : bool VSIIsLocal(const char *pszPath)
1079 : {
1080 204 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1081 :
1082 204 : return poFSHandler->IsLocal(pszPath);
1083 : }
1084 :
1085 : /************************************************************************/
1086 : /* VSIGetCanonicalFilename() */
1087 : /************************************************************************/
1088 :
1089 : /**
1090 : * \brief Returns the canonical filename.
1091 : *
1092 : * May be implemented by case-insensitive filesystems
1093 : * (currently Win32 and MacOSX) to return the filename with its actual case
1094 : * (i.e. the one that would be used when listing the content of the directory).
1095 : *
1096 : * @param pszPath UTF-8 encoded path
1097 : *
1098 : * @return UTF-8 encoded string, to free with VSIFree()
1099 : *
1100 : * @since GDAL 3.8
1101 : */
1102 :
1103 248 : char *VSIGetCanonicalFilename(const char *pszPath)
1104 : {
1105 248 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1106 :
1107 248 : return CPLStrdup(poFSHandler->GetCanonicalFilename(pszPath).c_str());
1108 : }
1109 :
1110 : /************************************************************************/
1111 : /* VSISupportsSequentialWrite() */
1112 : /************************************************************************/
1113 :
1114 : /**
1115 : * \brief Returns if the filesystem supports sequential write.
1116 : *
1117 : * @param pszPath the path of the filesystem object to be tested.
1118 : * UTF-8 encoded.
1119 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1120 : * local temporary file before uploading to the target location.
1121 : *
1122 : * @return TRUE or FALSE
1123 : *
1124 : * @since GDAL 3.6
1125 : */
1126 :
1127 91 : bool VSISupportsSequentialWrite(const char *pszPath, bool bAllowLocalTempFile)
1128 : {
1129 91 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1130 :
1131 91 : return poFSHandler->SupportsSequentialWrite(pszPath, bAllowLocalTempFile);
1132 : }
1133 :
1134 : /************************************************************************/
1135 : /* VSISupportsRandomWrite() */
1136 : /************************************************************************/
1137 :
1138 : /**
1139 : * \brief Returns if the filesystem supports random write.
1140 : *
1141 : * @param pszPath the path of the filesystem object to be tested.
1142 : * UTF-8 encoded.
1143 : * @param bAllowLocalTempFile whether the file system is allowed to use a
1144 : * local temporary file before uploading to the target location.
1145 : *
1146 : * @return TRUE or FALSE
1147 : *
1148 : * @since GDAL 3.6
1149 : */
1150 :
1151 71 : bool VSISupportsRandomWrite(const char *pszPath, bool bAllowLocalTempFile)
1152 : {
1153 71 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1154 :
1155 71 : return poFSHandler->SupportsRandomWrite(pszPath, bAllowLocalTempFile);
1156 : }
1157 :
1158 : /************************************************************************/
1159 : /* VSIHasOptimizedReadMultiRange() */
1160 : /************************************************************************/
1161 :
1162 : /**
1163 : * \brief Returns if the filesystem supports efficient multi-range reading.
1164 : *
1165 : * Currently only returns TRUE for /vsicurl/ and derived file systems.
1166 : *
1167 : * @param pszPath the path of the filesystem object to be tested.
1168 : * UTF-8 encoded.
1169 : *
1170 : * @return TRUE if the file system is known to have an efficient multi-range
1171 : * reading.
1172 : *
1173 : * @since GDAL 2.3
1174 : */
1175 :
1176 9796 : int VSIHasOptimizedReadMultiRange(const char *pszPath)
1177 : {
1178 9796 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszPath);
1179 :
1180 9796 : return poFSHandler->HasOptimizedReadMultiRange(pszPath);
1181 : }
1182 :
1183 : /************************************************************************/
1184 : /* VSIGetActualURL() */
1185 : /************************************************************************/
1186 :
1187 : /**
1188 : * \brief Returns the actual URL of a supplied filename.
1189 : *
1190 : * Currently only returns a non-NULL value for network-based virtual file
1191 : * systems. For example "/vsis3/bucket/filename" will be expanded as
1192 : * "https://bucket.s3.amazon.com/filename"
1193 : *
1194 : * Note that the lifetime of the returned string, is short, and may be
1195 : * invalidated by any following GDAL functions.
1196 : *
1197 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1198 : *
1199 : * @return the actual URL corresponding to the supplied filename, or NULL.
1200 : * Should not be freed.
1201 : *
1202 : * @since GDAL 2.3
1203 : */
1204 :
1205 9 : const char *VSIGetActualURL(const char *pszFilename)
1206 : {
1207 9 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1208 :
1209 9 : return poFSHandler->GetActualURL(pszFilename);
1210 : }
1211 :
1212 : /************************************************************************/
1213 : /* VSIGetSignedURL() */
1214 : /************************************************************************/
1215 :
1216 : /**
1217 : * \brief Returns a signed URL of a supplied filename.
1218 : *
1219 : * Currently only returns a non-NULL value for /vsis3/, /vsigs/, /vsiaz/ and
1220 : * /vsioss/ For example "/vsis3/bucket/filename" will be expanded as
1221 : * "https://bucket.s3.amazon.com/filename?X-Amz-Algorithm=AWS4-HMAC-SHA256..."
1222 : * Configuration options that apply for file opening (typically to provide
1223 : * credentials), and are returned by VSIGetFileSystemOptions(), are also valid
1224 : * in that context.
1225 : *
1226 : * @param pszFilename the path of the filesystem object. UTF-8 encoded.
1227 : * @param papszOptions list of options, or NULL. Depend on file system handler.
1228 : * For /vsis3/, /vsigs/, /vsiaz/ and /vsioss/, the following options are
1229 : * supported: <ul> <li>START_DATE=YYMMDDTHHMMSSZ: date and time in UTC following
1230 : * ISO 8601 standard, corresponding to the start of validity of the URL. If not
1231 : * specified, current date time.</li> <li>EXPIRATION_DELAY=number_of_seconds:
1232 : * number between 1 and 604800 (seven days) for the validity of the signed URL.
1233 : * Defaults to 3600 (one hour)</li> <li>VERB=GET/HEAD/DELETE/PUT/POST: HTTP VERB
1234 : * for which the request will be used. Default to GET.</li>
1235 : * </ul>
1236 : *
1237 : * /vsiaz/ supports additional options:
1238 : * <ul>
1239 : * <li>SIGNEDIDENTIFIER=value: to relate the given shared access signature
1240 : * to a corresponding stored access policy.</li>
1241 : * <li>SIGNEDPERMISSIONS=r|w: permissions associated with the shared access
1242 : * signature. Normally deduced from VERB.</li>
1243 : * </ul>
1244 : *
1245 : * @return a signed URL, or NULL. Should be freed with CPLFree().
1246 : * @since GDAL 2.3
1247 : */
1248 :
1249 25 : char *VSIGetSignedURL(const char *pszFilename, CSLConstList papszOptions)
1250 : {
1251 25 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
1252 :
1253 25 : return poFSHandler->GetSignedURL(pszFilename, papszOptions);
1254 : }
1255 :
1256 : /************************************************************************/
1257 : /* VSIFOpenL() */
1258 : /************************************************************************/
1259 :
1260 : /**
1261 : * \brief Open file.
1262 : *
1263 : * This function opens a file with the desired access. Large files (larger
1264 : * than 2GB) should be supported. Binary access is always implied and
1265 : * the "b" does not need to be included in the pszAccess string.
1266 : *
1267 : * Note that the "VSILFILE *" returned since GDAL 1.8.0 by this function is
1268 : * *NOT* a standard C library FILE *, and cannot be used with any functions
1269 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
1270 : *
1271 : * On windows it is possible to define the configuration option
1272 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
1273 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
1274 : *
1275 : * This method goes through the VSIFileHandler virtualization and may
1276 : * work on unusual filesystems such as in memory.
1277 : *
1278 : * Analog of the POSIX fopen() function.
1279 : *
1280 : * @param pszFilename the file to open. UTF-8 encoded.
1281 : * @param pszAccess access requested (i.e. "r", "r+", "w")
1282 : *
1283 : * @return NULL on failure, or the file handle.
1284 : */
1285 :
1286 228142 : VSILFILE *VSIFOpenL(const char *pszFilename, const char *pszAccess)
1287 :
1288 : {
1289 228142 : return VSIFOpenExL(pszFilename, pszAccess, false);
1290 : }
1291 :
1292 : /************************************************************************/
1293 : /* Open() */
1294 : /************************************************************************/
1295 :
1296 : #ifndef DOXYGEN_SKIP
1297 :
1298 5423 : VSIVirtualHandle *VSIFilesystemHandler::Open(const char *pszFilename,
1299 : const char *pszAccess)
1300 : {
1301 5423 : return Open(pszFilename, pszAccess, false, nullptr);
1302 : }
1303 :
1304 : /************************************************************************/
1305 : /* CopyFile() */
1306 : /************************************************************************/
1307 :
1308 2036 : int VSIFilesystemHandler::CopyFile(const char *pszSource, const char *pszTarget,
1309 : VSILFILE *fpSource, vsi_l_offset nSourceSize,
1310 : CSLConstList papszOptions,
1311 : GDALProgressFunc pProgressFunc,
1312 : void *pProgressData)
1313 : {
1314 2036 : VSIVirtualHandleUniquePtr poFileHandleAutoClose;
1315 2036 : if (!fpSource)
1316 : {
1317 2022 : fpSource = VSIFOpenExL(pszSource, "rb", TRUE);
1318 2022 : if (!fpSource)
1319 : {
1320 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSource);
1321 0 : return -1;
1322 : }
1323 2022 : poFileHandleAutoClose.reset(fpSource);
1324 : }
1325 2036 : if (nSourceSize == static_cast<vsi_l_offset>(-1) &&
1326 2 : pProgressFunc != nullptr && pszSource != nullptr)
1327 : {
1328 : VSIStatBufL sStat;
1329 2 : if (VSIStatL(pszSource, &sStat) == 0)
1330 : {
1331 2 : nSourceSize = sStat.st_size;
1332 : }
1333 : }
1334 :
1335 2036 : VSILFILE *fpOut = VSIFOpenEx2L(pszTarget, "wb", TRUE, papszOptions);
1336 2036 : if (!fpOut)
1337 : {
1338 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", pszTarget);
1339 1 : return -1;
1340 : }
1341 :
1342 4070 : CPLString osMsg;
1343 2035 : if (pszSource)
1344 2034 : osMsg.Printf("Copying of %s", pszSource);
1345 :
1346 2035 : int ret = 0;
1347 2035 : constexpr size_t nBufferSize = 10 * 4096;
1348 2035 : std::vector<GByte> abyBuffer(nBufferSize, 0);
1349 2035 : GUIntBig nOffset = 0;
1350 : while (true)
1351 : {
1352 2299 : size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpSource);
1353 2299 : size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
1354 2299 : if (nWritten != nRead)
1355 : {
1356 20 : CPLError(CE_Failure, CPLE_FileIO, "Copying of %s to %s failed",
1357 : pszSource, pszTarget);
1358 20 : ret = -1;
1359 20 : break;
1360 : }
1361 2279 : nOffset += nRead;
1362 2323 : if (pProgressFunc &&
1363 132 : !pProgressFunc(nSourceSize == 0 ? 1.0
1364 44 : : nSourceSize > 0 &&
1365 : nSourceSize != static_cast<vsi_l_offset>(-1)
1366 88 : ? double(nOffset) / nSourceSize
1367 : : 0.0,
1368 44 : pszSource ? osMsg.c_str() : nullptr, pProgressData))
1369 : {
1370 1 : ret = -1;
1371 1 : break;
1372 : }
1373 2278 : if (nRead < nBufferSize)
1374 : {
1375 2014 : break;
1376 : }
1377 264 : }
1378 :
1379 2035 : if (nSourceSize != static_cast<vsi_l_offset>(-1) && nOffset != nSourceSize)
1380 : {
1381 1 : CPLError(CE_Failure, CPLE_FileIO,
1382 : "Copying of %s to %s failed: %" PRIu64 " bytes were copied "
1383 : "whereas %" PRIu64 " were expected",
1384 : pszSource, pszTarget, static_cast<uint64_t>(nOffset),
1385 : static_cast<uint64_t>(nSourceSize));
1386 1 : ret = -1;
1387 : }
1388 :
1389 2035 : if (VSIFCloseL(fpOut) != 0)
1390 : {
1391 1 : ret = -1;
1392 : }
1393 2035 : return ret;
1394 : }
1395 :
1396 : /************************************************************************/
1397 : /* Sync() */
1398 : /************************************************************************/
1399 :
1400 28 : bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget,
1401 : const char *const *papszOptions,
1402 : GDALProgressFunc pProgressFunc,
1403 : void *pProgressData, char ***ppapszOutputs)
1404 : {
1405 28 : const char SOURCE_SEP = VSIGetDirectorySeparator(pszSource)[0];
1406 :
1407 28 : if (ppapszOutputs)
1408 : {
1409 0 : *ppapszOutputs = nullptr;
1410 : }
1411 :
1412 : VSIStatBufL sSource;
1413 56 : CPLString osSource(pszSource);
1414 56 : CPLString osSourceWithoutSlash(pszSource);
1415 34 : if (osSourceWithoutSlash.back() == '/' ||
1416 6 : osSourceWithoutSlash.back() == '\\')
1417 : {
1418 22 : osSourceWithoutSlash.resize(osSourceWithoutSlash.size() - 1);
1419 : }
1420 28 : if (VSIStatL(osSourceWithoutSlash, &sSource) < 0)
1421 : {
1422 1 : CPLError(CE_Failure, CPLE_FileIO, "%s does not exist", pszSource);
1423 1 : return false;
1424 : }
1425 :
1426 27 : if (VSI_ISDIR(sSource.st_mode))
1427 : {
1428 22 : CPLString osTargetDir(pszTarget);
1429 11 : if (osSource.back() != '/' && osSource.back() != '\\')
1430 : {
1431 : osTargetDir = CPLFormFilename(osTargetDir,
1432 1 : CPLGetFilename(pszSource), nullptr);
1433 : }
1434 :
1435 : VSIStatBufL sTarget;
1436 11 : bool ret = true;
1437 11 : if (VSIStatL(osTargetDir, &sTarget) < 0)
1438 : {
1439 9 : if (VSIMkdirRecursive(osTargetDir, 0755) < 0)
1440 : {
1441 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create directory %s",
1442 : osTargetDir.c_str());
1443 1 : return false;
1444 : }
1445 : }
1446 :
1447 10 : if (!CPLFetchBool(papszOptions, "STOP_ON_DIR", false))
1448 : {
1449 20 : CPLStringList aosChildOptions(CSLDuplicate(papszOptions));
1450 10 : if (!CPLFetchBool(papszOptions, "RECURSIVE", true))
1451 : {
1452 0 : aosChildOptions.SetNameValue("RECURSIVE", nullptr);
1453 0 : aosChildOptions.AddString("STOP_ON_DIR=TRUE");
1454 : }
1455 :
1456 10 : char **papszSrcFiles = VSIReadDir(osSourceWithoutSlash);
1457 10 : int nFileCount = 0;
1458 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter)
1459 : {
1460 17 : if (strcmp(*iter, ".") != 0 && strcmp(*iter, "..") != 0)
1461 : {
1462 17 : nFileCount++;
1463 : }
1464 : }
1465 10 : int iFile = 0;
1466 27 : for (auto iter = papszSrcFiles; iter && *iter; ++iter, ++iFile)
1467 : {
1468 17 : if (strcmp(*iter, ".") == 0 || strcmp(*iter, "..") == 0)
1469 : {
1470 0 : continue;
1471 : }
1472 : CPLString osSubSource(
1473 17 : CPLFormFilename(osSourceWithoutSlash, *iter, nullptr));
1474 : CPLString osSubTarget(
1475 17 : CPLFormFilename(osTargetDir, *iter, nullptr));
1476 : // coverity[divide_by_zero]
1477 34 : void *pScaledProgress = GDALCreateScaledProgress(
1478 17 : double(iFile) / nFileCount, double(iFile + 1) / nFileCount,
1479 : pProgressFunc, pProgressData);
1480 17 : ret = Sync((osSubSource + SOURCE_SEP).c_str(), osSubTarget,
1481 17 : aosChildOptions.List(), GDALScaledProgress,
1482 17 : pScaledProgress, nullptr);
1483 17 : GDALDestroyScaledProgress(pScaledProgress);
1484 17 : if (!ret)
1485 : {
1486 0 : break;
1487 : }
1488 : }
1489 10 : CSLDestroy(papszSrcFiles);
1490 : }
1491 10 : return ret;
1492 : }
1493 :
1494 : VSIStatBufL sTarget;
1495 32 : CPLString osTarget(pszTarget);
1496 16 : if (VSIStatL(osTarget, &sTarget) == 0)
1497 : {
1498 4 : bool bTargetIsFile = true;
1499 4 : if (VSI_ISDIR(sTarget.st_mode))
1500 : {
1501 : osTarget =
1502 2 : CPLFormFilename(osTarget, CPLGetFilename(pszSource), nullptr);
1503 3 : bTargetIsFile = VSIStatL(osTarget, &sTarget) == 0 &&
1504 1 : !CPL_TO_BOOL(VSI_ISDIR(sTarget.st_mode));
1505 : }
1506 4 : if (bTargetIsFile)
1507 : {
1508 3 : if (sSource.st_size == sTarget.st_size &&
1509 3 : sSource.st_mtime == sTarget.st_mtime && sSource.st_mtime != 0)
1510 : {
1511 2 : CPLDebug("VSI",
1512 : "%s and %s have same size and modification "
1513 : "date. Skipping copying",
1514 : osSourceWithoutSlash.c_str(), osTarget.c_str());
1515 2 : return true;
1516 : }
1517 : }
1518 : }
1519 :
1520 14 : VSILFILE *fpIn = VSIFOpenExL(osSourceWithoutSlash, "rb", TRUE);
1521 14 : if (fpIn == nullptr)
1522 : {
1523 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s",
1524 : osSourceWithoutSlash.c_str());
1525 0 : return false;
1526 : }
1527 :
1528 14 : VSILFILE *fpOut = VSIFOpenExL(osTarget.c_str(), "wb", TRUE);
1529 14 : if (fpOut == nullptr)
1530 : {
1531 1 : CPLError(CE_Failure, CPLE_FileIO, "Cannot create %s", osTarget.c_str());
1532 1 : VSIFCloseL(fpIn);
1533 1 : return false;
1534 : }
1535 :
1536 13 : bool ret = true;
1537 13 : constexpr size_t nBufferSize = 10 * 4096;
1538 26 : std::vector<GByte> abyBuffer(nBufferSize, 0);
1539 13 : GUIntBig nOffset = 0;
1540 13 : CPLString osMsg;
1541 13 : osMsg.Printf("Copying of %s", osSourceWithoutSlash.c_str());
1542 : while (true)
1543 : {
1544 16 : size_t nRead = VSIFReadL(&abyBuffer[0], 1, nBufferSize, fpIn);
1545 16 : size_t nWritten = VSIFWriteL(&abyBuffer[0], 1, nRead, fpOut);
1546 16 : if (nWritten != nRead)
1547 : {
1548 0 : CPLError(CE_Failure, CPLE_FileIO, "Copying of %s to %s failed",
1549 : osSourceWithoutSlash.c_str(), osTarget.c_str());
1550 0 : ret = false;
1551 0 : break;
1552 : }
1553 16 : nOffset += nRead;
1554 16 : if (pProgressFunc && !pProgressFunc(double(nOffset) / sSource.st_size,
1555 : osMsg.c_str(), pProgressData))
1556 : {
1557 0 : ret = false;
1558 0 : break;
1559 : }
1560 16 : if (nRead < nBufferSize)
1561 : {
1562 13 : break;
1563 : }
1564 3 : }
1565 :
1566 13 : VSIFCloseL(fpIn);
1567 13 : if (VSIFCloseL(fpOut) != 0)
1568 : {
1569 0 : ret = false;
1570 : }
1571 13 : return ret;
1572 : }
1573 :
1574 : /************************************************************************/
1575 : /* VSIDIREntry() */
1576 : /************************************************************************/
1577 :
1578 6580 : VSIDIREntry::VSIDIREntry()
1579 : : pszName(nullptr), nMode(0), nSize(0), nMTime(0), bModeKnown(false),
1580 6580 : bSizeKnown(false), bMTimeKnown(false), papszExtra(nullptr)
1581 : {
1582 6580 : }
1583 :
1584 : /************************************************************************/
1585 : /* VSIDIREntry() */
1586 : /************************************************************************/
1587 :
1588 6 : VSIDIREntry::VSIDIREntry(const VSIDIREntry &other)
1589 6 : : pszName(VSIStrdup(other.pszName)), nMode(other.nMode), nSize(other.nSize),
1590 6 : nMTime(other.nMTime), bModeKnown(other.bModeKnown),
1591 6 : bSizeKnown(other.bSizeKnown), bMTimeKnown(other.bMTimeKnown),
1592 6 : papszExtra(CSLDuplicate(other.papszExtra))
1593 : {
1594 6 : }
1595 :
1596 : /************************************************************************/
1597 : /* ~VSIDIREntry() */
1598 : /************************************************************************/
1599 :
1600 13172 : VSIDIREntry::~VSIDIREntry()
1601 : {
1602 6586 : CPLFree(pszName);
1603 6586 : CSLDestroy(papszExtra);
1604 6586 : }
1605 :
1606 : /************************************************************************/
1607 : /* ~VSIDIR() */
1608 : /************************************************************************/
1609 :
1610 6187 : VSIDIR::~VSIDIR()
1611 : {
1612 6187 : }
1613 :
1614 : /************************************************************************/
1615 : /* VSIDIRGeneric */
1616 : /************************************************************************/
1617 :
1618 : namespace
1619 : {
1620 : struct VSIDIRGeneric : public VSIDIR
1621 : {
1622 : CPLString osRootPath{};
1623 : CPLString osBasePath{};
1624 : char **papszContent = nullptr;
1625 : int nRecurseDepth = 0;
1626 : int nPos = 0;
1627 : VSIDIREntry entry{};
1628 : std::vector<VSIDIRGeneric *> aoStackSubDir{};
1629 : VSIFilesystemHandler *poFS = nullptr;
1630 : std::string m_osFilterPrefix{};
1631 :
1632 5838 : explicit VSIDIRGeneric(VSIFilesystemHandler *poFSIn) : poFS(poFSIn)
1633 : {
1634 5838 : }
1635 :
1636 : ~VSIDIRGeneric();
1637 :
1638 : const VSIDIREntry *NextDirEntry() override;
1639 :
1640 : VSIDIRGeneric(const VSIDIRGeneric &) = delete;
1641 : VSIDIRGeneric &operator=(const VSIDIRGeneric &) = delete;
1642 : };
1643 :
1644 : /************************************************************************/
1645 : /* ~VSIDIRGeneric() */
1646 : /************************************************************************/
1647 :
1648 17514 : VSIDIRGeneric::~VSIDIRGeneric()
1649 : {
1650 5843 : while (!aoStackSubDir.empty())
1651 : {
1652 5 : delete aoStackSubDir.back();
1653 5 : aoStackSubDir.pop_back();
1654 : }
1655 5838 : CSLDestroy(papszContent);
1656 11676 : }
1657 :
1658 : } // namespace
1659 :
1660 : /************************************************************************/
1661 : /* OpenDir() */
1662 : /************************************************************************/
1663 :
1664 5874 : VSIDIR *VSIFilesystemHandler::OpenDir(const char *pszPath, int nRecurseDepth,
1665 : const char *const *papszOptions)
1666 : {
1667 5874 : char **papszContent = VSIReadDir(pszPath);
1668 : VSIStatBufL sStatL;
1669 6793 : if (papszContent == nullptr &&
1670 919 : (VSIStatL(pszPath, &sStatL) != 0 || !VSI_ISDIR(sStatL.st_mode)))
1671 : {
1672 36 : return nullptr;
1673 : }
1674 5838 : VSIDIRGeneric *dir = new VSIDIRGeneric(this);
1675 5838 : dir->osRootPath = pszPath;
1676 11676 : if (!dir->osRootPath.empty() &&
1677 5838 : (dir->osRootPath.back() == '/' || dir->osRootPath.back() == '\\'))
1678 2 : dir->osRootPath.pop_back();
1679 5838 : dir->nRecurseDepth = nRecurseDepth;
1680 5838 : dir->papszContent = papszContent;
1681 5838 : dir->m_osFilterPrefix = CSLFetchNameValueDef(papszOptions, "PREFIX", "");
1682 5838 : return dir;
1683 : }
1684 :
1685 : /************************************************************************/
1686 : /* NextDirEntry() */
1687 : /************************************************************************/
1688 :
1689 532000 : const VSIDIREntry *VSIDIRGeneric::NextDirEntry()
1690 : {
1691 532000 : const char SEP = VSIGetDirectorySeparator(osRootPath.c_str())[0];
1692 :
1693 532001 : begin:
1694 532001 : if (VSI_ISDIR(entry.nMode) && nRecurseDepth != 0)
1695 : {
1696 2331 : CPLString osCurFile(osRootPath);
1697 2331 : if (!osCurFile.empty())
1698 2331 : osCurFile += SEP;
1699 2331 : osCurFile += entry.pszName;
1700 : auto subdir =
1701 2331 : static_cast<VSIDIRGeneric *>(poFS->VSIFilesystemHandler::OpenDir(
1702 2331 : osCurFile, nRecurseDepth - 1, nullptr));
1703 2331 : if (subdir)
1704 : {
1705 2331 : subdir->osRootPath = osRootPath;
1706 2331 : subdir->osBasePath = entry.pszName;
1707 2331 : subdir->m_osFilterPrefix = m_osFilterPrefix;
1708 2331 : aoStackSubDir.push_back(subdir);
1709 : }
1710 2331 : entry.nMode = 0;
1711 : }
1712 :
1713 534327 : while (!aoStackSubDir.empty())
1714 : {
1715 515864 : auto l_entry = aoStackSubDir.back()->NextDirEntry();
1716 515864 : if (l_entry)
1717 : {
1718 513538 : return l_entry;
1719 : }
1720 2326 : delete aoStackSubDir.back();
1721 2326 : aoStackSubDir.pop_back();
1722 : }
1723 :
1724 18463 : if (papszContent == nullptr)
1725 : {
1726 883 : return nullptr;
1727 : }
1728 :
1729 : while (true)
1730 : {
1731 17607 : if (!papszContent[nPos])
1732 : {
1733 4945 : return nullptr;
1734 : }
1735 : // Skip . and ..entries
1736 12662 : if (papszContent[nPos][0] == '.' &&
1737 920 : (papszContent[nPos][1] == '\0' ||
1738 920 : (papszContent[nPos][1] == '.' && papszContent[nPos][2] == '\0')))
1739 : {
1740 22 : nPos++;
1741 : }
1742 : else
1743 : {
1744 12640 : CPLFree(entry.pszName);
1745 12640 : CPLString osName(osBasePath);
1746 12640 : if (!osName.empty())
1747 8091 : osName += SEP;
1748 12640 : osName += papszContent[nPos];
1749 12640 : nPos++;
1750 :
1751 12640 : entry.pszName = CPLStrdup(osName);
1752 12640 : entry.nMode = 0;
1753 12640 : CPLString osCurFile(osRootPath);
1754 12640 : if (!osCurFile.empty())
1755 12640 : osCurFile += SEP;
1756 12640 : osCurFile += entry.pszName;
1757 :
1758 25270 : const auto StatFile = [&osCurFile, this]()
1759 : {
1760 : VSIStatBufL sStatL;
1761 12635 : if (VSIStatL(osCurFile, &sStatL) == 0)
1762 : {
1763 12635 : entry.nMode = sStatL.st_mode;
1764 12635 : entry.nSize = sStatL.st_size;
1765 12635 : entry.nMTime = sStatL.st_mtime;
1766 12635 : entry.bModeKnown = true;
1767 12635 : entry.bSizeKnown = true;
1768 12635 : entry.bMTimeKnown = true;
1769 : }
1770 : else
1771 : {
1772 0 : entry.nMode = 0;
1773 0 : entry.nSize = 0;
1774 0 : entry.nMTime = 0;
1775 0 : entry.bModeKnown = false;
1776 0 : entry.bSizeKnown = false;
1777 0 : entry.bMTimeKnown = false;
1778 : }
1779 12635 : };
1780 :
1781 12652 : if (!m_osFilterPrefix.empty() &&
1782 12 : m_osFilterPrefix.size() > osName.size())
1783 : {
1784 6 : if (STARTS_WITH(m_osFilterPrefix.c_str(), osName.c_str()) &&
1785 2 : m_osFilterPrefix[osName.size()] == SEP)
1786 : {
1787 1 : StatFile();
1788 1 : if (VSI_ISDIR(entry.nMode))
1789 : {
1790 1 : goto begin;
1791 : }
1792 : }
1793 3 : continue;
1794 : }
1795 12644 : if (!m_osFilterPrefix.empty() &&
1796 8 : !STARTS_WITH(osName.c_str(), m_osFilterPrefix.c_str()))
1797 : {
1798 2 : continue;
1799 : }
1800 :
1801 12634 : StatFile();
1802 :
1803 12634 : break;
1804 : }
1805 27 : }
1806 :
1807 12634 : return &(entry);
1808 : }
1809 :
1810 : /************************************************************************/
1811 : /* UnlinkBatch() */
1812 : /************************************************************************/
1813 :
1814 1 : int *VSIFilesystemHandler::UnlinkBatch(CSLConstList papszFiles)
1815 : {
1816 : int *panRet =
1817 1 : static_cast<int *>(CPLMalloc(sizeof(int) * CSLCount(papszFiles)));
1818 3 : for (int i = 0; papszFiles && papszFiles[i]; ++i)
1819 : {
1820 2 : panRet[i] = VSIUnlink(papszFiles[i]) == 0;
1821 : }
1822 1 : return panRet;
1823 : }
1824 :
1825 : /************************************************************************/
1826 : /* RmdirRecursive() */
1827 : /************************************************************************/
1828 :
1829 2239 : int VSIFilesystemHandler::RmdirRecursive(const char *pszDirname)
1830 : {
1831 4478 : CPLString osDirnameWithoutEndSlash(pszDirname);
1832 4478 : if (!osDirnameWithoutEndSlash.empty() &&
1833 2239 : (osDirnameWithoutEndSlash.back() == '/' ||
1834 2238 : osDirnameWithoutEndSlash.back() == '\\'))
1835 : {
1836 1 : osDirnameWithoutEndSlash.resize(osDirnameWithoutEndSlash.size() - 1);
1837 : }
1838 :
1839 2239 : const char SEP = VSIGetDirectorySeparator(pszDirname)[0];
1840 :
1841 4478 : CPLStringList aosOptions;
1842 : auto poDir =
1843 4478 : std::unique_ptr<VSIDIR>(OpenDir(pszDirname, -1, aosOptions.List()));
1844 2239 : if (!poDir)
1845 39 : return -1;
1846 4400 : std::vector<std::string> aosDirs;
1847 : while (true)
1848 : {
1849 33899 : auto entry = poDir->NextDirEntry();
1850 33899 : if (!entry)
1851 2200 : break;
1852 :
1853 63398 : const CPLString osFilename(osDirnameWithoutEndSlash + SEP +
1854 63398 : entry->pszName);
1855 31699 : if ((entry->nMode & S_IFDIR))
1856 : {
1857 1359 : aosDirs.push_back(osFilename);
1858 : }
1859 : else
1860 : {
1861 30340 : if (VSIUnlink(osFilename) != 0)
1862 0 : return -1;
1863 : }
1864 31699 : }
1865 :
1866 : // Sort in reverse order, so that inner-most directories are deleted first
1867 2200 : std::sort(aosDirs.begin(), aosDirs.end(),
1868 3219 : [](const std::string &a, const std::string &b) { return a > b; });
1869 :
1870 3559 : for (const auto &osDir : aosDirs)
1871 : {
1872 1359 : if (VSIRmdir(osDir.c_str()) != 0)
1873 0 : return -1;
1874 : }
1875 :
1876 2200 : return VSIRmdir(pszDirname);
1877 : }
1878 :
1879 : /************************************************************************/
1880 : /* GetFileMetadata() */
1881 : /************************************************************************/
1882 :
1883 0 : char **VSIFilesystemHandler::GetFileMetadata(const char * /* pszFilename*/,
1884 : const char * /*pszDomain*/,
1885 : CSLConstList /*papszOptions*/)
1886 : {
1887 0 : return nullptr;
1888 : }
1889 :
1890 : /************************************************************************/
1891 : /* SetFileMetadata() */
1892 : /************************************************************************/
1893 :
1894 0 : bool VSIFilesystemHandler::SetFileMetadata(const char * /* pszFilename*/,
1895 : CSLConstList /* papszMetadata */,
1896 : const char * /* pszDomain */,
1897 : CSLConstList /* papszOptions */)
1898 : {
1899 0 : CPLError(CE_Failure, CPLE_NotSupported, "SetFileMetadata() not supported");
1900 0 : return false;
1901 : }
1902 :
1903 : #endif
1904 :
1905 : /************************************************************************/
1906 : /* VSIFOpenExL() */
1907 : /************************************************************************/
1908 :
1909 : /**
1910 : * \brief Open file.
1911 : *
1912 : * This function opens a file with the desired access. Large files (larger
1913 : * than 2GB) should be supported. Binary access is always implied and
1914 : * the "b" does not need to be included in the pszAccess string.
1915 : *
1916 : * Note that the "VSILFILE *" returned by this function is
1917 : * *NOT* a standard C library FILE *, and cannot be used with any functions
1918 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
1919 : *
1920 : * On windows it is possible to define the configuration option
1921 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
1922 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
1923 : *
1924 : * This method goes through the VSIFileHandler virtualization and may
1925 : * work on unusual filesystems such as in memory.
1926 : *
1927 : * Analog of the POSIX fopen() function.
1928 : *
1929 : * @param pszFilename the file to open. UTF-8 encoded.
1930 : * @param pszAccess access requested (i.e. "r", "r+", "w")
1931 : * @param bSetError flag determining whether or not this open call
1932 : * should set VSIErrors on failure.
1933 : *
1934 : * @return NULL on failure, or the file handle.
1935 : *
1936 : * @since GDAL 2.1
1937 : */
1938 :
1939 345649 : VSILFILE *VSIFOpenExL(const char *pszFilename, const char *pszAccess,
1940 : int bSetError)
1941 :
1942 : {
1943 345649 : return VSIFOpenEx2L(pszFilename, pszAccess, bSetError, nullptr);
1944 : }
1945 :
1946 : /************************************************************************/
1947 : /* VSIFOpenEx2L() */
1948 : /************************************************************************/
1949 :
1950 : /**
1951 : * \brief Open file.
1952 : *
1953 : * This function opens a file with the desired access. Large files (larger
1954 : * than 2GB) should be supported. Binary access is always implied and
1955 : * the "b" does not need to be included in the pszAccess string.
1956 : *
1957 : * Note that the "VSILFILE *" returned by this function is
1958 : * *NOT* a standard C library FILE *, and cannot be used with any functions
1959 : * other than the "VSI*L" family of functions. They aren't "real" FILE objects.
1960 : *
1961 : * On windows it is possible to define the configuration option
1962 : * GDAL_FILE_IS_UTF8 to have pszFilename treated as being in the local
1963 : * encoding instead of UTF-8, restoring the pre-1.8.0 behavior of VSIFOpenL().
1964 : *
1965 : * This method goes through the VSIFileHandler virtualization and may
1966 : * work on unusual filesystems such as in memory.
1967 : *
1968 : * The following options are supported:
1969 : * <ul>
1970 : * <li>MIME headers such as Content-Type and Content-Encoding
1971 : * are supported for the /vsis3/, /vsigs/, /vsiaz/, /vsiadls/ file systems.</li>
1972 : * <li>DISABLE_READDIR_ON_OPEN=YES/NO (GDAL >= 3.6) for /vsicurl/ and other
1973 : * network-based file systems. By default, directory file listing is done,
1974 : * unless YES is specified.</li>
1975 : * <li>WRITE_THROUGH=YES (GDAL >= 3.8) for the Windows regular files to
1976 : * set the FILE_FLAG_WRITE_THROUGH flag to the CreateFile() function. In that
1977 : * mode, the data is written to the system cache but is flushed to disk without
1978 : * delay.</li>
1979 : * </ul>
1980 : *
1981 : * Analog of the POSIX fopen() function.
1982 : *
1983 : * @param pszFilename the file to open. UTF-8 encoded.
1984 : * @param pszAccess access requested (i.e. "r", "r+", "w")
1985 : * @param bSetError flag determining whether or not this open call
1986 : * should set VSIErrors on failure.
1987 : * @param papszOptions NULL or NULL-terminated list of strings. The content is
1988 : * highly file system dependent.
1989 : *
1990 : *
1991 : * @return NULL on failure, or the file handle.
1992 : *
1993 : * @since GDAL 3.3
1994 : */
1995 :
1996 374193 : VSILFILE *VSIFOpenEx2L(const char *pszFilename, const char *pszAccess,
1997 : int bSetError, CSLConstList papszOptions)
1998 :
1999 : {
2000 : // Too long filenames can cause excessive memory allocation due to
2001 : // recursion in some filesystem handlers
2002 374193 : constexpr size_t knMaxPath = 8192;
2003 374193 : if (CPLStrnlen(pszFilename, knMaxPath) == knMaxPath)
2004 0 : return nullptr;
2005 :
2006 374221 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2007 :
2008 374227 : VSILFILE *fp = poFSHandler->Open(pszFilename, pszAccess,
2009 374240 : CPL_TO_BOOL(bSetError), papszOptions);
2010 :
2011 : VSIDebug4("VSIFOpenEx2L(%s,%s,%d) = %p", pszFilename, pszAccess, bSetError,
2012 : fp);
2013 :
2014 374223 : return fp;
2015 : }
2016 :
2017 : /************************************************************************/
2018 : /* VSIFCloseL() */
2019 : /************************************************************************/
2020 :
2021 : /**
2022 : * \fn VSIVirtualHandle::Close()
2023 : * \brief Close file.
2024 : *
2025 : * This function closes the indicated file.
2026 : *
2027 : * This method goes through the VSIFileHandler virtualization and may
2028 : * work on unusual filesystems such as in memory.
2029 : *
2030 : * Analog of the POSIX fclose() function.
2031 : *
2032 : * @return 0 on success or -1 on failure.
2033 : */
2034 :
2035 : /**
2036 : * \brief Close file.
2037 : *
2038 : * This function closes the indicated file.
2039 : *
2040 : * This method goes through the VSIFileHandler virtualization and may
2041 : * work on unusual filesystems such as in memory.
2042 : *
2043 : * Analog of the POSIX fclose() function.
2044 : *
2045 : * @param fp file handle opened with VSIFOpenL(). Passing a nullptr produces
2046 : * undefined behavior.
2047 : *
2048 : * @return 0 on success or -1 on failure.
2049 : */
2050 :
2051 267087 : int VSIFCloseL(VSILFILE *fp)
2052 :
2053 : {
2054 : VSIDebug1("VSIFCloseL(%p)", fp);
2055 :
2056 267087 : const int nResult = fp->Close();
2057 :
2058 266781 : delete fp;
2059 :
2060 266734 : return nResult;
2061 : }
2062 :
2063 : /************************************************************************/
2064 : /* VSIFSeekL() */
2065 : /************************************************************************/
2066 :
2067 : /**
2068 : * \fn int VSIVirtualHandle::Seek( vsi_l_offset nOffset, int nWhence )
2069 : * \brief Seek to requested offset.
2070 : *
2071 : * Seek to the desired offset (nOffset) in the indicated file.
2072 : *
2073 : * This method goes through the VSIFileHandler virtualization and may
2074 : * work on unusual filesystems such as in memory.
2075 : *
2076 : * Analog of the POSIX fseek() call.
2077 : *
2078 : * Caution: vsi_l_offset is a unsigned type, so SEEK_CUR can only be used
2079 : * for positive seek. If negative seek is needed, use
2080 : * handle->Seek( handle->Tell() + negative_offset, SEEK_SET ).
2081 : *
2082 : * @param nOffset offset in bytes.
2083 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
2084 : *
2085 : * @return 0 on success or -1 one failure.
2086 : */
2087 :
2088 : /**
2089 : * \brief Seek to requested offset.
2090 : *
2091 : * Seek to the desired offset (nOffset) in the indicated file.
2092 : *
2093 : * This method goes through the VSIFileHandler virtualization and may
2094 : * work on unusual filesystems such as in memory.
2095 : *
2096 : * Analog of the POSIX fseek() call.
2097 : *
2098 : * Caution: vsi_l_offset is a unsigned type, so SEEK_CUR can only be used
2099 : * for positive seek. If negative seek is needed, use
2100 : * VSIFSeekL( fp, VSIFTellL(fp) + negative_offset, SEEK_SET ).
2101 : *
2102 : * @param fp file handle opened with VSIFOpenL().
2103 : * @param nOffset offset in bytes.
2104 : * @param nWhence one of SEEK_SET, SEEK_CUR or SEEK_END.
2105 : *
2106 : * @return 0 on success or -1 one failure.
2107 : */
2108 :
2109 7601040 : int VSIFSeekL(VSILFILE *fp, vsi_l_offset nOffset, int nWhence)
2110 :
2111 : {
2112 7601040 : return fp->Seek(nOffset, nWhence);
2113 : }
2114 :
2115 : /************************************************************************/
2116 : /* VSIFTellL() */
2117 : /************************************************************************/
2118 :
2119 : /**
2120 : * \fn VSIVirtualHandle::Tell()
2121 : * \brief Tell current file offset.
2122 : *
2123 : * Returns the current file read/write offset in bytes from the beginning of
2124 : * the file.
2125 : *
2126 : * This method goes through the VSIFileHandler virtualization and may
2127 : * work on unusual filesystems such as in memory.
2128 : *
2129 : * Analog of the POSIX ftell() call.
2130 : *
2131 : * @return file offset in bytes.
2132 : */
2133 :
2134 : /**
2135 : * \brief Tell current file offset.
2136 : *
2137 : * Returns the current file read/write offset in bytes from the beginning of
2138 : * the file.
2139 : *
2140 : * This method goes through the VSIFileHandler virtualization and may
2141 : * work on unusual filesystems such as in memory.
2142 : *
2143 : * Analog of the POSIX ftell() call.
2144 : *
2145 : * @param fp file handle opened with VSIFOpenL().
2146 : *
2147 : * @return file offset in bytes.
2148 : */
2149 :
2150 5295370 : vsi_l_offset VSIFTellL(VSILFILE *fp)
2151 :
2152 : {
2153 5295370 : return fp->Tell();
2154 : }
2155 :
2156 : /************************************************************************/
2157 : /* VSIRewindL() */
2158 : /************************************************************************/
2159 :
2160 : /**
2161 : * \brief Rewind the file pointer to the beginning of the file.
2162 : *
2163 : * This is equivalent to VSIFSeekL( fp, 0, SEEK_SET )
2164 : *
2165 : * Analog of the POSIX rewind() call.
2166 : *
2167 : * @param fp file handle opened with VSIFOpenL().
2168 : */
2169 :
2170 70202 : void VSIRewindL(VSILFILE *fp)
2171 :
2172 : {
2173 70202 : CPL_IGNORE_RET_VAL(VSIFSeekL(fp, 0, SEEK_SET));
2174 70197 : }
2175 :
2176 : /************************************************************************/
2177 : /* VSIFFlushL() */
2178 : /************************************************************************/
2179 :
2180 : /**
2181 : * \fn VSIVirtualHandle::Flush()
2182 : * \brief Flush pending writes to disk.
2183 : *
2184 : * For files in write or update mode and on filesystem types where it is
2185 : * applicable, all pending output on the file is flushed to the physical disk.
2186 : *
2187 : * This method goes through the VSIFileHandler virtualization and may
2188 : * work on unusual filesystems such as in memory.
2189 : *
2190 : * Analog of the POSIX fflush() call.
2191 : *
2192 : * On Windows regular files, this method does nothing, unless the
2193 : * VSI_FLUSH configuration option is set to YES (and only when the file has
2194 : * *not* been opened with the WRITE_THROUGH option).
2195 : *
2196 : * @return 0 on success or -1 on error.
2197 : */
2198 :
2199 : /**
2200 : * \brief Flush pending writes to disk.
2201 : *
2202 : * For files in write or update mode and on filesystem types where it is
2203 : * applicable, all pending output on the file is flushed to the physical disk.
2204 : *
2205 : * This method goes through the VSIFileHandler virtualization and may
2206 : * work on unusual filesystems such as in memory.
2207 : *
2208 : * Analog of the POSIX fflush() call.
2209 : *
2210 : * On Windows regular files, this method does nothing, unless the
2211 : * VSI_FLUSH configuration option is set to YES (and only when the file has
2212 : * *not* been opened with the WRITE_THROUGH option).
2213 : *
2214 : * @param fp file handle opened with VSIFOpenL().
2215 : *
2216 : * @return 0 on success or -1 on error.
2217 : */
2218 :
2219 67307 : int VSIFFlushL(VSILFILE *fp)
2220 :
2221 : {
2222 67307 : return fp->Flush();
2223 : }
2224 :
2225 : /************************************************************************/
2226 : /* VSIFReadL() */
2227 : /************************************************************************/
2228 :
2229 : /**
2230 : * \fn VSIVirtualHandle::Read( void *pBuffer, size_t nSize, size_t nCount )
2231 : * \brief Read bytes from file.
2232 : *
2233 : * Reads nCount objects of nSize bytes from the indicated file at the
2234 : * current offset into the indicated buffer.
2235 : *
2236 : * This method goes through the VSIFileHandler virtualization and may
2237 : * work on unusual filesystems such as in memory.
2238 : *
2239 : * Analog of the POSIX fread() call.
2240 : *
2241 : * @param pBuffer the buffer into which the data should be read (at least
2242 : * nCount * nSize bytes in size.
2243 : * @param nSize size of objects to read in bytes.
2244 : * @param nCount number of objects to read.
2245 : *
2246 : * @return number of objects successfully read.
2247 : */
2248 :
2249 : /**
2250 : * \brief Read bytes from file.
2251 : *
2252 : * Reads nCount objects of nSize bytes from the indicated file at the
2253 : * current offset into the indicated buffer.
2254 : *
2255 : * This method goes through the VSIFileHandler virtualization and may
2256 : * work on unusual filesystems such as in memory.
2257 : *
2258 : * Analog of the POSIX fread() call.
2259 : *
2260 : * @param pBuffer the buffer into which the data should be read (at least
2261 : * nCount * nSize bytes in size.
2262 : * @param nSize size of objects to read in bytes.
2263 : * @param nCount number of objects to read.
2264 : * @param fp file handle opened with VSIFOpenL().
2265 : *
2266 : * @return number of objects successfully read.
2267 : */
2268 :
2269 14058100 : size_t VSIFReadL(void *pBuffer, size_t nSize, size_t nCount, VSILFILE *fp)
2270 :
2271 : {
2272 14058100 : return fp->Read(pBuffer, nSize, nCount);
2273 : }
2274 :
2275 : /************************************************************************/
2276 : /* VSIFReadMultiRangeL() */
2277 : /************************************************************************/
2278 :
2279 : /**
2280 : * \fn VSIVirtualHandle::ReadMultiRange( int nRanges, void ** ppData,
2281 : * const vsi_l_offset* panOffsets,
2282 : * const size_t* panSizes )
2283 : * \brief Read several ranges of bytes from file.
2284 : *
2285 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
2286 : * offset panOffsets[i] into the buffer ppData[i].
2287 : *
2288 : * Ranges must be sorted in ascending start offset, and must not overlap each
2289 : * other.
2290 : *
2291 : * This method goes through the VSIFileHandler virtualization and may
2292 : * work on unusual filesystems such as in memory or /vsicurl/.
2293 : *
2294 : * @param nRanges number of ranges to read.
2295 : * @param ppData array of nRanges buffer into which the data should be read
2296 : * (ppData[i] must be at list panSizes[i] bytes).
2297 : * @param panOffsets array of nRanges offsets at which the data should be read.
2298 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
2299 : *
2300 : * @return 0 in case of success, -1 otherwise.
2301 : * @since GDAL 1.9.0
2302 : */
2303 :
2304 : /**
2305 : * \brief Read several ranges of bytes from file.
2306 : *
2307 : * Reads nRanges objects of panSizes[i] bytes from the indicated file at the
2308 : * offset panOffsets[i] into the buffer ppData[i].
2309 : *
2310 : * Ranges must be sorted in ascending start offset, and must not overlap each
2311 : * other.
2312 : *
2313 : * This method goes through the VSIFileHandler virtualization and may
2314 : * work on unusual filesystems such as in memory or /vsicurl/.
2315 : *
2316 : * @param nRanges number of ranges to read.
2317 : * @param ppData array of nRanges buffer into which the data should be read
2318 : * (ppData[i] must be at list panSizes[i] bytes).
2319 : * @param panOffsets array of nRanges offsets at which the data should be read.
2320 : * @param panSizes array of nRanges sizes of objects to read (in bytes).
2321 : * @param fp file handle opened with VSIFOpenL().
2322 : *
2323 : * @return 0 in case of success, -1 otherwise.
2324 : * @since GDAL 1.9.0
2325 : */
2326 :
2327 603 : int VSIFReadMultiRangeL(int nRanges, void **ppData,
2328 : const vsi_l_offset *panOffsets, const size_t *panSizes,
2329 : VSILFILE *fp)
2330 : {
2331 603 : return fp->ReadMultiRange(nRanges, ppData, panOffsets, panSizes);
2332 : }
2333 :
2334 : /************************************************************************/
2335 : /* VSIFWriteL() */
2336 : /************************************************************************/
2337 :
2338 : /**
2339 : * \fn VSIVirtualHandle::Write( const void *pBuffer,
2340 : * size_t nSize, size_t nCount )
2341 : * \brief Write bytes to file.
2342 : *
2343 : * Writes nCount objects of nSize bytes to the indicated file at the
2344 : * current offset into the indicated buffer.
2345 : *
2346 : * This method goes through the VSIFileHandler virtualization and may
2347 : * work on unusual filesystems such as in memory.
2348 : *
2349 : * Analog of the POSIX fwrite() call.
2350 : *
2351 : * @param pBuffer the buffer from which the data should be written (at least
2352 : * nCount * nSize bytes in size.
2353 : * @param nSize size of objects to write in bytes.
2354 : * @param nCount number of objects to write.
2355 : *
2356 : * @return number of objects successfully written.
2357 : */
2358 :
2359 : /**
2360 : * \brief Write bytes to file.
2361 : *
2362 : * Writes nCount objects of nSize bytes to the indicated file at the
2363 : * current offset into the indicated buffer.
2364 : *
2365 : * This method goes through the VSIFileHandler virtualization and may
2366 : * work on unusual filesystems such as in memory.
2367 : *
2368 : * Analog of the POSIX fwrite() call.
2369 : *
2370 : * @param pBuffer the buffer from which the data should be written (at least
2371 : * nCount * nSize bytes in size.
2372 : * @param nSize size of objects to write in bytes.
2373 : * @param nCount number of objects to write.
2374 : * @param fp file handle opened with VSIFOpenL().
2375 : *
2376 : * @return number of objects successfully written.
2377 : */
2378 :
2379 4200380 : size_t VSIFWriteL(const void *pBuffer, size_t nSize, size_t nCount,
2380 : VSILFILE *fp)
2381 :
2382 : {
2383 4200380 : return fp->Write(pBuffer, nSize, nCount);
2384 : }
2385 :
2386 : /************************************************************************/
2387 : /* VSIFEofL() */
2388 : /************************************************************************/
2389 :
2390 : /**
2391 : * \fn VSIVirtualHandle::Eof()
2392 : * \brief Test for end of file.
2393 : *
2394 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
2395 : * previous read operation. The end-of-file flag is cleared by a successful
2396 : * VSIFSeekL() call.
2397 : *
2398 : * This method goes through the VSIFileHandler virtualization and may
2399 : * work on unusual filesystems such as in memory.
2400 : *
2401 : * Analog of the POSIX feof() call.
2402 : *
2403 : * @return TRUE if at EOF else FALSE.
2404 : */
2405 :
2406 : /**
2407 : * \brief Test for end of file.
2408 : *
2409 : * Returns TRUE (non-zero) if an end-of-file condition occurred during the
2410 : * previous read operation. The end-of-file flag is cleared by a successful
2411 : * VSIFSeekL() call.
2412 : *
2413 : * This method goes through the VSIFileHandler virtualization and may
2414 : * work on unusual filesystems such as in memory.
2415 : *
2416 : * Analog of the POSIX feof() call.
2417 : *
2418 : * @param fp file handle opened with VSIFOpenL().
2419 : *
2420 : * @return TRUE if at EOF else FALSE.
2421 : */
2422 :
2423 294393 : int VSIFEofL(VSILFILE *fp)
2424 :
2425 : {
2426 294393 : return fp->Eof();
2427 : }
2428 :
2429 : /************************************************************************/
2430 : /* VSIFTruncateL() */
2431 : /************************************************************************/
2432 :
2433 : /**
2434 : * \fn VSIVirtualHandle::Truncate( vsi_l_offset nNewSize )
2435 : * \brief Truncate/expand the file to the specified size
2436 :
2437 : * This method goes through the VSIFileHandler virtualization and may
2438 : * work on unusual filesystems such as in memory.
2439 : *
2440 : * Analog of the POSIX ftruncate() call.
2441 : *
2442 : * @param nNewSize new size in bytes.
2443 : *
2444 : * @return 0 on success
2445 : * @since GDAL 1.9.0
2446 : */
2447 :
2448 : /**
2449 : * \brief Truncate/expand the file to the specified size
2450 :
2451 : * This method goes through the VSIFileHandler virtualization and may
2452 : * work on unusual filesystems such as in memory.
2453 : *
2454 : * Analog of the POSIX ftruncate() call.
2455 : *
2456 : * @param fp file handle opened with VSIFOpenL().
2457 : * @param nNewSize new size in bytes.
2458 : *
2459 : * @return 0 on success
2460 : * @since GDAL 1.9.0
2461 : */
2462 :
2463 1191 : int VSIFTruncateL(VSILFILE *fp, vsi_l_offset nNewSize)
2464 :
2465 : {
2466 1191 : return fp->Truncate(nNewSize);
2467 : }
2468 :
2469 : /************************************************************************/
2470 : /* VSIFPrintfL() */
2471 : /************************************************************************/
2472 :
2473 : /**
2474 : * \brief Formatted write to file.
2475 : *
2476 : * Provides fprintf() style formatted output to a VSI*L file. This formats
2477 : * an internal buffer which is written using VSIFWriteL().
2478 : *
2479 : * Analog of the POSIX fprintf() call.
2480 : *
2481 : * @param fp file handle opened with VSIFOpenL().
2482 : * @param pszFormat the printf() style format string.
2483 : *
2484 : * @return the number of bytes written or -1 on an error.
2485 : */
2486 :
2487 91532 : int VSIFPrintfL(VSILFILE *fp, CPL_FORMAT_STRING(const char *pszFormat), ...)
2488 :
2489 : {
2490 : va_list args;
2491 :
2492 91532 : va_start(args, pszFormat);
2493 91532 : CPLString osResult;
2494 91532 : osResult.vPrintf(pszFormat, args);
2495 91532 : va_end(args);
2496 :
2497 : return static_cast<int>(
2498 183064 : VSIFWriteL(osResult.c_str(), 1, osResult.length(), fp));
2499 : }
2500 :
2501 : /************************************************************************/
2502 : /* VSIVirtualHandle::Printf() */
2503 : /************************************************************************/
2504 :
2505 : /**
2506 : * \brief Formatted write to file.
2507 : *
2508 : * Provides fprintf() style formatted output to a VSI*L file. This formats
2509 : * an internal buffer which is written using VSIFWriteL().
2510 : *
2511 : * Analog of the POSIX fprintf() call.
2512 : *
2513 : * @param pszFormat the printf() style format string.
2514 : *
2515 : * @return the number of bytes written or -1 on an error.
2516 : */
2517 :
2518 770 : int VSIVirtualHandle::Printf(CPL_FORMAT_STRING(const char *pszFormat), ...)
2519 : {
2520 : va_list args;
2521 :
2522 770 : va_start(args, pszFormat);
2523 770 : CPLString osResult;
2524 770 : osResult.vPrintf(pszFormat, args);
2525 770 : va_end(args);
2526 :
2527 1540 : return static_cast<int>(Write(osResult.c_str(), 1, osResult.length()));
2528 : }
2529 :
2530 : /************************************************************************/
2531 : /* VSIFPutcL() */
2532 : /************************************************************************/
2533 :
2534 : // TODO: should we put in conformance with POSIX regarding the return
2535 : // value. As of today (2015-08-29), no code in GDAL sources actually
2536 : // check the return value.
2537 :
2538 : /**
2539 : * \brief Write a single byte to the file
2540 : *
2541 : * Writes the character nChar, cast to an unsigned char, to file.
2542 : *
2543 : * Almost an analog of the POSIX fputc() call, except that it returns
2544 : * the number of character written (1 or 0), and not the (cast)
2545 : * character itself or EOF.
2546 : *
2547 : * @param nChar character to write.
2548 : * @param fp file handle opened with VSIFOpenL().
2549 : *
2550 : * @return 1 in case of success, 0 on error.
2551 : */
2552 :
2553 2080 : int VSIFPutcL(int nChar, VSILFILE *fp)
2554 :
2555 : {
2556 2080 : const unsigned char cChar = static_cast<unsigned char>(nChar);
2557 2080 : return static_cast<int>(VSIFWriteL(&cChar, 1, 1, fp));
2558 : }
2559 :
2560 : /************************************************************************/
2561 : /* VSIFGetRangeStatusL() */
2562 : /************************************************************************/
2563 :
2564 : /**
2565 : * \fn VSIVirtualHandle::GetRangeStatus( vsi_l_offset nOffset,
2566 : * vsi_l_offset nLength )
2567 : * \brief Return if a given file range contains data or holes filled with zeroes
2568 : *
2569 : * This uses the filesystem capabilities of querying which regions of
2570 : * a sparse file are allocated or not. This is currently only
2571 : * implemented for Linux (and no other Unix derivatives) and Windows.
2572 : *
2573 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
2574 : * extent is filled with zeroes! It must be interpreted as "may
2575 : * contain non-zero data".
2576 : *
2577 : * @param nOffset offset of the start of the extent.
2578 : * @param nLength extent length.
2579 : *
2580 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
2581 : * VSI_RANGE_STATUS_HOLE
2582 : * @since GDAL 2.2
2583 : */
2584 :
2585 : /**
2586 : * \brief Return if a given file range contains data or holes filled with zeroes
2587 : *
2588 : * This uses the filesystem capabilities of querying which regions of
2589 : * a sparse file are allocated or not. This is currently only
2590 : * implemented for Linux (and no other Unix derivatives) and Windows.
2591 : *
2592 : * Note: A return of VSI_RANGE_STATUS_DATA doesn't exclude that the
2593 : * extent is filled with zeroes! It must be interpreted as "may
2594 : * contain non-zero data".
2595 : *
2596 : * @param fp file handle opened with VSIFOpenL().
2597 : * @param nOffset offset of the start of the extent.
2598 : * @param nLength extent length.
2599 : *
2600 : * @return extent status: VSI_RANGE_STATUS_UNKNOWN, VSI_RANGE_STATUS_DATA or
2601 : * VSI_RANGE_STATUS_HOLE
2602 : * @since GDAL 2.2
2603 : */
2604 :
2605 407 : VSIRangeStatus VSIFGetRangeStatusL(VSILFILE *fp, vsi_l_offset nOffset,
2606 : vsi_l_offset nLength)
2607 : {
2608 407 : return fp->GetRangeStatus(nOffset, nLength);
2609 : }
2610 :
2611 : /************************************************************************/
2612 : /* VSIIngestFile() */
2613 : /************************************************************************/
2614 :
2615 : /**
2616 : * \brief Ingest a file into memory.
2617 : *
2618 : * Read the whole content of a file into a memory buffer.
2619 : *
2620 : * Either fp or pszFilename can be NULL, but not both at the same time.
2621 : *
2622 : * If fp is passed non-NULL, it is the responsibility of the caller to
2623 : * close it.
2624 : *
2625 : * If non-NULL, the returned buffer is guaranteed to be NUL-terminated.
2626 : *
2627 : * @param fp file handle opened with VSIFOpenL().
2628 : * @param pszFilename filename.
2629 : * @param ppabyRet pointer to the target buffer. *ppabyRet must be freed with
2630 : * VSIFree()
2631 : * @param pnSize pointer to variable to store the file size. May be NULL.
2632 : * @param nMaxSize maximum size of file allowed. If no limit, set to a negative
2633 : * value.
2634 : *
2635 : * @return TRUE in case of success.
2636 : *
2637 : * @since GDAL 1.11
2638 : */
2639 :
2640 9112 : int VSIIngestFile(VSILFILE *fp, const char *pszFilename, GByte **ppabyRet,
2641 : vsi_l_offset *pnSize, GIntBig nMaxSize)
2642 : {
2643 9112 : if (fp == nullptr && pszFilename == nullptr)
2644 0 : return FALSE;
2645 9112 : if (ppabyRet == nullptr)
2646 0 : return FALSE;
2647 :
2648 9112 : *ppabyRet = nullptr;
2649 9112 : if (pnSize != nullptr)
2650 2731 : *pnSize = 0;
2651 :
2652 9112 : bool bFreeFP = false;
2653 9112 : if (nullptr == fp)
2654 : {
2655 8017 : fp = VSIFOpenL(pszFilename, "rb");
2656 8017 : if (nullptr == fp)
2657 : {
2658 325 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open file '%s'",
2659 : pszFilename);
2660 325 : return FALSE;
2661 : }
2662 7692 : bFreeFP = true;
2663 : }
2664 : else
2665 : {
2666 1095 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
2667 0 : return FALSE;
2668 : }
2669 :
2670 8787 : vsi_l_offset nDataLen = 0;
2671 :
2672 8787 : if (pszFilename == nullptr || strcmp(pszFilename, "/vsistdin/") == 0)
2673 : {
2674 100 : vsi_l_offset nDataAlloc = 0;
2675 100 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
2676 : {
2677 0 : if (bFreeFP)
2678 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2679 0 : return FALSE;
2680 : }
2681 : while (true)
2682 : {
2683 507 : if (nDataLen + 8192 + 1 > nDataAlloc)
2684 : {
2685 233 : nDataAlloc = (nDataAlloc * 4) / 3 + 8192 + 1;
2686 : if (nDataAlloc >
2687 : static_cast<vsi_l_offset>(static_cast<size_t>(nDataAlloc)))
2688 : {
2689 : CPLError(CE_Failure, CPLE_AppDefined,
2690 : "Input file too large to be opened");
2691 : VSIFree(*ppabyRet);
2692 : *ppabyRet = nullptr;
2693 : if (bFreeFP)
2694 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2695 : return FALSE;
2696 : }
2697 : GByte *pabyNew = static_cast<GByte *>(
2698 233 : VSIRealloc(*ppabyRet, static_cast<size_t>(nDataAlloc)));
2699 233 : if (pabyNew == nullptr)
2700 : {
2701 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
2702 : "Cannot allocate " CPL_FRMT_GIB " bytes",
2703 : nDataAlloc);
2704 0 : VSIFree(*ppabyRet);
2705 0 : *ppabyRet = nullptr;
2706 0 : if (bFreeFP)
2707 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2708 0 : return FALSE;
2709 : }
2710 233 : *ppabyRet = pabyNew;
2711 : }
2712 : const int nRead =
2713 507 : static_cast<int>(VSIFReadL(*ppabyRet + nDataLen, 1, 8192, fp));
2714 507 : nDataLen += nRead;
2715 :
2716 507 : if (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize))
2717 : {
2718 0 : CPLError(CE_Failure, CPLE_AppDefined,
2719 : "Input file too large to be opened");
2720 0 : VSIFree(*ppabyRet);
2721 0 : *ppabyRet = nullptr;
2722 0 : if (pnSize != nullptr)
2723 0 : *pnSize = 0;
2724 0 : if (bFreeFP)
2725 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2726 0 : return FALSE;
2727 : }
2728 :
2729 507 : if (pnSize != nullptr)
2730 245 : *pnSize += nRead;
2731 507 : (*ppabyRet)[nDataLen] = '\0';
2732 507 : if (nRead == 0)
2733 100 : break;
2734 507 : }
2735 : }
2736 : else
2737 : {
2738 8687 : if (VSIFSeekL(fp, 0, SEEK_END) != 0)
2739 : {
2740 0 : if (bFreeFP)
2741 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2742 0 : return FALSE;
2743 : }
2744 8687 : nDataLen = VSIFTellL(fp);
2745 :
2746 : // With "large" VSI I/O API we can read data chunks larger than
2747 : // VSIMalloc could allocate. Catch it here.
2748 8687 : if (nDataLen !=
2749 : static_cast<vsi_l_offset>(static_cast<size_t>(nDataLen)) ||
2750 : nDataLen + 1 < nDataLen
2751 : // opening a directory returns nDataLen = INT_MAX (on 32bit) or
2752 : // INT64_MAX (on 64bit)
2753 12342 : || nDataLen + 1 > std::numeric_limits<size_t>::max() / 2 ||
2754 3655 : (nMaxSize >= 0 && nDataLen > static_cast<vsi_l_offset>(nMaxSize)))
2755 : {
2756 0 : CPLError(CE_Failure, CPLE_AppDefined,
2757 : "Input file too large to be opened");
2758 0 : if (bFreeFP)
2759 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2760 0 : return FALSE;
2761 : }
2762 :
2763 8687 : if (VSIFSeekL(fp, 0, SEEK_SET) != 0)
2764 : {
2765 0 : if (bFreeFP)
2766 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2767 0 : return FALSE;
2768 : }
2769 :
2770 8687 : *ppabyRet =
2771 8687 : static_cast<GByte *>(VSIMalloc(static_cast<size_t>(nDataLen + 1)));
2772 8687 : if (nullptr == *ppabyRet)
2773 : {
2774 0 : CPLError(CE_Failure, CPLE_OutOfMemory,
2775 : "Cannot allocate " CPL_FRMT_GIB " bytes", nDataLen + 1);
2776 0 : if (bFreeFP)
2777 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2778 0 : return FALSE;
2779 : }
2780 :
2781 8687 : (*ppabyRet)[nDataLen] = '\0';
2782 8687 : if (nDataLen !=
2783 8687 : VSIFReadL(*ppabyRet, 1, static_cast<size_t>(nDataLen), fp))
2784 : {
2785 0 : CPLError(CE_Failure, CPLE_FileIO,
2786 : "Cannot read " CPL_FRMT_GIB " bytes", nDataLen);
2787 0 : VSIFree(*ppabyRet);
2788 0 : *ppabyRet = nullptr;
2789 0 : if (bFreeFP)
2790 0 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2791 0 : return FALSE;
2792 : }
2793 8687 : if (pnSize != nullptr)
2794 2375 : *pnSize = nDataLen;
2795 : }
2796 8787 : if (bFreeFP)
2797 7692 : CPL_IGNORE_RET_VAL(VSIFCloseL(fp));
2798 8787 : return TRUE;
2799 : }
2800 :
2801 : /************************************************************************/
2802 : /* VSIOverwriteFile() */
2803 : /************************************************************************/
2804 :
2805 : /**
2806 : * \brief Overwrite an existing file with content from another one
2807 : *
2808 : * @param fpTarget file handle opened with VSIFOpenL() with "rb+" flag.
2809 : * @param pszSourceFilename source filename
2810 : *
2811 : * @return TRUE in case of success.
2812 : *
2813 : * @since GDAL 3.1
2814 : */
2815 :
2816 4 : int VSIOverwriteFile(VSILFILE *fpTarget, const char *pszSourceFilename)
2817 : {
2818 4 : VSILFILE *fpSource = VSIFOpenL(pszSourceFilename, "rb");
2819 4 : if (fpSource == nullptr)
2820 : {
2821 0 : CPLError(CE_Failure, CPLE_FileIO, "Cannot open %s", pszSourceFilename);
2822 0 : return false;
2823 : }
2824 :
2825 4 : const size_t nBufferSize = 4096;
2826 4 : void *pBuffer = CPLMalloc(nBufferSize);
2827 4 : VSIFSeekL(fpTarget, 0, SEEK_SET);
2828 4 : bool bRet = true;
2829 : while (true)
2830 : {
2831 4 : size_t nRead = VSIFReadL(pBuffer, 1, nBufferSize, fpSource);
2832 4 : size_t nWritten = VSIFWriteL(pBuffer, 1, nRead, fpTarget);
2833 4 : if (nWritten != nRead)
2834 : {
2835 0 : bRet = false;
2836 0 : break;
2837 : }
2838 4 : if (nRead < nBufferSize)
2839 4 : break;
2840 0 : }
2841 :
2842 4 : if (bRet)
2843 : {
2844 4 : bRet = VSIFTruncateL(fpTarget, VSIFTellL(fpTarget)) == 0;
2845 4 : if (!bRet)
2846 : {
2847 0 : CPLError(CE_Failure, CPLE_FileIO, "Truncation failed");
2848 : }
2849 : }
2850 :
2851 4 : CPLFree(pBuffer);
2852 4 : VSIFCloseL(fpSource);
2853 4 : return bRet;
2854 : }
2855 :
2856 : /************************************************************************/
2857 : /* VSIFGetNativeFileDescriptorL() */
2858 : /************************************************************************/
2859 :
2860 : /**
2861 : * \fn VSIVirtualHandle::GetNativeFileDescriptor()
2862 : * \brief Returns the "native" file descriptor for the virtual handle.
2863 : *
2864 : * This will only return a non-NULL value for "real" files handled by the
2865 : * operating system (to be opposed to GDAL virtual file systems).
2866 : *
2867 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
2868 : * On Windows systems, this will be the HANDLE.
2869 : *
2870 : * @return the native file descriptor, or NULL.
2871 : */
2872 :
2873 : /**
2874 : * \brief Returns the "native" file descriptor for the virtual handle.
2875 : *
2876 : * This will only return a non-NULL value for "real" files handled by the
2877 : * operating system (to be opposed to GDAL virtual file systems).
2878 : *
2879 : * On POSIX systems, this will be a integer value ("fd") cast as a void*.
2880 : * On Windows systems, this will be the HANDLE.
2881 : *
2882 : * @param fp file handle opened with VSIFOpenL().
2883 : *
2884 : * @return the native file descriptor, or NULL.
2885 : */
2886 :
2887 64 : void *VSIFGetNativeFileDescriptorL(VSILFILE *fp)
2888 : {
2889 64 : return fp->GetNativeFileDescriptor();
2890 : }
2891 :
2892 : /************************************************************************/
2893 : /* VSIGetDiskFreeSpace() */
2894 : /************************************************************************/
2895 :
2896 : /**
2897 : * \brief Return free disk space available on the filesystem.
2898 : *
2899 : * This function returns the free disk space available on the filesystem.
2900 : *
2901 : * @param pszDirname a directory of the filesystem to query.
2902 : * @return The free space in bytes. Or -1 in case of error.
2903 : * @since GDAL 2.1
2904 : */
2905 :
2906 73 : GIntBig VSIGetDiskFreeSpace(const char *pszDirname)
2907 : {
2908 73 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszDirname);
2909 :
2910 73 : return poFSHandler->GetDiskFreeSpace(pszDirname);
2911 : }
2912 :
2913 : /************************************************************************/
2914 : /* VSIGetFileSystemsPrefixes() */
2915 : /************************************************************************/
2916 :
2917 : /**
2918 : * \brief Return the list of prefixes for virtual file system handlers
2919 : * currently registered.
2920 : *
2921 : * Typically: "", "/vsimem/", "/vsicurl/", etc
2922 : *
2923 : * @return a NULL terminated list of prefixes. Must be freed with CSLDestroy()
2924 : * @since GDAL 2.3
2925 : */
2926 :
2927 2 : char **VSIGetFileSystemsPrefixes(void)
2928 : {
2929 2 : return VSIFileManager::GetPrefixes();
2930 : }
2931 :
2932 : /************************************************************************/
2933 : /* VSIGetFileSystemOptions() */
2934 : /************************************************************************/
2935 :
2936 : /**
2937 : * \brief Return the list of options associated with a virtual file system
2938 : * handler as a serialized XML string.
2939 : *
2940 : * Those options may be set as configuration options with CPLSetConfigOption().
2941 : *
2942 : * @param pszFilename a filename, or prefix of a virtual file system handler.
2943 : * @return a string, which must not be freed, or NULL if no options is declared.
2944 : * @since GDAL 2.3
2945 : */
2946 :
2947 33 : const char *VSIGetFileSystemOptions(const char *pszFilename)
2948 : {
2949 33 : VSIFilesystemHandler *poFSHandler = VSIFileManager::GetHandler(pszFilename);
2950 :
2951 33 : return poFSHandler->GetOptions();
2952 : }
2953 :
2954 : /************************************************************************/
2955 : /* VSISetPathSpecificOption() */
2956 : /************************************************************************/
2957 :
2958 : static std::mutex oMutexPathSpecificOptions;
2959 :
2960 : // key is a path prefix
2961 : // value is a map of key, value pair
2962 : static std::map<std::string, std::map<std::string, std::string>>
2963 : oMapPathSpecificOptions;
2964 :
2965 : /**
2966 : * \brief Set a credential (or more generally an option related to a
2967 : * virtual file system) for a given path prefix.
2968 : * @deprecated in GDAL 3.6 for the better named VSISetPathSpecificOption()
2969 : * @see VSISetPathSpecificOption()
2970 : */
2971 0 : void VSISetCredential(const char *pszPathPrefix, const char *pszKey,
2972 : const char *pszValue)
2973 : {
2974 0 : VSISetPathSpecificOption(pszPathPrefix, pszKey, pszValue);
2975 0 : }
2976 :
2977 : /**
2978 : * \brief Set a path specific option for a given path prefix.
2979 : *
2980 : * Such option is typically, but not limited to, a credential setting for a
2981 : * virtual file system.
2982 : *
2983 : * That option may also be set as a configuration option with
2984 : * CPLSetConfigOption(), but this function allows to specify them with a
2985 : * granularity at the level of a file path, which makes it easier if using the
2986 : * same virtual file system but with different credentials (e.g. different
2987 : * credentials for bucket "/vsis3/foo" and "/vsis3/bar")
2988 : *
2989 : * This is supported for the following virtual file systems:
2990 : * /vsis3/, /vsigs/, /vsiaz/, /vsioss/, /vsiwebhdfs, /vsiswift.
2991 : * Note: setting them for a path starting with /vsiXXX/ will also apply for
2992 : * /vsiXXX_streaming/ requests.
2993 : *
2994 : * Note that no particular care is taken to store them in RAM in a secure way.
2995 : * So they might accidentally hit persistent storage if swapping occurs, or
2996 : * someone with access to the memory allocated by the process may be able to
2997 : * read them.
2998 : *
2999 : * @param pszPathPrefix a path prefix of a virtual file system handler.
3000 : * Typically of the form "/vsiXXX/bucket". Must NOT be
3001 : * NULL.
3002 : * @param pszKey Option name. Must NOT be NULL.
3003 : * @param pszValue Option value. May be NULL to erase it.
3004 : *
3005 : * @since GDAL 3.6
3006 : */
3007 :
3008 77 : void VSISetPathSpecificOption(const char *pszPathPrefix, const char *pszKey,
3009 : const char *pszValue)
3010 : {
3011 154 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
3012 77 : auto oIter = oMapPathSpecificOptions.find(pszPathPrefix);
3013 154 : CPLString osKey(pszKey);
3014 77 : osKey.toupper();
3015 77 : if (oIter == oMapPathSpecificOptions.end())
3016 : {
3017 18 : if (pszValue != nullptr)
3018 18 : oMapPathSpecificOptions[pszPathPrefix][osKey] = pszValue;
3019 : }
3020 59 : else if (pszValue != nullptr)
3021 56 : oIter->second[osKey] = pszValue;
3022 : else
3023 3 : oIter->second.erase(osKey);
3024 77 : }
3025 :
3026 : /************************************************************************/
3027 : /* VSIClearPathSpecificOptions() */
3028 : /************************************************************************/
3029 :
3030 : /**
3031 : * \brief Clear path specific options set with VSISetPathSpecificOption()
3032 : * @deprecated in GDAL 3.6 for the better named VSIClearPathSpecificOptions()
3033 : * @see VSIClearPathSpecificOptions()
3034 : */
3035 0 : void VSIClearCredentials(const char *pszPathPrefix)
3036 : {
3037 0 : return VSIClearPathSpecificOptions(pszPathPrefix);
3038 : }
3039 :
3040 : /**
3041 : * \brief Clear path specific options set with VSISetPathSpecificOption()
3042 : *
3043 : * Note that no particular care is taken to remove them from RAM in a secure
3044 : * way.
3045 : *
3046 : * @param pszPathPrefix If set to NULL, all path specific options are cleared.
3047 : * If set to not-NULL, only those set with
3048 : * VSISetPathSpecificOption(pszPathPrefix, ...) will be
3049 : * cleared.
3050 : *
3051 : * @since GDAL 3.6
3052 : */
3053 15 : void VSIClearPathSpecificOptions(const char *pszPathPrefix)
3054 : {
3055 30 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
3056 15 : if (pszPathPrefix == nullptr)
3057 : {
3058 2 : oMapPathSpecificOptions.clear();
3059 : }
3060 : else
3061 : {
3062 13 : oMapPathSpecificOptions.erase(pszPathPrefix);
3063 : }
3064 15 : }
3065 :
3066 : /************************************************************************/
3067 : /* VSIGetPathSpecificOption() */
3068 : /************************************************************************/
3069 :
3070 : /**
3071 : * \brief Get the value of a credential (or more generally an option related to
3072 : * a virtual file system) for a given path.
3073 : * @deprecated in GDAL 3.6 for the better named VSIGetPathSpecificOption()
3074 : * @see VSIGetPathSpecificOption()
3075 : */
3076 0 : const char *VSIGetCredential(const char *pszPath, const char *pszKey,
3077 : const char *pszDefault)
3078 : {
3079 0 : return VSIGetPathSpecificOption(pszPath, pszKey, pszDefault);
3080 : }
3081 :
3082 : /**
3083 : * \brief Get the value a path specific option.
3084 : *
3085 : * Such option is typically, but not limited to, a credential setting for a
3086 : * virtual file system.
3087 : *
3088 : * If no match occurs, CPLGetConfigOption(pszKey, pszDefault) is returned.
3089 : *
3090 : * Mostly to be used by virtual file system implementations.
3091 : *
3092 : * @since GDAL 3.6
3093 : * @see VSISetPathSpecificOption()
3094 : */
3095 111309 : const char *VSIGetPathSpecificOption(const char *pszPath, const char *pszKey,
3096 : const char *pszDefault)
3097 : {
3098 : {
3099 111309 : std::lock_guard<std::mutex> oLock(oMutexPathSpecificOptions);
3100 126088 : for (auto it = oMapPathSpecificOptions.rbegin();
3101 140867 : it != oMapPathSpecificOptions.rend(); ++it)
3102 : {
3103 15026 : if (STARTS_WITH(pszPath, it->first.c_str()))
3104 : {
3105 1519 : auto oIter = it->second.find(CPLString(pszKey).toupper());
3106 1519 : if (oIter != it->second.end())
3107 247 : return oIter->second.c_str();
3108 : }
3109 : }
3110 : }
3111 111062 : return CPLGetConfigOption(pszKey, pszDefault);
3112 : }
3113 :
3114 : /************************************************************************/
3115 : /* VSIDuplicateFileSystemHandler() */
3116 : /************************************************************************/
3117 :
3118 : /**
3119 : * \brief Duplicate an existing file system handler.
3120 : *
3121 : * A number of virtual file system for remote object stores use protocols
3122 : * identical or close to popular ones (typically AWS S3), but with slightly
3123 : * different settings (at the very least the endpoint).
3124 : *
3125 : * This functions allows to duplicate the source virtual file system handler
3126 : * as a new one with a different prefix (when the source virtual file system
3127 : * handler supports the duplication operation).
3128 : *
3129 : * VSISetPathSpecificOption() will typically be called afterwards to change
3130 : * configurable settings on the cloned file system handler (e.g. AWS_S3_ENDPOINT
3131 : * for a clone of /vsis3/).
3132 : *
3133 : * @since GDAL 3.7
3134 : */
3135 4 : bool VSIDuplicateFileSystemHandler(const char *pszSourceFSName,
3136 : const char *pszNewFSName)
3137 : {
3138 : VSIFilesystemHandler *poTargetFSHandler =
3139 4 : VSIFileManager::GetHandler(pszNewFSName);
3140 4 : if (poTargetFSHandler != VSIFileManager::GetHandler("/"))
3141 : {
3142 1 : CPLError(CE_Failure, CPLE_AppDefined,
3143 : "%s is already a known virtual file system", pszNewFSName);
3144 1 : return false;
3145 : }
3146 :
3147 : VSIFilesystemHandler *poSourceFSHandler =
3148 3 : VSIFileManager::GetHandler(pszSourceFSName);
3149 3 : if (!poSourceFSHandler)
3150 : {
3151 0 : CPLError(CE_Failure, CPLE_AppDefined,
3152 : "%s is not a known virtual file system", pszSourceFSName);
3153 0 : return false;
3154 : }
3155 :
3156 3 : poTargetFSHandler = poSourceFSHandler->Duplicate(pszNewFSName);
3157 3 : if (!poTargetFSHandler)
3158 2 : return false;
3159 :
3160 1 : VSIFileManager::InstallHandler(pszNewFSName, poTargetFSHandler);
3161 1 : return true;
3162 : }
3163 :
3164 : /************************************************************************/
3165 : /* ==================================================================== */
3166 : /* VSIFileManager() */
3167 : /* ==================================================================== */
3168 : /************************************************************************/
3169 :
3170 : #ifndef DOXYGEN_SKIP
3171 :
3172 : /*
3173 : ** Notes on Multithreading:
3174 : **
3175 : ** The VSIFileManager maintains a list of file type handlers (mem, large
3176 : ** file, etc). It should be thread safe.
3177 : **/
3178 :
3179 : /************************************************************************/
3180 : /* VSIFileManager() */
3181 : /************************************************************************/
3182 :
3183 1225 : VSIFileManager::VSIFileManager() : poDefaultHandler(nullptr)
3184 : {
3185 1225 : }
3186 :
3187 : /************************************************************************/
3188 : /* ~VSIFileManager() */
3189 : /************************************************************************/
3190 :
3191 849 : VSIFileManager::~VSIFileManager()
3192 : {
3193 1698 : std::set<VSIFilesystemHandler *> oSetAlreadyDeleted;
3194 23774 : for (std::map<std::string, VSIFilesystemHandler *>::const_iterator iter =
3195 849 : oHandlers.begin();
3196 48397 : iter != oHandlers.end(); ++iter)
3197 : {
3198 23774 : if (oSetAlreadyDeleted.find(iter->second) == oSetAlreadyDeleted.end())
3199 : {
3200 22076 : oSetAlreadyDeleted.insert(iter->second);
3201 22076 : delete iter->second;
3202 : }
3203 : }
3204 :
3205 849 : delete poDefaultHandler;
3206 849 : }
3207 :
3208 : /************************************************************************/
3209 : /* Get() */
3210 : /************************************************************************/
3211 :
3212 : static VSIFileManager *poManager = nullptr;
3213 : static CPLMutex *hVSIFileManagerMutex = nullptr;
3214 :
3215 2281210 : VSIFileManager *VSIFileManager::Get()
3216 : {
3217 4562560 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
3218 2281360 : if (poManager != nullptr)
3219 : {
3220 2280130 : return poManager;
3221 : }
3222 :
3223 1225 : poManager = new VSIFileManager;
3224 1225 : VSIInstallLargeFileHandler();
3225 1225 : VSIInstallSubFileHandler();
3226 1225 : VSIInstallMemFileHandler();
3227 : #ifdef HAVE_LIBZ
3228 1225 : VSIInstallGZipFileHandler();
3229 1225 : VSIInstallZipFileHandler();
3230 : #endif
3231 : #ifdef HAVE_LIBARCHIVE
3232 : VSIInstall7zFileHandler();
3233 : VSIInstallRarFileHandler();
3234 : #endif
3235 : #ifdef HAVE_CURL
3236 1225 : VSIInstallCurlFileHandler();
3237 1225 : VSIInstallCurlStreamingFileHandler();
3238 1225 : VSIInstallS3FileHandler();
3239 1225 : VSIInstallS3StreamingFileHandler();
3240 1225 : VSIInstallGSFileHandler();
3241 1225 : VSIInstallGSStreamingFileHandler();
3242 1225 : VSIInstallAzureFileHandler();
3243 1225 : VSIInstallAzureStreamingFileHandler();
3244 1225 : VSIInstallADLSFileHandler();
3245 1225 : VSIInstallOSSFileHandler();
3246 1225 : VSIInstallOSSStreamingFileHandler();
3247 1225 : VSIInstallSwiftFileHandler();
3248 1225 : VSIInstallSwiftStreamingFileHandler();
3249 1225 : VSIInstallWebHdfsHandler();
3250 : #endif
3251 1225 : VSIInstallStdinHandler();
3252 1225 : VSIInstallHdfsHandler();
3253 1225 : VSIInstallStdoutHandler();
3254 1225 : VSIInstallSparseFileHandler();
3255 1225 : VSIInstallTarFileHandler();
3256 1225 : VSIInstallCachedFileHandler();
3257 1225 : VSIInstallCryptFileHandler();
3258 :
3259 1225 : return poManager;
3260 : }
3261 :
3262 : /************************************************************************/
3263 : /* GetPrefixes() */
3264 : /************************************************************************/
3265 :
3266 494 : char **VSIFileManager::GetPrefixes()
3267 : {
3268 988 : CPLMutexHolder oHolder(&hVSIFileManagerMutex);
3269 988 : CPLStringList aosList;
3270 14326 : for (const auto &oIter : Get()->oHandlers)
3271 : {
3272 13832 : if (oIter.first != "/vsicurl?")
3273 : {
3274 13338 : aosList.AddString(oIter.first.c_str());
3275 : }
3276 : }
3277 988 : return aosList.StealList();
3278 : }
3279 :
3280 : /************************************************************************/
3281 : /* GetHandler() */
3282 : /************************************************************************/
3283 :
3284 2245250 : VSIFilesystemHandler *VSIFileManager::GetHandler(const char *pszPath)
3285 :
3286 : {
3287 2245250 : VSIFileManager *poThis = Get();
3288 2245340 : const size_t nPathLen = strlen(pszPath);
3289 :
3290 45380700 : for (std::map<std::string, VSIFilesystemHandler *>::const_iterator iter =
3291 2245340 : poThis->oHandlers.begin();
3292 93006000 : iter != poThis->oHandlers.end(); ++iter)
3293 : {
3294 46877700 : const char *pszIterKey = iter->first.c_str();
3295 46885100 : const size_t nIterKeyLen = iter->first.size();
3296 46892700 : if (strncmp(pszPath, pszIterKey, nIterKeyLen) == 0)
3297 1512760 : return iter->second;
3298 :
3299 : // "/vsimem\foo" should be handled as "/vsimem/foo".
3300 45476000 : if (nIterKeyLen && nPathLen > nIterKeyLen &&
3301 40950100 : pszIterKey[nIterKeyLen - 1] == '/' &&
3302 35733400 : pszPath[nIterKeyLen - 1] == '\\' &&
3303 27 : strncmp(pszPath, pszIterKey, nIterKeyLen - 1) == 0)
3304 0 : return iter->second;
3305 :
3306 : // /vsimem should be treated as a match for /vsimem/.
3307 45476000 : if (nPathLen + 1 == nIterKeyLen &&
3308 467908 : strncmp(pszPath, pszIterKey, nPathLen) == 0)
3309 96041 : return iter->second;
3310 : }
3311 :
3312 732492 : return poThis->poDefaultHandler;
3313 : }
3314 :
3315 : /************************************************************************/
3316 : /* InstallHandler() */
3317 : /************************************************************************/
3318 :
3319 35517 : void VSIFileManager::InstallHandler(const std::string &osPrefix,
3320 : VSIFilesystemHandler *poHandler)
3321 :
3322 : {
3323 35517 : if (osPrefix == "")
3324 1225 : Get()->poDefaultHandler = poHandler;
3325 : else
3326 34292 : Get()->oHandlers[osPrefix] = poHandler;
3327 35517 : }
3328 :
3329 : /************************************************************************/
3330 : /* RemoveHandler() */
3331 : /************************************************************************/
3332 :
3333 3 : void VSIFileManager::RemoveHandler(const std::string &osPrefix)
3334 : {
3335 3 : if (osPrefix == "")
3336 0 : Get()->poDefaultHandler = nullptr;
3337 : else
3338 3 : Get()->oHandlers.erase(osPrefix);
3339 3 : }
3340 :
3341 : /************************************************************************/
3342 : /* VSICleanupFileManager() */
3343 : /************************************************************************/
3344 :
3345 849 : void VSICleanupFileManager()
3346 :
3347 : {
3348 849 : if (poManager)
3349 : {
3350 849 : delete poManager;
3351 849 : poManager = nullptr;
3352 : }
3353 :
3354 849 : if (hVSIFileManagerMutex != nullptr)
3355 : {
3356 849 : CPLDestroyMutex(hVSIFileManagerMutex);
3357 849 : hVSIFileManagerMutex = nullptr;
3358 : }
3359 :
3360 : #ifdef HAVE_CURL
3361 849 : VSICURLDestroyCacheFileProp();
3362 : #endif
3363 849 : }
3364 :
3365 : /************************************************************************/
3366 : /* Truncate() */
3367 : /************************************************************************/
3368 :
3369 2 : int VSIVirtualHandle::Truncate(vsi_l_offset nNewSize)
3370 : {
3371 2 : const vsi_l_offset nOriginalPos = Tell();
3372 2 : if (Seek(0, SEEK_END) == 0 && nNewSize >= Tell())
3373 : {
3374 : // Fill with zeroes
3375 2 : std::vector<GByte> aoBytes(4096, 0);
3376 1 : vsi_l_offset nCurOffset = nOriginalPos;
3377 3 : while (nCurOffset < nNewSize)
3378 : {
3379 2 : constexpr vsi_l_offset nMaxOffset = 4096;
3380 : const int nSize =
3381 2 : static_cast<int>(std::min(nMaxOffset, nNewSize - nCurOffset));
3382 2 : if (Write(&aoBytes[0], nSize, 1) != 1)
3383 : {
3384 0 : Seek(nOriginalPos, SEEK_SET);
3385 0 : return -1;
3386 : }
3387 2 : nCurOffset += nSize;
3388 : }
3389 1 : return Seek(nOriginalPos, SEEK_SET) == 0 ? 0 : -1;
3390 : }
3391 :
3392 1 : CPLDebug("VSI", "Truncation is not supported in generic implementation "
3393 : "of Truncate()");
3394 1 : Seek(nOriginalPos, SEEK_SET);
3395 1 : return -1;
3396 : }
3397 :
3398 : /************************************************************************/
3399 : /* ReadMultiRange() */
3400 : /************************************************************************/
3401 :
3402 602 : int VSIVirtualHandle::ReadMultiRange(int nRanges, void **ppData,
3403 : const vsi_l_offset *panOffsets,
3404 : const size_t *panSizes)
3405 : {
3406 602 : int nRet = 0;
3407 602 : const vsi_l_offset nCurOffset = Tell();
3408 46073 : for (int i = 0; i < nRanges; i++)
3409 : {
3410 45492 : if (Seek(panOffsets[i], SEEK_SET) < 0)
3411 : {
3412 0 : nRet = -1;
3413 0 : break;
3414 : }
3415 :
3416 45492 : const size_t nRead = Read(ppData[i], 1, panSizes[i]);
3417 45492 : if (panSizes[i] != nRead)
3418 : {
3419 21 : nRet = -1;
3420 21 : break;
3421 : }
3422 : }
3423 :
3424 602 : Seek(nCurOffset, SEEK_SET);
3425 :
3426 602 : return nRet;
3427 : }
3428 :
3429 : #endif // #ifndef DOXYGEN_SKIP
3430 :
3431 : /************************************************************************/
3432 : /* HasPRead() */
3433 : /************************************************************************/
3434 :
3435 : /** Returns whether this file handle supports the PRead() method.
3436 : *
3437 : * @since GDAL 3.6
3438 : */
3439 0 : bool VSIVirtualHandle::HasPRead() const
3440 : {
3441 0 : return false;
3442 : }
3443 :
3444 : /************************************************************************/
3445 : /* PRead() */
3446 : /************************************************************************/
3447 :
3448 : /** Do a parallel-compatible read operation.
3449 : *
3450 : * This methods reads into pBuffer up to nSize bytes starting at offset nOffset
3451 : * in the file. The current file offset is not affected by this method.
3452 : *
3453 : * The implementation is thread-safe: several threads can issue PRead()
3454 : * concurrently on the same VSIVirtualHandle object.
3455 : *
3456 : * This method has the same semantics as pread() Linux operation. It is only
3457 : * available if HasPRead() returns true.
3458 : *
3459 : * @param pBuffer output buffer (must be at least nSize bytes large).
3460 : * @param nSize number of bytes to read in the file.
3461 : * @param nOffset file offset from which to read.
3462 : * @return number of bytes read.
3463 : * @since GDAL 3.6
3464 : */
3465 0 : size_t VSIVirtualHandle::PRead(CPL_UNUSED void *pBuffer,
3466 : CPL_UNUSED size_t nSize,
3467 : CPL_UNUSED vsi_l_offset nOffset) const
3468 : {
3469 0 : return 0;
3470 : }
|
