Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: VSI Virtual File System
4 : * Purpose: Implementation of buffered reader IO functions.
5 : * Author: Even Rouault, even.rouault at spatialys.com
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2010-2011, Even Rouault <even dot rouault at spatialys.com>
9 : *
10 : * Permission is hereby granted, free of charge, to any person obtaining a
11 : * copy of this software and associated documentation files (the "Software"),
12 : * to deal in the Software without restriction, including without limitation
13 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 : * and/or sell copies of the Software, and to permit persons to whom the
15 : * Software is furnished to do so, subject to the following conditions:
16 : *
17 : * The above copyright notice and this permission notice shall be included
18 : * in all copies or substantial portions of the Software.
19 : *
20 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 : * DEALINGS IN THE SOFTWARE.
27 : ****************************************************************************/
28 :
29 : //! @cond Doxygen_Suppress
30 :
31 : // The intent of this class is to be a wrapper around an underlying virtual
32 : // handle and add very basic caching of last read bytes, so that a backward
33 : // seek of a few bytes doesn't require a seek on the underlying virtual handle.
34 : // This enable us to improve dramatically the performance of CPLReadLine2L() on
35 : // a gzip file.
36 :
37 : #include "cpl_port.h"
38 : #include "cpl_vsi_virtual.h"
39 :
40 : #include <cstddef>
41 : #include <cstring>
42 : #if HAVE_FCNTL_H
43 : #include <fcntl.h>
44 : #endif
45 :
46 : #include <algorithm>
47 : #include <vector>
48 :
49 : #include "cpl_conv.h"
50 : #include "cpl_error.h"
51 : #include "cpl_vsi.h"
52 :
53 : constexpr int MAX_BUFFER_SIZE = 65536;
54 :
55 : class VSIBufferedReaderHandle final : public VSIVirtualHandle
56 : {
57 : CPL_DISALLOW_COPY_ASSIGN(VSIBufferedReaderHandle)
58 :
59 : VSIVirtualHandle *m_poBaseHandle = nullptr;
60 : GByte *pabyBuffer = nullptr;
61 : GUIntBig nBufferOffset = 0;
62 : int nBufferSize = 0;
63 : GUIntBig nCurOffset = 0;
64 : bool bNeedBaseHandleSeek = false;
65 : bool bEOF = false;
66 : vsi_l_offset nCheatFileSize = 0;
67 :
68 : int SeekBaseTo(vsi_l_offset nTargetOffset);
69 :
70 : public:
71 : explicit VSIBufferedReaderHandle(VSIVirtualHandle *poBaseHandle);
72 : VSIBufferedReaderHandle(VSIVirtualHandle *poBaseHandle,
73 : const GByte *pabyBeginningContent,
74 : vsi_l_offset nCheatFileSizeIn);
75 : // TODO(schwehr): Add override when support dropped for VS2008.
76 : ~VSIBufferedReaderHandle() override;
77 :
78 : int Seek(vsi_l_offset nOffset, int nWhence) override;
79 : vsi_l_offset Tell() override;
80 : size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
81 : size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
82 : int Eof() override;
83 : int Flush() override;
84 : int Close() override;
85 : };
86 :
87 : //! @endcond
88 :
89 : /************************************************************************/
90 : /* VSICreateBufferedReaderHandle() */
91 : /************************************************************************/
92 :
93 4914 : VSIVirtualHandle *VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle)
94 : {
95 4914 : return new VSIBufferedReaderHandle(poBaseHandle);
96 : }
97 :
98 : VSIVirtualHandle *
99 7 : VSICreateBufferedReaderHandle(VSIVirtualHandle *poBaseHandle,
100 : const GByte *pabyBeginningContent,
101 : vsi_l_offset nCheatFileSizeIn)
102 : {
103 : return new VSIBufferedReaderHandle(poBaseHandle, pabyBeginningContent,
104 7 : nCheatFileSizeIn);
105 : }
106 :
107 : //! @cond Doxygen_Suppress
108 :
109 : /************************************************************************/
110 : /* VSIBufferedReaderHandle() */
111 : /************************************************************************/
112 :
113 4914 : VSIBufferedReaderHandle::VSIBufferedReaderHandle(VSIVirtualHandle *poBaseHandle)
114 : : m_poBaseHandle(poBaseHandle),
115 4914 : pabyBuffer(static_cast<GByte *>(CPLMalloc(MAX_BUFFER_SIZE)))
116 : {
117 4914 : }
118 :
119 7 : VSIBufferedReaderHandle::VSIBufferedReaderHandle(
120 : VSIVirtualHandle *poBaseHandle, const GByte *pabyBeginningContent,
121 7 : vsi_l_offset nCheatFileSizeIn)
122 : : m_poBaseHandle(poBaseHandle),
123 14 : pabyBuffer(static_cast<GByte *>(CPLMalloc(
124 7 : std::max(MAX_BUFFER_SIZE, static_cast<int>(poBaseHandle->Tell()))))),
125 14 : nBufferOffset(0), nBufferSize(static_cast<int>(poBaseHandle->Tell())),
126 : nCurOffset(0), bNeedBaseHandleSeek(true), bEOF(false),
127 14 : nCheatFileSize(nCheatFileSizeIn)
128 : {
129 7 : memcpy(pabyBuffer, pabyBeginningContent, nBufferSize);
130 7 : }
131 :
132 : /************************************************************************/
133 : /* ~VSIBufferedReaderHandle() */
134 : /************************************************************************/
135 :
136 9842 : VSIBufferedReaderHandle::~VSIBufferedReaderHandle()
137 : {
138 4921 : delete m_poBaseHandle;
139 4921 : CPLFree(pabyBuffer);
140 9842 : }
141 :
142 : /************************************************************************/
143 : /* Seek() */
144 : /************************************************************************/
145 :
146 218306 : int VSIBufferedReaderHandle::Seek(vsi_l_offset nOffset, int nWhence)
147 : {
148 : #ifdef DEBUG_VERBOSE
149 : CPLDebug("BUFFERED", "Seek(%d,%d)", static_cast<int>(nOffset),
150 : static_cast<int>(nWhence));
151 : #endif
152 218306 : bEOF = false;
153 218306 : int ret = 0;
154 218306 : if (nWhence == SEEK_CUR)
155 : {
156 825 : nCurOffset += nOffset;
157 : }
158 217481 : else if (nWhence == SEEK_END)
159 : {
160 1928 : if (nCheatFileSize)
161 : {
162 2 : nCurOffset = nCheatFileSize;
163 : }
164 : else
165 : {
166 1926 : ret = m_poBaseHandle->Seek(nOffset, nWhence);
167 1926 : nCurOffset = m_poBaseHandle->Tell();
168 1926 : bNeedBaseHandleSeek = true;
169 : }
170 : }
171 : else
172 : {
173 215553 : nCurOffset = nOffset;
174 : }
175 :
176 218306 : return ret;
177 : }
178 :
179 : /************************************************************************/
180 : /* Tell() */
181 : /************************************************************************/
182 :
183 109435 : vsi_l_offset VSIBufferedReaderHandle::Tell()
184 : {
185 : #ifdef DEBUG_VERBOSE
186 : CPLDebug("BUFFERED", "Tell() = %d", static_cast<int>(nCurOffset));
187 : #endif
188 109435 : return nCurOffset;
189 : }
190 :
191 : /************************************************************************/
192 : /* SeekBaseTo() */
193 : /************************************************************************/
194 :
195 27313 : int VSIBufferedReaderHandle::SeekBaseTo(vsi_l_offset nTargetOffset)
196 : {
197 27313 : if (m_poBaseHandle->Seek(nTargetOffset, SEEK_SET) == 0)
198 27313 : return TRUE;
199 :
200 0 : nCurOffset = m_poBaseHandle->Tell();
201 0 : if (nCurOffset > nTargetOffset)
202 0 : return FALSE;
203 :
204 0 : const vsi_l_offset nMaxOffset = 8192;
205 :
206 0 : std::vector<char> oTemp(nMaxOffset, 0);
207 0 : char *pabyTemp = &oTemp[0];
208 :
209 : while (true)
210 : {
211 : const size_t nToRead = static_cast<size_t>(
212 0 : std::min(nMaxOffset, nTargetOffset - nCurOffset));
213 0 : const size_t nRead = m_poBaseHandle->Read(pabyTemp, 1, nToRead);
214 :
215 0 : nCurOffset += nRead;
216 :
217 0 : if (nRead < nToRead)
218 : {
219 0 : bEOF = true;
220 0 : return FALSE;
221 : }
222 0 : if (nToRead < nMaxOffset)
223 0 : break;
224 0 : }
225 0 : return TRUE;
226 : }
227 :
228 : /************************************************************************/
229 : /* Read() */
230 : /************************************************************************/
231 :
232 289605 : size_t VSIBufferedReaderHandle::Read(void *pBuffer, size_t nSize, size_t nMemb)
233 : {
234 289605 : const size_t nTotalToRead = nSize * nMemb;
235 : #ifdef DEBUG_VERBOSE
236 : CPLDebug("BUFFERED", "Read(%d)", static_cast<int>(nTotalToRead));
237 : #endif
238 :
239 289605 : if (nSize == 0)
240 0 : return 0;
241 :
242 289605 : if (nBufferSize != 0 && nCurOffset >= nBufferOffset &&
243 269842 : nCurOffset <= nBufferOffset + nBufferSize)
244 : {
245 : // We try to read from an offset located within the buffer.
246 : const size_t nReadInBuffer = static_cast<size_t>(std::min(
247 : nTotalToRead,
248 262392 : static_cast<size_t>(nBufferOffset + nBufferSize - nCurOffset)));
249 262392 : memcpy(pBuffer, pabyBuffer + nCurOffset - nBufferOffset, nReadInBuffer);
250 262392 : const size_t nToReadInFile = nTotalToRead - nReadInBuffer;
251 262392 : if (nToReadInFile > 0)
252 : {
253 : // The beginning of the data to read is located in the buffer
254 : // but the end must be read from the file.
255 257713 : if (bNeedBaseHandleSeek)
256 : {
257 100 : if (!SeekBaseTo(nBufferOffset + nBufferSize))
258 : {
259 0 : nCurOffset += nReadInBuffer;
260 0 : return nReadInBuffer / nSize;
261 : }
262 : }
263 257713 : bNeedBaseHandleSeek = false;
264 : #ifdef DEBUG_VERBOSE
265 : CPLAssert(m_poBaseHandle->Tell() == nBufferOffset + nBufferSize);
266 : #endif
267 :
268 515426 : const size_t nReadInFile = m_poBaseHandle->Read(
269 : static_cast<GByte *>(pBuffer) + nReadInBuffer, 1,
270 257713 : nToReadInFile);
271 257713 : const size_t nRead = nReadInBuffer + nReadInFile;
272 :
273 257713 : nBufferSize = static_cast<int>(
274 257713 : std::min(nRead, static_cast<size_t>(MAX_BUFFER_SIZE)));
275 257713 : nBufferOffset = nCurOffset + nRead - nBufferSize;
276 257713 : memcpy(pabyBuffer,
277 257713 : static_cast<GByte *>(pBuffer) + nRead - nBufferSize,
278 257713 : nBufferSize);
279 :
280 257713 : nCurOffset += nRead;
281 : #ifdef DEBUG_VERBOSE
282 : CPLAssert(m_poBaseHandle->Tell() == nBufferOffset + nBufferSize);
283 : CPLAssert(m_poBaseHandle->Tell() == nCurOffset);
284 : #endif
285 :
286 257713 : bEOF = CPL_TO_BOOL(m_poBaseHandle->Eof());
287 :
288 257713 : return nRead / nSize;
289 : }
290 : else
291 : {
292 : // The data to read is completely located within the buffer.
293 4679 : nCurOffset += nTotalToRead;
294 4679 : return nTotalToRead / nSize;
295 : }
296 : }
297 : else
298 : {
299 : // We try either to read before or after the buffer, so a seek is
300 : // necessary.
301 27213 : if (!SeekBaseTo(nCurOffset))
302 0 : return 0;
303 27213 : bNeedBaseHandleSeek = false;
304 : const size_t nReadInFile =
305 27213 : m_poBaseHandle->Read(pBuffer, 1, nTotalToRead);
306 27213 : nBufferSize = static_cast<int>(
307 27213 : std::min(nReadInFile, static_cast<size_t>(MAX_BUFFER_SIZE)));
308 27213 : nBufferOffset = nCurOffset + nReadInFile - nBufferSize;
309 27213 : memcpy(pabyBuffer,
310 27213 : static_cast<GByte *>(pBuffer) + nReadInFile - nBufferSize,
311 27213 : nBufferSize);
312 :
313 27213 : nCurOffset += nReadInFile;
314 : #ifdef DEBUG_VERBOSE
315 : CPLAssert(m_poBaseHandle->Tell() == nBufferOffset + nBufferSize);
316 : CPLAssert(m_poBaseHandle->Tell() == nCurOffset);
317 : #endif
318 :
319 27213 : bEOF = CPL_TO_BOOL(m_poBaseHandle->Eof());
320 :
321 27213 : return nReadInFile / nSize;
322 : }
323 : }
324 :
325 : /************************************************************************/
326 : /* Write() */
327 : /************************************************************************/
328 :
329 0 : size_t VSIBufferedReaderHandle::Write(const void * /* pBuffer */,
330 : size_t /* nSize */, size_t /* nMemb */)
331 : {
332 0 : CPLError(CE_Failure, CPLE_NotSupported,
333 : "VSIFWriteL is not supported on buffer reader streams");
334 0 : return 0;
335 : }
336 :
337 : /************************************************************************/
338 : /* Eof() */
339 : /************************************************************************/
340 :
341 2626 : int VSIBufferedReaderHandle::Eof()
342 : {
343 2626 : return bEOF;
344 : }
345 :
346 : /************************************************************************/
347 : /* Flush() */
348 : /************************************************************************/
349 :
350 0 : int VSIBufferedReaderHandle::Flush()
351 : {
352 0 : return 0;
353 : }
354 :
355 : /************************************************************************/
356 : /* Close() */
357 : /************************************************************************/
358 :
359 4921 : int VSIBufferedReaderHandle::Close()
360 : {
361 4921 : if (m_poBaseHandle)
362 : {
363 4921 : m_poBaseHandle->Close();
364 4921 : delete m_poBaseHandle;
365 4921 : m_poBaseHandle = nullptr;
366 : }
367 4921 : return 0;
368 : }
369 :
370 : //! @endcond
|
