Line data Source code
1 : /******************************************************************************
2 : *
3 : * Project: Feather Translator
4 : * Purpose: Implements OGRFeatherDriver.
5 : * Author: Even Rouault, <even.rouault at spatialys.com>
6 : *
7 : ******************************************************************************
8 : * Copyright (c) 2022, Planet Labs
9 : *
10 : * Permission is hereby granted, free of charge, to any person obtaining a
11 : * copy of this software and associated documentation files (the "Software"),
12 : * to deal in the Software without restriction, including without limitation
13 : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14 : * and/or sell copies of the Software, and to permit persons to whom the
15 : * Software is furnished to do so, subject to the following conditions:
16 : *
17 : * The above copyright notice and this permission notice shall be included
18 : * in all copies or substantial portions of the Software.
19 : *
20 : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21 : * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23 : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 : * DEALINGS IN THE SOFTWARE.
27 : ****************************************************************************/
28 :
29 : #include "ogrsf_frmts.h"
30 : #include "gdal_priv.h"
31 :
32 : #include "ogrfeatherdrivercore.h"
33 :
34 : /************************************************************************/
35 : /* OGRFeatherDriverIsArrowIPCStreamBasic() */
36 : /************************************************************************/
37 :
38 41004 : static int OGRFeatherDriverIsArrowIPCStreamBasic(GDALOpenInfo *poOpenInfo)
39 : {
40 : // WARNING: if making changes in that method, reflect them in
41 : // IsArrowIPCStream() in ogrfeatherdriver.cpp
42 :
43 41004 : if (STARTS_WITH_CI(poOpenInfo->pszFilename, "ARROW_IPC_STREAM:"))
44 3 : return true;
45 :
46 41001 : constexpr int CONTINUATION_SIZE = 4; // 0xFFFFFFFF
47 41001 : constexpr int METADATA_SIZE_SIZE = 4;
48 :
49 : // See
50 : // https://arrow.apache.org/docs/format/Columnar.html#encapsulated-message-format
51 41001 : if (poOpenInfo->fpL != nullptr &&
52 1370 : poOpenInfo->nHeaderBytes >= CONTINUATION_SIZE + METADATA_SIZE_SIZE &&
53 1276 : memcmp(poOpenInfo->pabyHeader, "\xFF\xFF\xFF\xFF", CONTINUATION_SIZE) ==
54 : 0)
55 : {
56 12 : const char *pszExt = CPLGetExtension(poOpenInfo->pszFilename);
57 12 : if (EQUAL(pszExt, "arrows") || EQUAL(pszExt, "ipc"))
58 3 : return true;
59 :
60 9 : const uint32_t nMetadataSize =
61 9 : CPL_LSBUINT32PTR(poOpenInfo->pabyHeader + CONTINUATION_SIZE);
62 9 : if (strcmp(poOpenInfo->pszFilename, "/vsistdin/") == 0)
63 : {
64 : // Padding after metadata and before body is not necessarily present
65 : // but the body must be at least 4 bytes
66 0 : constexpr int PADDING_MAX_SIZE = 4;
67 :
68 : // /vsistdin/ cannot seek back beyond first MB
69 0 : if (nMetadataSize >
70 : 1024 * 1024 -
71 : (CONTINUATION_SIZE + METADATA_SIZE_SIZE + PADDING_MAX_SIZE))
72 : {
73 0 : return false;
74 : }
75 0 : const int nSizeToRead = CONTINUATION_SIZE + METADATA_SIZE_SIZE +
76 0 : nMetadataSize + PADDING_MAX_SIZE;
77 0 : if (!poOpenInfo->TryToIngest(nSizeToRead))
78 : {
79 0 : return false;
80 : }
81 :
82 0 : return GDAL_IDENTIFY_UNKNOWN;
83 : }
84 :
85 9 : VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
86 9 : const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
87 9 : VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
88 9 : if (nMetadataSize >
89 9 : nFileSize - (CONTINUATION_SIZE + METADATA_SIZE_SIZE))
90 4 : return false;
91 :
92 5 : return GDAL_IDENTIFY_UNKNOWN;
93 : }
94 40989 : return false;
95 : }
96 :
97 : /************************************************************************/
98 : /* OGRFeatherDriverIsArrowFileFormat() */
99 : /************************************************************************/
100 :
101 : template <size_t N> constexpr int constexpr_length(const char (&)[N])
102 : {
103 : return static_cast<int>(N - 1);
104 : }
105 :
106 41525 : bool OGRFeatherDriverIsArrowFileFormat(GDALOpenInfo *poOpenInfo)
107 : {
108 : // See https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
109 41525 : bool bRet = false;
110 41525 : constexpr const char SIGNATURE[] = "ARROW1";
111 41525 : constexpr int SIGNATURE_SIZE = constexpr_length(SIGNATURE);
112 : static_assert(SIGNATURE_SIZE == 6, "SIGNATURE_SIZE == 6");
113 41525 : constexpr int SIGNATURE_PLUS_PADDING = SIGNATURE_SIZE + 2;
114 41525 : constexpr int FOOTERSIZE_SIZE = 4;
115 41525 : if (poOpenInfo->fpL != nullptr &&
116 1894 : poOpenInfo->nHeaderBytes >=
117 1770 : SIGNATURE_PLUS_PADDING + FOOTERSIZE_SIZE + SIGNATURE_SIZE &&
118 1770 : memcmp(poOpenInfo->pabyHeader, SIGNATURE, SIGNATURE_SIZE) == 0)
119 : {
120 1064 : VSIFSeekL(poOpenInfo->fpL, 0, SEEK_END);
121 1064 : const auto nFileSize = VSIFTellL(poOpenInfo->fpL);
122 1064 : VSIFSeekL(poOpenInfo->fpL,
123 : nFileSize - (FOOTERSIZE_SIZE + SIGNATURE_SIZE), SEEK_SET);
124 1064 : uint32_t nFooterSize = 0;
125 : static_assert(sizeof(nFooterSize) == FOOTERSIZE_SIZE,
126 : "sizeof(nFooterSize) == FOOTERSIZE_SIZE");
127 1064 : VSIFReadL(&nFooterSize, 1, sizeof(nFooterSize), poOpenInfo->fpL);
128 1064 : CPL_LSBPTR32(&nFooterSize);
129 1064 : unsigned char abyTrailingBytes[SIGNATURE_SIZE] = {0};
130 1064 : VSIFReadL(&abyTrailingBytes[0], 1, SIGNATURE_SIZE, poOpenInfo->fpL);
131 2128 : bRet = memcmp(abyTrailingBytes, SIGNATURE, SIGNATURE_SIZE) == 0 &&
132 1064 : nFooterSize < nFileSize;
133 1064 : VSIFSeekL(poOpenInfo->fpL, 0, SEEK_SET);
134 : }
135 41525 : return bRet;
136 : }
137 :
138 : /************************************************************************/
139 : /* Identify() */
140 : /************************************************************************/
141 :
142 41004 : int OGRFeatherDriverIdentify(GDALOpenInfo *poOpenInfo)
143 : {
144 41004 : int ret = OGRFeatherDriverIsArrowIPCStreamBasic(poOpenInfo);
145 41004 : if (ret == GDAL_IDENTIFY_TRUE || ret == GDAL_IDENTIFY_UNKNOWN)
146 11 : return ret;
147 40993 : return OGRFeatherDriverIsArrowFileFormat(poOpenInfo);
148 : }
149 :
150 : /************************************************************************/
151 : /* OGRFeatherDriverSetCommonMetadata() */
152 : /************************************************************************/
153 :
154 1232 : void OGRFeatherDriverSetCommonMetadata(GDALDriver *poDriver)
155 : {
156 1232 : poDriver->SetDescription(DRIVER_NAME);
157 1232 : poDriver->SetMetadataItem(GDAL_DCAP_VECTOR, "YES");
158 1232 : poDriver->SetMetadataItem(GDAL_DCAP_CREATE_LAYER, "YES");
159 1232 : poDriver->SetMetadataItem(GDAL_DCAP_CREATE_FIELD, "YES");
160 1232 : poDriver->SetMetadataItem(GDAL_DMD_LONGNAME,
161 1232 : "(Geo)Arrow IPC File Format / Stream");
162 1232 : poDriver->SetMetadataItem(GDAL_DMD_EXTENSIONS, "arrow feather arrows ipc");
163 1232 : poDriver->SetMetadataItem(GDAL_DMD_HELPTOPIC,
164 1232 : "drivers/vector/feather.html");
165 1232 : poDriver->SetMetadataItem(GDAL_DCAP_VIRTUALIO, "YES");
166 1232 : poDriver->SetMetadataItem(GDAL_DCAP_MEASURED_GEOMETRIES, "YES");
167 1232 : poDriver->SetMetadataItem(GDAL_DCAP_Z_GEOMETRIES, "YES");
168 1232 : poDriver->SetMetadataItem(GDAL_DMD_SUPPORTED_SQL_DIALECTS, "OGRSQL SQLITE");
169 :
170 1232 : poDriver->SetMetadataItem(
171 : GDAL_DMD_CREATIONFIELDDATATYPES,
172 : "Integer Integer64 Real String Date Time DateTime "
173 1232 : "Binary IntegerList Integer64List RealList StringList");
174 1232 : poDriver->SetMetadataItem(GDAL_DMD_CREATIONFIELDDATASUBTYPES,
175 1232 : "Boolean Int16 Float32 JSON UUID");
176 1232 : poDriver->SetMetadataItem(GDAL_DMD_CREATION_FIELD_DEFN_FLAGS,
177 : "WidthPrecision Nullable "
178 1232 : "Comment AlternativeName Domain");
179 :
180 1232 : poDriver->pfnIdentify = OGRFeatherDriverIdentify;
181 1232 : poDriver->SetMetadataItem(GDAL_DCAP_OPEN, "YES");
182 1232 : poDriver->SetMetadataItem(GDAL_DCAP_CREATE, "YES");
183 1232 : }
184 :
185 : /************************************************************************/
186 : /* DeclareDeferredOGRArrowPlugin() */
187 : /************************************************************************/
188 :
189 : #ifdef PLUGIN_FILENAME
190 1522 : void DeclareDeferredOGRArrowPlugin()
191 : {
192 1522 : if (GDALGetDriverByName(DRIVER_NAME) != nullptr)
193 : {
194 301 : return;
195 : }
196 1221 : auto poDriver = new GDALPluginDriverProxy(PLUGIN_FILENAME);
197 : #ifdef PLUGIN_INSTALLATION_MESSAGE
198 : poDriver->SetMetadataItem(GDAL_DMD_PLUGIN_INSTALLATION_MESSAGE,
199 : PLUGIN_INSTALLATION_MESSAGE);
200 : #endif
201 1221 : OGRFeatherDriverSetCommonMetadata(poDriver);
202 1221 : GetGDALDriverManager()->DeclareDeferredPluginDriver(poDriver);
203 : }
204 : #endif
|
