Line data    Source code

       1             : /******************************************************************************
       2             :  * $Id$
       3             :  *
       4             :  * Project:  Elasticsearch Translator
       5             :  * Purpose:
       6             :  * Author:
       7             :  *
       8             :  ******************************************************************************
       9             :  * Copyright (c) 2011, Adam Estrada
      10             :  *
      11             :  * Permission is hereby granted, free of charge, to any person obtaining a
      12             :  * copy of this software and associated documentation files (the "Software"),
      13             :  * to deal in the Software without restriction, including without limitation
      14             :  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
      15             :  * and/or sell copies of the Software, and to permit persons to whom the
      16             :  * Software is furnished to do so, subject to the following conditions:
      17             :  *
      18             :  * The above copyright notice and this permission notice shall be included
      19             :  * in all copies or substantial portions of the Software.
      20             :  *
      21             :  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
      22             :  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
      23             :  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
      24             :  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
      25             :  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
      26             :  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
      27             :  * DEALINGS IN THE SOFTWARE.
      28             :  ****************************************************************************/
      29             : 
      30             : #ifndef OGR_ELASTIC_H_INCLUDED
      31             : #define OGR_ELASTIC_H_INCLUDED
      32             : 
      33             : #include "ogrsf_frmts.h"
      34             : 
      35             : #include "cpl_json_header.h"
      36             : #include "cpl_hash_set.h"
      37             : #include "ogr_p.h"
      38             : #include "cpl_http.h"
      39             : #include "cpl_json.h"
      40             : 
      41             : #include <map>
      42             : #include <memory>
      43             : #include <set>
      44             : #include <vector>
      45             : 
      46             : typedef enum
      47             : {
      48             :     ES_GEOMTYPE_AUTO,
      49             :     ES_GEOMTYPE_GEO_POINT,
      50             :     ES_GEOMTYPE_GEO_SHAPE
      51             : } ESGeometryTypeMapping;
      52             : 
      53             : class OGRElasticDataSource;
      54             : 
      55             : class OGRESSortDesc
      56             : {
      57             :   public:
      58             :     CPLString osColumn;
      59             :     bool bAsc;
      60             : 
      61           5 :     OGRESSortDesc(const CPLString &osColumnIn, bool bAscIn)
      62           5 :         : osColumn(osColumnIn), bAsc(bAscIn)
      63             :     {
      64           5 :     }
      65             : };
      66             : 
      67             : /************************************************************************/
      68             : /*                          OGRElasticLayer                             */
      69             : /************************************************************************/
      70             : 
      71             : class OGRElasticLayer final : public OGRLayer
      72             : {
      73             :     OGRElasticDataSource *m_poDS = nullptr;
      74             : 
      75             :     CPLString m_osIndexName{};
      76             :     CPLString m_osMappingName{};
      77             : 
      78             :     OGRFeatureDefn *m_poFeatureDefn = nullptr;
      79             :     bool m_bFeatureDefnFinalized = false;
      80             :     bool m_bAddSourceIndexName = false;
      81             : 
      82             :     bool m_bManualMapping = false;
      83             :     bool m_bSerializeMapping = false;
      84             :     CPLString m_osWriteMapFilename{};
      85             :     bool m_bStoreFields = false;
      86             :     char **m_papszStoredFields = nullptr;
      87             :     char **m_papszNotAnalyzedFields = nullptr;
      88             :     char **m_papszNotIndexedFields = nullptr;
      89             :     char **m_papszFieldsWithRawValue = nullptr;
      90             : 
      91             :     CPLString m_osESSearch{};
      92             :     std::vector<OGRESSortDesc> m_aoSortColumns{};
      93             : 
      94             :     CPLString m_osBulkContent{};
      95             :     int m_nBulkUpload{};
      96             : 
      97             :     CPLString m_osFID{};
      98             : 
      99             :     std::vector<std::vector<CPLString>> m_aaosFieldPaths{};
     100             :     std::map<CPLString, int> m_aosMapToFieldIndex{};
     101             : 
     102             :     std::vector<std::vector<CPLString>> m_aaosGeomFieldPaths{};
     103             :     std::map<CPLString, int> m_aosMapToGeomFieldIndex{};
     104             :     std::vector<OGRCoordinateTransformation *> m_apoCT{};
     105             :     std::vector<int> m_abIsGeoPoint{};
     106             :     ESGeometryTypeMapping m_eGeomTypeMapping = ES_GEOMTYPE_AUTO;
     107             :     CPLString m_osPrecision{};
     108             : 
     109             :     CPLString m_osScrollID{};
     110             :     GIntBig m_iCurID = 0;
     111             :     GIntBig m_nNextFID = -1;  // for creation
     112             :     int m_iCurFeatureInPage = 0;
     113             :     std::vector<OGRFeature *> m_apoCachedFeatures{};
     114             :     bool m_bEOF = false;
     115             : 
     116             :     json_object *m_poSpatialFilter = nullptr;
     117             :     CPLString m_osJSONFilter{};
     118             :     bool m_bFilterMustBeClientSideEvaluated = false;
     119             :     json_object *m_poJSONFilter = nullptr;
     120             : 
     121             :     bool m_bIgnoreSourceID = false;
     122             :     bool m_bDotAsNestedField = true;
     123             : 
     124             :     bool m_bAddPretty = false;
     125             :     bool m_bGeoShapeAsGeoJSON = false;
     126             : 
     127             :     CPLString m_osSingleQueryTimeout{};
     128             :     double m_dfSingleQueryTimeout = 0;
     129             :     double m_dfFeatureIterationTimeout = 0;
     130             :     //! Timestamp after which the query must be terminated
     131             :     double m_dfEndTimeStamp = 0;
     132             : 
     133             :     GIntBig m_nReadFeaturesSinceResetReading = 0;
     134             :     GIntBig m_nSingleQueryTerminateAfter = 0;
     135             :     GIntBig m_nFeatureIterationTerminateAfter = 0;
     136             :     CPLString m_osSingleQueryTerminateAfter;
     137             : 
     138             :     bool m_bUseSingleQueryParams = false;
     139             : 
     140             :     void CopyMembersTo(OGRElasticLayer *poNew);
     141             : 
     142             :     bool PushIndex();
     143             :     CPLString BuildMap();
     144             : 
     145             :     OGRErr WriteMapIfNecessary();
     146             :     OGRFeature *GetNextRawFeature();
     147             :     void BuildFeature(OGRFeature *poFeature, json_object *poSource,
     148             :                       CPLString osPath);
     149             :     void CreateFieldFromSchema(const char *pszName, const char *pszPrefix,
     150             :                                std::vector<CPLString> aosPath,
     151             :                                json_object *poObj);
     152             :     void AddOrUpdateField(const char *pszAttrName, const char *pszKey,
     153             :                           json_object *poObj, char chNestedAttributeSeparator,
     154             :                           std::vector<CPLString> &aosPath);
     155             : 
     156             :     CPLString BuildMappingURL(bool bMappingApi);
     157             : 
     158             :     CPLString BuildJSonFromFeature(OGRFeature *poFeature);
     159             : 
     160             :     static CPLString BuildPathFromArray(const std::vector<CPLString> &aosPath);
     161             : 
     162             :     void AddFieldDefn(const char *pszName, OGRFieldType eType,
     163             :                       const std::vector<CPLString> &aosPath,
     164             :                       OGRFieldSubType eSubType = OFSTNone);
     165             :     void AddGeomFieldDefn(const char *pszName, OGRwkbGeometryType eType,
     166             :                           const std::vector<CPLString> &aosPath,
     167             :                           int bIsGeoPoint);
     168             : 
     169             :     CPLString BuildQuery(bool bCountOnly);
     170             :     json_object *GetValue(int nFieldIdx, swq_expr_node *poValNode);
     171             :     json_object *TranslateSQLToFilter(swq_expr_node *poNode);
     172             :     json_object *BuildSort();
     173             : 
     174             :     void AddTimeoutTerminateAfterToURL(CPLString &osURL);
     175             : 
     176             :   public:
     177             :     OGRElasticLayer(const char *pszLayerName, const char *pszIndexName,
     178             :                     const char *pszMappingName, OGRElasticDataSource *poDS,
     179             :                     CSLConstList papszOptions,
     180             :                     const char *pszESSearch = nullptr);
     181             :     OGRElasticLayer(const char *pszLayerName,
     182             :                     OGRElasticLayer *poReferenceLayer);
     183             :     virtual ~OGRElasticLayer();
     184             : 
     185             :     virtual void ResetReading() override;
     186             :     virtual OGRFeature *GetNextFeature() override;
     187             : 
     188             :     virtual OGRErr ICreateFeature(OGRFeature *poFeature) override;
     189             :     virtual OGRErr ISetFeature(OGRFeature *poFeature) override;
     190             :     OGRErr IUpsertFeature(OGRFeature *poFeature) override;
     191             :     virtual OGRErr CreateField(const OGRFieldDefn *poField,
     192             :                                int bApproxOK) override;
     193             :     virtual OGRErr CreateGeomField(const OGRGeomFieldDefn *poField,
     194             :                                    int bApproxOK) override;
     195             : 
     196          45 :     virtual const char *GetName() override
     197             :     {
     198          45 :         return m_poFeatureDefn->GetName();
     199             :     }
     200             : 
     201             :     virtual OGRFeatureDefn *GetLayerDefn() override;
     202             :     virtual const char *GetFIDColumn() override;
     203             : 
     204             :     virtual int TestCapability(const char *) override;
     205             : 
     206             :     virtual GIntBig GetFeatureCount(int bForce) override;
     207             : 
     208           9 :     virtual void SetSpatialFilter(OGRGeometry *poGeom) override
     209             :     {
     210           9 :         SetSpatialFilter(0, poGeom);
     211           9 :     }
     212             : 
     213             :     virtual void SetSpatialFilter(int iGeomField, OGRGeometry *poGeom) override;
     214             :     virtual OGRErr SetAttributeFilter(const char *pszFilter) override;
     215             : 
     216           0 :     virtual OGRErr GetExtent(OGREnvelope *psExtent, int bForce = TRUE) override
     217             :     {
     218           0 :         return GetExtent(0, psExtent, bForce);
     219             :     }
     220             : 
     221             :     virtual OGRErr GetExtent(int iGeomField, OGREnvelope *psExtent,
     222             :                              int bForce = TRUE) override;
     223             : 
     224             :     virtual OGRErr SyncToDisk() override;
     225             : 
     226             :     GDALDataset *GetDataset() override;
     227             : 
     228             :     void FinalizeFeatureDefn(bool bReadFeatures = true);
     229             :     void InitFeatureDefnFromMapping(json_object *poSchema,
     230             :                                     const char *pszPrefix,
     231             :                                     const std::vector<CPLString> &aosPath);
     232             : 
     233           7 :     const CPLString &GetIndexName() const
     234             :     {
     235           7 :         return m_osIndexName;
     236             :     }
     237             : 
     238           1 :     const CPLString &GetMappingName() const
     239             :     {
     240           1 :         return m_osMappingName;
     241             :     }
     242             : 
     243          21 :     void SetIgnoreSourceID(bool bFlag)
     244             :     {
     245          21 :         m_bIgnoreSourceID = bFlag;
     246          21 :     }
     247             : 
     248           6 :     void SetManualMapping()
     249             :     {
     250           6 :         m_bManualMapping = true;
     251           6 :     }
     252             : 
     253          21 :     void SetDotAsNestedField(bool bFlag)
     254             :     {
     255          21 :         m_bDotAsNestedField = bFlag;
     256          21 :     }
     257             : 
     258          21 :     void SetFID(const CPLString &m_osFIDIn)
     259             :     {
     260          21 :         m_osFID = m_osFIDIn;
     261          21 :     }
     262             : 
     263          21 :     void SetNextFID(GIntBig nNextFID)
     264             :     {
     265          21 :         m_nNextFID = nNextFID;
     266          21 :     }
     267             : 
     268             :     OGRElasticLayer *Clone();
     269             : 
     270           3 :     void SetOrderBy(const std::vector<OGRESSortDesc> &v)
     271             :     {
     272           3 :         m_aoSortColumns = v;
     273           3 :     }
     274             : 
     275           3 :     void SetFeatureDefnFinalized()
     276             :     {
     277           3 :         m_bFeatureDefnFinalized = true;
     278           3 :     }
     279             : 
     280             :     void GetGeomFieldProperties(int iGeomField, std::vector<CPLString> &aosPath,
     281             :                                 bool &bIsGeoPoint);
     282             : 
     283             :     static void ClampEnvelope(OGREnvelope &sEnvelope);
     284             : };
     285             : 
     286             : /************************************************************************/
     287             : /*                      OGRElasticAggregationLayer                      */
     288             : /************************************************************************/
     289             : 
     290             : class OGRElasticAggregationLayer final
     291             :     : public OGRLayer,
     292             :       public OGRGetNextFeatureThroughRaw<OGRElasticAggregationLayer>
     293             : {
     294             : 
     295             :     OGRElasticDataSource *m_poDS = nullptr;
     296             :     std::string m_osIndexName{};
     297             :     std::string m_osGeometryField{};
     298             :     OGRFeatureDefn *m_poFeatureDefn = nullptr;
     299             :     bool m_bFeaturesRequested = false;
     300             :     int m_iCurFeature = 0;
     301             :     bool m_bRequestHasSpatialFilter = false;
     302             :     int m_nGeohashGridMaxSize = 10000;
     303             :     int m_nGeohashGridPrecision = -1;
     304             :     CPLJSONObject m_oFieldDef{};
     305             :     CPLJSONObject m_oAggregatedFieldsRequest{};
     306             :     std::vector<std::unique_ptr<OGRFeature>> m_apoCachedFeatures{};
     307             : 
     308             :     explicit OGRElasticAggregationLayer(OGRElasticDataSource *poDS);
     309             :     std::string BuildRequest();
     310             :     void IssueAggregationRequest();
     311             :     OGRFeature *GetNextRawFeature();
     312             : 
     313             :   public:
     314             :     ~OGRElasticAggregationLayer() override;
     315             : 
     316           3 :     OGRFeatureDefn *GetLayerDefn() override
     317             :     {
     318           3 :         return m_poFeatureDefn;
     319             :     }
     320             : 
     321             :     void ResetReading() override;
     322           7 :     DEFINE_GET_NEXT_FEATURE_THROUGH_RAW(OGRElasticAggregationLayer)
     323             :     GIntBig GetFeatureCount(int bForce) override;
     324             :     int TestCapability(const char *) override;
     325             : 
     326             :     using OGRLayer::SetSpatialFilter;
     327             :     void SetSpatialFilter(OGRGeometry *poGeom) override;
     328             : 
     329             :     GDALDataset *GetDataset() override;
     330             : 
     331             :     static std::unique_ptr<OGRElasticAggregationLayer>
     332             :     Build(OGRElasticDataSource *poDS, const char *pszAggregation);
     333             : };
     334             : 
     335             : /************************************************************************/
     336             : /*                         OGRElasticDataSource                         */
     337             : /************************************************************************/
     338             : 
     339             : class OGRElasticDataSource final : public GDALDataset
     340             : {
     341             :     char *m_pszName;
     342             :     CPLString m_osURL;
     343             :     CPLString m_osUserPwd;
     344             :     CPLString m_osFID;
     345             : 
     346             :     std::set<CPLString> m_oSetLayers;
     347             :     std::vector<std::unique_ptr<OGRElasticLayer>> m_apoLayers;
     348             :     std::unique_ptr<OGRElasticAggregationLayer> m_poAggregationLayer{};
     349             :     bool m_bAllLayersListed = false;
     350             :     std::map<OGRLayer *, OGRLayer *> m_oMapResultSet;
     351             :     std::map<std::string, std::string> m_oMapHeadersFromEnv{};
     352             : 
     353             :     bool CheckVersion();
     354             :     int GetLayerIndex(const char *pszName);
     355             :     void FetchMapping(const char *pszIndexName);
     356             :     bool OpenAggregation(const char *pszAggregation);
     357             :     std::vector<std::string> GetIndexList(const char *pszQueriedIndexName);
     358             : 
     359             :   public:
     360             :     OGRElasticDataSource();
     361             :     virtual ~OGRElasticDataSource();
     362             : 
     363             :     bool m_bOverwrite;
     364             :     int m_nBulkUpload;
     365             :     char *m_pszWriteMap;
     366             :     char *m_pszMapping;
     367             :     int m_nBatchSize;
     368             :     int m_nFeatureCountToEstablishFeatureDefn;
     369             :     bool m_bJSonField;
     370             :     bool m_bFlattenNestedAttributes;
     371             :     bool m_bAddSourceIndexName = false;  // Only used for wildcard layers
     372             :     int m_nMajorVersion = 0;
     373             :     int m_nMinorVersion = 0;
     374             : 
     375             :     bool Open(GDALOpenInfo *poOpenInfo);
     376             : 
     377             :     int Create(const char *pszFilename, char **papszOptions);
     378             : 
     379             :     CPLHTTPResult *HTTPFetch(const char *pszURL, CSLConstList papszOptions);
     380             : 
     381         237 :     const char *GetURL()
     382             :     {
     383         237 :         return m_osURL.c_str();
     384             :     }
     385             : 
     386           0 :     virtual const char *GetName()
     387             :     {
     388           0 :         return m_pszName;
     389             :     }
     390             : 
     391             :     virtual int GetLayerCount() override;
     392             :     virtual OGRLayer *GetLayer(int) override;
     393             :     virtual OGRLayer *GetLayerByName(const char *pszName) override;
     394             : 
     395             :     OGRLayer *ICreateLayer(const char *pszName,
     396             :                            const OGRGeomFieldDefn *poGeomFieldDefn,
     397             :                            CSLConstList papszOptions) override;
     398             :     virtual OGRErr DeleteLayer(int iLayer) override;
     399             : 
     400             :     virtual OGRLayer *ExecuteSQL(const char *pszSQLCommand,
     401             :                                  OGRGeometry *poSpatialFilter,
     402             :                                  const char *pszDialect) override;
     403             :     virtual void ReleaseResultSet(OGRLayer *poLayer) override;
     404             : 
     405             :     virtual int TestCapability(const char *) override;
     406             : 
     407             :     bool UploadFile(const CPLString &url, const CPLString &data,
     408             :                     const CPLString &osVerb = CPLString());
     409             :     void Delete(const CPLString &url);
     410             : 
     411             :     json_object *RunRequest(
     412             :         const char *pszURL, const char *pszPostContent = nullptr,
     413             :         const std::vector<int> &anSilentedHTTPErrors = std::vector<int>());
     414             : 
     415          90 :     const CPLString &GetFID() const
     416             :     {
     417          90 :         return m_osFID;
     418             :     }
     419             : 
     420             :     void FetchMapping(const char *pszIndexName, std::set<CPLString> &oSetLayers,
     421             :                       std::vector<std::unique_ptr<OGRElasticLayer>> &apoLayers);
     422             : };
     423             : 
     424             : #endif /* ndef _OGR_Elastic_H_INCLUDED */