QGIS API Documentation  2.14.0-Essen
qgsgmlschema.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgsgmlschema.cpp
3  --------------------------------------
4  Date : February 2013
5  Copyright : (C) 2013 by Radim Blazek
6  Email : [email protected]
7  ***************************************************************************
8  * *
9  * This program is free software; you can redistribute it and/or modify *
10  * it under the terms of the GNU General Public License as published by *
11  * the Free Software Foundation; either version 2 of the License, or *
12  * (at your option) any later version. *
13  * *
14  ***************************************************************************/
15 #include "qgsgmlschema.h"
16 #include "qgsrectangle.h"
18 #include "qgserror.h"
19 #include "qgsgeometry.h"
20 #include "qgslogger.h"
22 #include <QBuffer>
23 #include <QList>
24 #include <QNetworkRequest>
25 #include <QNetworkReply>
26 #include <QProgressDialog>
27 #include <QSet>
28 #include <QSettings>
29 #include <QUrl>
30 
31 #include <limits>
32 
33 const char NS_SEPARATOR = '?';
34 const QString GML_NAMESPACE = "http://www.opengis.net/gml";
35 
37 {
38 }
39 
41  : mName( name )
42  , mPath( path )
43 {
44 }
45 
47 {
48  for ( int i = 0; i < mFields.size(); i++ )
49  {
50  if ( mFields[i].name() == name ) return i;
51  }
52  return -1;
53 }
54 
55 // --------------------------- QgsGmlSchema -------------------------------
57  : QObject()
58  , mCurrentFeature( nullptr )
59  , mFeatureCount( 0 )
60  , mLevel( 0 )
61  , mSkipLevel( std::numeric_limits<int>::max() )
62 {
63  mGeometryTypes << "Point" << "MultiPoint"
64  << "LineString" << "MultiLineString"
65  << "Polygon" << "MultiPolygon";
66 }
67 
69 {
70 
71 }
72 
73 QString QgsGmlSchema::readAttribute( const QString& attributeName, const XML_Char** attr ) const
74 {
75  int i = 0;
76  while ( attr[i] )
77  {
78  if ( attributeName.compare( attr[i] ) == 0 )
79  {
80  return QString( attr[i+1] );
81  }
82  i += 2;
83  }
84  return QString();
85 }
86 
88 {
89  QDomDocument dom;
90  QString errorMsg;
91  int errorLine;
92  int errorColumn;
93  if ( !dom.setContent( xml, false, &errorMsg, &errorLine, &errorColumn ) )
94  {
95  // TODO: error
96  return false;
97  }
98 
99  QDomElement docElem = dom.documentElement();
100 
101  QList<QDomElement> elementElements = domElements( docElem, "element" );
102 
103  //QgsDebugMsg( QString( "%1 elemets read" ).arg( elementElements.size() ) );
104 
105  Q_FOREACH ( const QDomElement& elementElement, elementElements )
106  {
107  QString name = elementElement.attribute( "name" );
108  QString type = elementElement.attribute( "type" );
109 
110  QString gmlBaseType = xsdComplexTypeGmlBaseType( docElem, stripNS( type ) );
111  //QgsDebugMsg( QString( "gmlBaseType = %1" ).arg( gmlBaseType ) );
112  //QgsDebugMsg( QString( "name = %1 gmlBaseType = %2" ).arg( name ).arg( gmlBaseType ) );
113  // We should only use gml:AbstractFeatureType descendants which have
114  // ancestor listed in gml:FeatureAssociationType (featureMember) descendant
115  // But we could only loose some data if XSD was not correct, I think.
116 
117  if ( gmlBaseType == "AbstractFeatureType" )
118  {
119  // Get feature type definition
120  QgsGmlFeatureClass featureClass( name, "" );
121  xsdFeatureClass( docElem, stripNS( type ), featureClass );
122  mFeatureClassMap.insert( name, featureClass );
123  }
124  // A feature may have more geometries, we take just the first one
125  }
126 
127  return true;
128 }
129 
130 bool QgsGmlSchema::xsdFeatureClass( const QDomElement &element, const QString & typeName, QgsGmlFeatureClass & featureClass )
131 {
132  //QgsDebugMsg("typeName = " + typeName );
133  QDomElement complexTypeElement = domElement( element, "complexType", "name", typeName );
134  if ( complexTypeElement.isNull() ) return false;
135 
136  // extension or restriction
137  QDomElement extrest = domElement( complexTypeElement, "complexContent.extension" );
138  if ( extrest.isNull() )
139  {
140  extrest = domElement( complexTypeElement, "complexContent.restriction" );
141  }
142  if ( extrest.isNull() ) return false;
143 
144  QString extrestName = extrest.attribute( "base" );
145  if ( extrestName == "gml:AbstractFeatureType" )
146  {
147  // In theory we should add gml:AbstractFeatureType default attributes gml:description
148  // and gml:name but it does not seem to be a common practice and we would probably
149  // confuse most users
150  }
151  else
152  {
153  // Get attributes from extrest
154  if ( !xsdFeatureClass( element, stripNS( extrestName ), featureClass ) ) return false;
155  }
156 
157  // Supported geometry types
158  QStringList geometryPropertyTypes;
159  Q_FOREACH ( const QString& geom, mGeometryTypes )
160  {
161  geometryPropertyTypes << geom + "PropertyType";
162  }
163 
164  QStringList geometryAliases;
165  geometryAliases << "location" << "centerOf" << "position" << "extentOf"
166  << "coverage" << "edgeOf" << "centerLineOf" << "multiLocation"
167  << "multiCenterOf" << "multiPosition" << "multiCenterLineOf"
168  << "multiEdgeOf" << "multiCoverage" << "multiExtentOf";
169 
170  // Add attributes from current comple type
171  QList<QDomElement> sequenceElements = domElements( extrest, "sequence.element" );
172  Q_FOREACH ( const QDomElement& sequenceElement, sequenceElements )
173  {
174  QString fieldName = sequenceElement.attribute( "name" );
175  QString fieldTypeName = stripNS( sequenceElement.attribute( "type" ) );
176  QString ref = sequenceElement.attribute( "ref" );
177  //QgsDebugMsg ( QString("fieldName = %1 fieldTypeName = %2 ref = %3").arg(fieldName).arg(fieldTypeName).arg(ref) );
178 
179  if ( !ref.isEmpty() )
180  {
181  if ( ref.startsWith( "gml:" ) )
182  {
183  if ( geometryAliases.contains( stripNS( ref ) ) )
184  {
185  featureClass.geometryAttributes().append( stripNS( ref ) );
186  }
187  else
188  {
189  QgsDebugMsg( QString( "Unknown referenced GML element: %1" ).arg( ref ) );
190  }
191  }
192  else
193  {
194  // TODO: get type from referenced element
195  QgsDebugMsg( QString( "field %1.%2 is referencing %3 - not supported" ).arg( typeName, fieldName ) );
196  }
197  continue;
198  }
199 
200  if ( fieldName.isEmpty() )
201  {
202  QgsDebugMsg( QString( "field in %1 without name" ).arg( typeName ) );
203  continue;
204  }
205 
206  // type is either type attribute
207  if ( fieldTypeName.isEmpty() )
208  {
209  // or type is inheriting from xs:simpleType
210  QDomElement sequenceElementRestriction = domElement( sequenceElement, "simpleType.restriction" );
211  fieldTypeName = stripNS( sequenceElementRestriction.attribute( "base" ) );
212  }
213 
214  QVariant::Type fieldType = QVariant::String;
215  if ( fieldTypeName.isEmpty() )
216  {
217  QgsDebugMsg( QString( "Cannot get %1.%2 field type" ).arg( typeName, fieldName ) );
218  }
219  else
220  {
221  if ( geometryPropertyTypes.contains( fieldTypeName ) )
222  {
223  // Geometry attribute
224  featureClass.geometryAttributes().append( fieldName );
225  continue;
226  }
227 
228  if ( fieldTypeName == "decimal" )
229  {
230  fieldType = QVariant::Double;
231  }
232  else if ( fieldTypeName == "integer" )
233  {
234  fieldType = QVariant::Int;
235  }
236  }
237 
238  QgsField field( fieldName, fieldType, fieldTypeName );
239  featureClass.fields().append( field );
240  }
241 
242  return true;
243 }
244 
245 QString QgsGmlSchema::xsdComplexTypeGmlBaseType( const QDomElement &element, const QString & name )
246 {
247  //QgsDebugMsg("name = " + name );
248  QDomElement complexTypeElement = domElement( element, "complexType", "name", name );
249  if ( complexTypeElement.isNull() ) return "";
250 
251  QDomElement extrest = domElement( complexTypeElement, "complexContent.extension" );
252  if ( extrest.isNull() )
253  {
254  extrest = domElement( complexTypeElement, "complexContent.restriction" );
255  }
256  if ( extrest.isNull() ) return "";
257 
258  QString extrestName = extrest.attribute( "base" );
259  if ( extrestName.startsWith( "gml:" ) )
260  {
261  // GML base type found
262  return stripNS( extrestName );
263  }
264  // Continue recursively until GML base type is reached
265  return xsdComplexTypeGmlBaseType( element, stripNS( extrestName ) );
266 }
267 
268 QString QgsGmlSchema::stripNS( const QString & name )
269 {
270  return name.contains( ':' ) ? name.section( ':', 1 ) : name;
271 }
272 
273 QList<QDomElement> QgsGmlSchema::domElements( const QDomElement &element, const QString & path )
274 {
275  QList<QDomElement> list;
276 
277  QStringList names = path.split( '.' );
278  if ( names.isEmpty() ) return list;
279  QString name = names.value( 0 );
280  names.removeFirst();
281 
282  QDomNode n1 = element.firstChild();
283  while ( !n1.isNull() )
284  {
285  QDomElement el = n1.toElement();
286  if ( !el.isNull() )
287  {
288  QString tagName = stripNS( el.tagName() );
289  if ( tagName == name )
290  {
291  if ( names.isEmpty() )
292  {
293  list.append( el );
294  }
295  else
296  {
297  list.append( domElements( el, names.join( "." ) ) );
298  }
299  }
300  }
301  n1 = n1.nextSibling();
302  }
303 
304  return list;
305 }
306 
307 QDomElement QgsGmlSchema::domElement( const QDomElement &element, const QString & path )
308 {
309  return domElements( element, path ).value( 0 );
310 }
311 
312 QList<QDomElement> QgsGmlSchema::domElements( QList<QDomElement> &elements, const QString & attr, const QString & attrVal )
313 {
314  QList<QDomElement> list;
315  Q_FOREACH ( const QDomElement& el, elements )
316  {
317  if ( el.attribute( attr ) == attrVal )
318  {
319  list << el;
320  }
321  }
322  return list;
323 }
324 
325 QDomElement QgsGmlSchema::domElement( const QDomElement &element, const QString & path, const QString & attr, const QString & attrVal )
326 {
327  QList<QDomElement> list = domElements( element, path );
328  return domElements( list, attr, attrVal ).value( 0 );
329 }
330 
332 {
333  QgsDebugMsg( "Entered" );
334  mLevel = 0;
335  mSkipLevel = std::numeric_limits<int>::max();
336  XML_Parser p = XML_ParserCreateNS( nullptr, NS_SEPARATOR );
337  XML_SetUserData( p, this );
338  XML_SetElementHandler( p, QgsGmlSchema::start, QgsGmlSchema::end );
339  XML_SetCharacterDataHandler( p, QgsGmlSchema::chars );
340  int atEnd = 1;
341  int res = XML_Parse( p, data.constData(), data.size(), atEnd );
342 
343  if ( res == 0 )
344  {
345  QString err = QString( XML_ErrorString( XML_GetErrorCode( p ) ) );
346  QgsDebugMsg( QString( "XML_Parse returned %1 error %2" ).arg( res ).arg( err ) );
347  mError = QgsError( err, "GML schema" );
348  mError.append( tr( "Cannot guess schema" ) );
349  }
350 
351  return res != 0;
352 }
353 
354 void QgsGmlSchema::startElement( const XML_Char* el, const XML_Char** attr )
355 {
356  Q_UNUSED( attr );
357  mLevel++;
358 
359  QString elementName = QString::fromUtf8( el );
360  QgsDebugMsgLevel( QString( "-> %1 %2 %3" ).arg( mLevel ).arg( elementName, mLevel >= mSkipLevel ? "skip" : "" ), 5 );
361 
362  if ( mLevel >= mSkipLevel )
363  {
364  //QgsDebugMsg( QString("skip level %1").arg( mLevel ) );
365  return;
366  }
367 
368  mParsePathStack.append( elementName );
369  QString path = mParsePathStack.join( "." );
370 
371  QStringList splitName = elementName.split( NS_SEPARATOR );
372  QString localName = splitName.last();
373  QString ns = splitName.size() > 1 ? splitName.first() : "";
374  //QgsDebugMsg( "ns = " + ns + " localName = " + localName );
375 
376  ParseMode parseMode = modeStackTop();
377  //QgsDebugMsg ( QString("localName = %1 parseMode = %2").arg(localName).arg(parseMode) );
378 
379  if ( ns == GML_NAMESPACE && localName == "boundedBy" )
380  {
381  // gml:boundedBy in feature or feature collection -> skip
382  mSkipLevel = mLevel + 1;
383  }
384  else if ( localName.compare( "featureMembers", Qt::CaseInsensitive ) == 0 )
385  {
386  mParseModeStack.push( QgsGmlSchema::featureMembers );
387  }
388  // GML does not specify that gml:FeatureAssociationType elements should end
389  // with 'Member' apart standard gml:featureMember, but it is quite usual to
390  // that the names ends with 'Member', e.g.: osgb:topographicMember, cityMember,...
391  // so this is really fail if the name does not contain 'Member'
392 
393  else if ( localName.endsWith( "member", Qt::CaseInsensitive ) )
394  {
395  mParseModeStack.push( QgsGmlSchema::featureMember );
396  }
397  // UMN Mapserver simple GetFeatureInfo response layer element (ends with _layer)
398  else if ( elementName.endsWith( "_layer" ) )
399  {
400  // do nothing, we catch _feature children
401  }
402  // UMN Mapserver simple GetFeatureInfo response feature element (ends with _feature)
403  // or featureMember children.
404  // QGIS mapserver 2.2 GetFeatureInfo is using <Feature id="###"> for feature member,
405  // without any feature class distinction.
406  else if ( elementName.endsWith( "_feature" )
407  || parseMode == QgsGmlSchema::featureMember
408  || parseMode == QgsGmlSchema::featureMembers
409  || localName.compare( "feature", Qt::CaseInsensitive ) == 0 )
410  {
411  QgsDebugMsg( "is feature path = " + path );
412  if ( mFeatureClassMap.count( localName ) == 0 )
413  {
414  mFeatureClassMap.insert( localName, QgsGmlFeatureClass( localName, path ) );
415  }
416  mCurrentFeatureName = localName;
417  mParseModeStack.push( QgsGmlSchema::feature );
418  }
419  else if ( parseMode == QgsGmlSchema::attribute && ns == GML_NAMESPACE && mGeometryTypes.indexOf( localName ) >= 0 )
420  {
421  // Geometry (Point,MultiPoint,...) in geometry attribute
422  QStringList &geometryAttributes = mFeatureClassMap[mCurrentFeatureName].geometryAttributes();
423  if ( geometryAttributes.count( mAttributeName ) == 0 )
424  {
425  geometryAttributes.append( mAttributeName );
426  }
427  mSkipLevel = mLevel + 1; // no need to parse children
428  }
429  else if ( parseMode == QgsGmlSchema::feature )
430  {
431  // An element in feature should be ordinary or geometry attribute
432  //QgsDebugMsg( "is attribute");
433 
434  // Usually localName is attribute name, e.g.
435  // <gml:desc>My description</gml:desc>
436  // but QGIS server (2.2) is using:
437  // <Attribute value="My description" name="desc"/>
438  QString name = readAttribute( "name", attr );
439  //QgsDebugMsg ( "attribute name = " + name );
440  if ( localName.compare( "attribute", Qt::CaseInsensitive ) == 0
441  && !name.isEmpty() )
442  {
443  QString value = readAttribute( "value", attr );
444  //QgsDebugMsg ( "attribute value = " + value );
445  addAttribute( name, value );
446  }
447  else
448  {
449  mAttributeName = localName;
450  mParseModeStack.push( QgsGmlSchema::attribute );
451  mStringCash.clear();
452  }
453  }
454 }
455 
456 void QgsGmlSchema::endElement( const XML_Char* el )
457 {
458  QString elementName = QString::fromUtf8( el );
459  QgsDebugMsgLevel( QString( "<- %1 %2" ).arg( mLevel ).arg( elementName ), 5 );
460 
461  if ( mLevel >= mSkipLevel )
462  {
463  //QgsDebugMsg( QString("skip level %1").arg( mLevel ) );
464  mLevel--;
465  return;
466  }
467  else
468  {
469  // clear possible skip level
470  mSkipLevel = std::numeric_limits<int>::max();
471  }
472 
473  QStringList splitName = elementName.split( NS_SEPARATOR );
474  QString localName = splitName.last();
475  QString ns = splitName.size() > 1 ? splitName.first() : "";
476 
477  QgsGmlSchema::ParseMode parseMode = modeStackTop();
478 
479  if ( parseMode == QgsGmlSchema::featureMembers )
480  {
481  modeStackPop();
482  }
483  else if ( parseMode == QgsGmlSchema::attribute && localName == mAttributeName )
484  {
485  // End of attribute
486  //QgsDebugMsg("end attribute");
487  modeStackPop(); // go up to feature
488 
489  if ( mFeatureClassMap[mCurrentFeatureName].geometryAttributes().count( mAttributeName ) == 0 )
490  {
491  addAttribute( mAttributeName, mStringCash );
492  }
493  }
494  else if ( ns == GML_NAMESPACE && localName == "boundedBy" )
495  {
496  // was skipped
497  }
498  else if ( localName.endsWith( "member", Qt::CaseInsensitive ) )
499  {
500  modeStackPop();
501  }
502  mParsePathStack.removeLast();
503  mLevel--;
504 }
505 
506 void QgsGmlSchema::characters( const XML_Char* chars, int len )
507 {
508  //QgsDebugMsg( QString("level %1 : %2").arg( mLevel ).arg( QString::fromUtf8( chars, len ) ) );
509  if ( mLevel >= mSkipLevel )
510  {
511  //QgsDebugMsg( QString("skip level %1").arg( mLevel ) );
512  return;
513  }
514 
515  //save chars in mStringCash attribute mode for value type analysis
516  if ( modeStackTop() == QgsGmlSchema::attribute )
517  {
518  mStringCash.append( QString::fromUtf8( chars, len ) );
519  }
520 }
521 
522 void QgsGmlSchema::addAttribute( const QString& name, const QString& value )
523 {
524  // It is not geometry attribute -> analyze value
525  bool ok;
526  value.toInt( &ok );
527  QVariant::Type type = QVariant::String;
528  if ( ok )
529  {
530  type = QVariant::Int;
531  }
532  else
533  {
534  value.toDouble( &ok );
535  if ( ok )
536  {
537  type = QVariant::Double;
538  }
539  }
540  //QgsDebugMsg( "mStringCash = " + mStringCash + " type = " + QVariant::typeToName( type ) );
541  //QMap<QString, QgsField> & fields = mFeatureClassMap[mCurrentFeatureName].fields();
542  QList<QgsField> & fields = mFeatureClassMap[mCurrentFeatureName].fields();
543  int fieldIndex = mFeatureClassMap[mCurrentFeatureName].fieldIndex( name );
544  if ( fieldIndex == -1 )
545  {
546  QgsField field( name, type );
547  fields.append( field );
548  }
549  else
550  {
551  QgsField &field = fields[fieldIndex];
552  // check if type is sufficient
553  if (( field.type() == QVariant::Int && ( type == QVariant::String || type == QVariant::Double ) ) ||
554  ( field.type() == QVariant::Double && type == QVariant::String ) )
555  {
556  field.setType( type );
557  }
558  }
559 }
560 
562 {
563  return mFeatureClassMap.keys();
564 }
565 
567 {
568  if ( mFeatureClassMap.count( typeName ) == 0 ) return QList<QgsField>();
569  return mFeatureClassMap[typeName].fields();
570 }
571 
573 {
574  if ( mFeatureClassMap.count( typeName ) == 0 ) return QStringList();
575  return mFeatureClassMap[typeName].geometryAttributes();
576 }
QString & append(QChar ch)
bool guessSchema(const QByteArray &data)
Guess GML schema from data if XSD does not exist.
QString attribute(const QString &name, const QString &defValue) const
#define QgsDebugMsg(str)
Definition: qgslogger.h:33
QStringList split(const QString &sep, SplitBehavior behavior, Qt::CaseSensitivity cs) const
void push(const T &t)
void removeFirst()
bool contains(const QString &str, Qt::CaseSensitivity cs) const
QDomElement documentElement() const
QString join(const QString &separator) const
double toDouble(bool *ok) const
QString tr(const char *sourceText, const char *disambiguation, int n)
int size() const
QDomNode nextSibling() const
T value(int i) const
const QString GML_NAMESPACE
void clear()
QDomElement toElement() const
double ANALYSIS_EXPORT max(double x, double y)
Returns the maximum of two doubles or the first argument if both are equal.
QList< Key > keys() const
QList< QgsField > & fields()
Definition: qgsgmlschema.h:45
const char * name() const
int count(const T &value) const
void append(const T &value)
QString fromUtf8(const char *str, int size)
#define QgsDebugMsgLevel(str, level)
Definition: qgslogger.h:34
QString path() const
Definition: qgsgmlschema.h:49
QStringList typeNames() const
Get list of dot separated paths to feature classes parsed from GML or XSD.
int toInt(bool *ok, int base) const
bool isEmpty() const
bool isEmpty() const
const char * constData() const
bool startsWith(const QString &s, Qt::CaseSensitivity cs) const
bool endsWith(const QString &s, Qt::CaseSensitivity cs) const
QList< QgsField > fields(const QString &typeName)
Get fields for type/class name parsed from GML or XSD.
T & first()
Encapsulate a field in an attribute table or data source.
Definition: qgsfield.h:44
bool contains(QChar ch, Qt::CaseSensitivity cs) const
bool isNull() const
void setType(QVariant::Type type)
Set variant type.
Definition: qgsfield.cpp:125
QDomNode firstChild() const
QgsError is container for error messages (report).
Definition: qgserror.h:77
void append(const QString &theMessage, const QString &theTag)
Append new error message.
Definition: qgserror.cpp:40
T & last()
void removeLast()
QString section(QChar sep, int start, int end, QFlags< QString::SectionFlag > flags) const
int indexOf(const QRegExp &rx, int from) const
iterator insert(const Key &key, const T &value)
const char NS_SEPARATOR
QString tagName() const
QStringList & geometryAttributes()
Definition: qgsgmlschema.h:51
int size() const
int compare(const QString &other) const
int count(const Key &key) const
int fieldIndex(const QString &name)
bool parseXSD(const QByteArray &xml)
Get fields info from XSD.
bool setContent(const QByteArray &data, bool namespaceProcessing, QString *errorMsg, int *errorLine, int *errorColumn)
QVariant::Type type() const
Gets variant type of the field as it will be retrieved from data source.
Definition: qgsfield.cpp:89
QStringList geometryAttributes(const QString &typeName)
Get list of geometry attributes for type/class name.