QGIS API Documentation  2.14.0-Essen
qgsosmimport.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgsosmimport.cpp
3  --------------------------------------
4  Date : January 2013
5  Copyright : (C) 2013 by Martin Dobias
6  Email : wonder dot sk at gmail dot com
7  ***************************************************************************
8  * *
9  * This program is free software; you can redistribute it and/or modify *
10  * it under the terms of the GNU General Public License as published by *
11  * the Free Software Foundation; either version 2 of the License, or *
12  * (at your option) any later version. *
13  * *
14  ***************************************************************************/
15 
16 #include "qgsosmimport.h"
17 #include "qgsslconnect.h"
18 
19 #include <QStringList>
20 #include <QXmlStreamReader>
21 
22 
23 QgsOSMXmlImport::QgsOSMXmlImport( const QString& xmlFilename, const QString& dbFilename )
24  : mXmlFileName( xmlFilename )
25  , mDbFileName( dbFilename )
26  , mDatabase( nullptr )
27  , mStmtInsertNode( nullptr )
28  , mStmtInsertNodeTag( nullptr )
29  , mStmtInsertWay( nullptr )
30  , mStmtInsertWayNode( nullptr )
31  , mStmtInsertWayTag( nullptr )
32 {
33 
34 }
35 
37 {
38  mError.clear();
39 
40  // open input
41  mInputFile.setFileName( mXmlFileName );
42  if ( !mInputFile.open( QIODevice::ReadOnly ) )
43  {
44  mError = QString( "Cannot open input file: %1" ).arg( mXmlFileName );
45  return false;
46  }
47 
48  // open output
49 
50  if ( QFile::exists( mDbFileName ) )
51  {
52  if ( !QFile( mDbFileName ).remove() )
53  {
54  mError = QString( "Database file cannot be overwritten: %1" ).arg( mDbFileName );
55  return false;
56  }
57  }
58 
59  if ( !createDatabase() )
60  {
61  // mError is set in createDatabase()
62  return false;
63  }
64 
65  qDebug( "starting import" );
66 
67  int retX = sqlite3_exec( mDatabase, "BEGIN", nullptr, nullptr, nullptr );
68  Q_ASSERT( retX == SQLITE_OK );
69  Q_UNUSED( retX );
70 
71  // start parsing
72 
73  QXmlStreamReader xml( &mInputFile );
74 
75  while ( !xml.atEnd() )
76  {
77  xml.readNext();
78 
79  if ( xml.isEndDocument() )
80  break;
81 
82  if ( xml.isStartElement() )
83  {
84  if ( xml.name() == "osm" )
85  readRoot( xml );
86  else
87  xml.raiseError( "Invalid root tag" );
88  }
89  }
90 
91  int retY = sqlite3_exec( mDatabase, "COMMIT", nullptr, nullptr, nullptr );
92  Q_ASSERT( retY == SQLITE_OK );
93  Q_UNUSED( retY );
94 
95  createIndexes();
96 
97  if ( xml.hasError() )
98  {
99  mError = QString( "XML error: %1" ).arg( xml.errorString() );
100  return false;
101  }
102 
103  closeDatabase();
104 
105  return true;
106 }
107 
109 {
110  // index on tags for faster access
111  const char* sqlIndexes[] =
112  {
113  "CREATE INDEX nodes_tags_idx ON nodes_tags(id)",
114  "CREATE INDEX ways_tags_idx ON ways_tags(id)",
115  "CREATE INDEX ways_nodes_way ON ways_nodes(way_id)"
116  };
117  int count = sizeof( sqlIndexes ) / sizeof( const char* );
118  for ( int i = 0; i < count; ++i )
119  {
120  int ret = sqlite3_exec( mDatabase, sqlIndexes[i], nullptr, nullptr, nullptr );
121  if ( ret != SQLITE_OK )
122  {
123  mError = "Error creating indexes!";
124  return false;
125  }
126  }
127 
128  return true;
129 }
130 
131 
133 {
134  char **results;
135  int rows, columns;
136  if ( QgsSLConnect::sqlite3_open_v2( mDbFileName.toUtf8().data(), &mDatabase, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr ) != SQLITE_OK )
137  return false;
138 
139  bool above41 = false;
140  int ret = sqlite3_get_table( mDatabase, "select spatialite_version()", &results, &rows, &columns, nullptr );
141  if ( ret == SQLITE_OK && rows == 1 && columns == 1 )
142  {
143  QString version = QString::fromUtf8( results[1] );
144  QStringList parts = version.split( ' ', QString::SkipEmptyParts );
145  if ( parts.size() >= 1 )
146  {
147  QStringList verparts = parts[0].split( '.', QString::SkipEmptyParts );
148  above41 = verparts.size() >= 2 && ( verparts[0].toInt() > 4 || ( verparts[0].toInt() == 4 && verparts[1].toInt() >= 1 ) );
149  }
150  }
151  sqlite3_free_table( results );
152 
153  const char* sqlInitStatements[] =
154  {
155  "PRAGMA cache_size = 100000", // TODO!!!
156  "PRAGMA synchronous = OFF", // TODO!!!
157  above41 ? "SELECT InitSpatialMetadata(1)" : "SELECT InitSpatialMetadata()",
158  "CREATE TABLE nodes ( id INTEGER PRIMARY KEY, lat REAL, lon REAL )",
159  "CREATE TABLE nodes_tags ( id INTEGER, k TEXT, v TEXT )",
160  "CREATE TABLE ways ( id INTEGER PRIMARY KEY )",
161  "CREATE TABLE ways_nodes ( way_id INTEGER, node_id INTEGER, way_pos INTEGER )",
162  "CREATE TABLE ways_tags ( id INTEGER, k TEXT, v TEXT )",
163  };
164 
165  int initCount = sizeof( sqlInitStatements ) / sizeof( const char* );
166  for ( int i = 0; i < initCount; ++i )
167  {
168  char* errMsg;
169  if ( sqlite3_exec( mDatabase, sqlInitStatements[i], nullptr, nullptr, &errMsg ) != SQLITE_OK )
170  {
171  mError = QString( "Error executing SQL command:\n%1\nSQL:\n%2" )
172  .arg( QString::fromUtf8( errMsg ), QString::fromUtf8( sqlInitStatements[i] ) );
173  sqlite3_free( errMsg );
174  closeDatabase();
175  return false;
176  }
177  }
178 
179  const char* sqlInsertStatements[] =
180  {
181  "INSERT INTO nodes ( id, lat, lon ) VALUES (?,?,?)",
182  "INSERT INTO nodes_tags ( id, k, v ) VALUES (?,?,?)",
183  "INSERT INTO ways ( id ) VALUES (?)",
184  "INSERT INTO ways_nodes ( way_id, node_id, way_pos ) VALUES (?,?,?)",
185  "INSERT INTO ways_tags ( id, k, v ) VALUES (?,?,?)"
186  };
187  sqlite3_stmt** sqliteInsertStatements[] =
188  {
189  &mStmtInsertNode,
190  &mStmtInsertNodeTag,
191  &mStmtInsertWay,
192  &mStmtInsertWayNode,
193  &mStmtInsertWayTag
194  };
195  Q_ASSERT( sizeof( sqlInsertStatements ) / sizeof( const char* ) == sizeof( sqliteInsertStatements ) / sizeof( sqlite3_stmt** ) );
196  int insertCount = sizeof( sqlInsertStatements ) / sizeof( const char* );
197 
198  for ( int i = 0; i < insertCount; ++i )
199  {
200  if ( sqlite3_prepare_v2( mDatabase, sqlInsertStatements[i], -1, sqliteInsertStatements[i], nullptr ) != SQLITE_OK )
201  {
202  const char* errMsg = sqlite3_errmsg( mDatabase ); // does not require free
203  mError = QString( "Error preparing SQL command:\n%1\nSQL:\n%2" )
204  .arg( QString::fromUtf8( errMsg ), QString::fromUtf8( sqlInsertStatements[i] ) );
205  closeDatabase();
206  return false;
207  }
208  }
209 
210  return true;
211 }
212 
213 
214 void QgsOSMXmlImport::deleteStatement( sqlite3_stmt*& stmt )
215 {
216  if ( stmt )
217  {
218  sqlite3_finalize( stmt );
219  stmt = nullptr;
220  }
221 }
222 
223 
225 {
226  if ( !mDatabase )
227  return false;
228 
229  deleteStatement( mStmtInsertNode );
230  deleteStatement( mStmtInsertNodeTag );
231  deleteStatement( mStmtInsertWay );
232  deleteStatement( mStmtInsertWayNode );
233  deleteStatement( mStmtInsertWayTag );
234 
235  Q_ASSERT( !mStmtInsertNode );
236 
237  QgsSLConnect::sqlite3_close( mDatabase );
238  mDatabase = nullptr;
239  return true;
240 }
241 
242 
244 {
245  int i = 0;
246  int percent = -1;
247 
248  while ( !xml.atEnd() )
249  {
250  xml.readNext();
251 
252  if ( xml.isEndElement() ) // </osm>
253  break;
254 
255  if ( xml.isStartElement() )
256  {
257  if ( ++i == 500 )
258  {
259  int new_percent = 100 * mInputFile.pos() / mInputFile.size();
260  if ( new_percent > percent )
261  {
262  emit progress( new_percent );
263  percent = new_percent;
264  }
265  i = 0;
266  }
267 
268  if ( xml.name() == "node" )
269  readNode( xml );
270  else if ( xml.name() == "way" )
271  readWay( xml );
272  else
273  xml.skipCurrentElement();
274  }
275  }
276 }
277 
278 
280 {
281  // <node id="2197214" lat="50.0682113" lon="14.4348483" user="viduka" uid="595326" visible="true" version="10" changeset="10714591" timestamp="2012-02-17T19:58:49Z">
282  QXmlStreamAttributes attrs = xml.attributes();
283  QgsOSMId id = attrs.value( "id" ).toString().toLongLong();
284  double lat = attrs.value( "lat" ).toString().toDouble();
285  double lon = attrs.value( "lon" ).toString().toDouble();
286 
287  // insert to DB
288  sqlite3_bind_int64( mStmtInsertNode, 1, id );
289  sqlite3_bind_double( mStmtInsertNode, 2, lat );
290  sqlite3_bind_double( mStmtInsertNode, 3, lon );
291 
292  if ( sqlite3_step( mStmtInsertNode ) != SQLITE_DONE )
293  {
294  xml.raiseError( QString( "Storing node %1 failed." ).arg( id ) );
295  }
296 
297  sqlite3_reset( mStmtInsertNode );
298 
299  while ( !xml.atEnd() )
300  {
301  xml.readNext();
302 
303  if ( xml.isEndElement() ) // </node>
304  break;
305 
306  if ( xml.isStartElement() )
307  {
308  if ( xml.name() == "tag" )
309  readTag( false, id, xml );
310  else
311  xml.raiseError( "Invalid tag in <node>" );
312  }
313  }
314 }
315 
317 {
318  QXmlStreamAttributes attrs = xml.attributes();
319  QByteArray k = attrs.value( "k" ).toString().toUtf8();
320  QByteArray v = attrs.value( "v" ).toString().toUtf8();
321  xml.skipCurrentElement();
322 
323  sqlite3_stmt* stmtInsertTag = way ? mStmtInsertWayTag : mStmtInsertNodeTag;
324 
325  sqlite3_bind_int64( stmtInsertTag, 1, id );
326  sqlite3_bind_text( stmtInsertTag, 2, k.constData(), -1, SQLITE_STATIC );
327  sqlite3_bind_text( stmtInsertTag, 3, v.constData(), -1, SQLITE_STATIC );
328 
329  int res = sqlite3_step( stmtInsertTag );
330  if ( res != SQLITE_DONE )
331  {
332  xml.raiseError( QString( "Storing tag failed [%1]" ).arg( res ) );
333  }
334 
335  sqlite3_reset( stmtInsertTag );
336 }
337 
339 {
340  /*
341  <way id="141756602" user="Vratislav Filler" uid="527259" visible="true" version="1" changeset="10145142" timestamp="2011-12-18T10:43:14Z">
342  <nd ref="318529958"/>
343  <nd ref="1551725779"/>
344  <nd ref="1551725792"/>
345  <nd ref="809695938"/>
346  <nd ref="1551725689"/>
347  <nd ref="809695935"/>
348  <tag k="highway" v="service"/>
349  <tag k="oneway" v="yes"/>
350  </way>
351  */
352  QXmlStreamAttributes attrs = xml.attributes();
353  QgsOSMId id = attrs.value( "id" ).toString().toLongLong();
354 
355  // insert to DB
356  sqlite3_bind_int64( mStmtInsertWay, 1, id );
357 
358  if ( sqlite3_step( mStmtInsertWay ) != SQLITE_DONE )
359  {
360  xml.raiseError( QString( "Storing way %1 failed." ).arg( id ) );
361  }
362 
363  sqlite3_reset( mStmtInsertWay );
364 
365  int way_pos = 0;
366 
367  while ( !xml.atEnd() )
368  {
369  xml.readNext();
370 
371  if ( xml.isEndElement() ) // </way>
372  break;
373 
374  if ( xml.isStartElement() )
375  {
376  if ( xml.name() == "nd" )
377  {
378  QgsOSMId node_id = xml.attributes().value( "ref" ).toString().toLongLong();
379 
380  sqlite3_bind_int64( mStmtInsertWayNode, 1, id );
381  sqlite3_bind_int64( mStmtInsertWayNode, 2, node_id );
382  sqlite3_bind_int( mStmtInsertWayNode, 3, way_pos );
383 
384  if ( sqlite3_step( mStmtInsertWayNode ) != SQLITE_DONE )
385  {
386  xml.raiseError( QString( "Storing ways_nodes %1 - %2 failed." ).arg( id ).arg( node_id ) );
387  }
388 
389  sqlite3_reset( mStmtInsertWayNode );
390 
391  way_pos++;
392 
393  xml.skipCurrentElement();
394  }
395  else if ( xml.name() == "tag" )
396  readTag( true, id, xml );
397  else
398  xml.skipCurrentElement();
399  }
400  }
401 }
bool atEnd() const
QString errorString() const
QgsOSMXmlImport(const QString &xmlFileName=QString(), const QString &dbFileName=QString())
QStringList split(const QString &sep, SplitBehavior behavior, Qt::CaseSensitivity cs) const
QString toString() const
virtual qint64 pos() const
void setFileName(const QString &name)
bool exists() const
QStringRef value(const QString &namespaceUri, const QString &name) const
double toDouble(bool *ok) const
qint64 QgsOSMId
Definition: qgsosmbase.h:10
static int sqlite3_close(sqlite3 *)
void raiseError(const QString &message)
int size() const
void clear()
void skipCurrentElement()
QString fromUtf8(const char *str, int size)
static int sqlite3_open_v2(const char *filename, sqlite3 **ppDb, int flags, const char *zVfs)
const char * constData() const
void readWay(QXmlStreamReader &xml)
TokenType readNext()
virtual bool open(QFlags< QIODevice::OpenModeFlag > mode)
void readTag(bool way, QgsOSMId id, QXmlStreamReader &xml)
void progress(int percent)
virtual qint64 size() const
bool isEndDocument() const
bool isStartElement() const
bool hasError() const
QStringList split(const QString &sep, const QString &str, bool allowEmptyEntries)
char * data()
QXmlStreamAttributes attributes() const
void readNode(QXmlStreamReader &xml)
QStringRef name() const
QString arg(qlonglong a, int fieldWidth, int base, const QChar &fillChar) const
qlonglong toLongLong(bool *ok, int base) const
bool import()
Run import.
void readRoot(QXmlStreamReader &xml)
bool isEndElement() const
void deleteStatement(sqlite3_stmt *&stmt)
QByteArray toUtf8() const