QGIS API Documentation  3.21.0-Master (5b68dc587e)
qgsabstractcontentcache.h
Go to the documentation of this file.
1 /***************************************************************************
2  qgsabstractcontentcache.h
3  ---------------
4  begin : December 2018
5  copyright : (C) 2018 by Nyall Dawson
6  email : nyall dot dawson at gmail dot com
7  ***************************************************************************/
8 
9 /***************************************************************************
10  * *
11  * This program is free software; you can redistribute it and/or modify *
12  * it under the terms of the GNU General Public License as published by *
13  * the Free Software Foundation; either version 2 of the License, or *
14  * (at your option) any later version. *
15  * *
16  ***************************************************************************/
17 
18 #ifndef QGSABSTRACTCONTENTCACHE_H
19 #define QGSABSTRACTCONTENTCACHE_H
20 
21 #include "qgis_core.h"
22 #include "qgis_sip.h"
23 #include "qgslogger.h"
24 #include "qgsmessagelog.h"
25 #include "qgsapplication.h"
28 
29 #include <QObject>
30 #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0)
31 #include <QMutex>
32 #else
33 #include <QRecursiveMutex>
34 #endif
35 #include <QCache>
36 #include <QSet>
37 #include <QDateTime>
38 #include <QList>
39 #include <QFile>
40 #include <QNetworkReply>
41 #include <QFileInfo>
42 #include <QUrl>
43 
55 {
56  public:
57 
61  QgsAbstractContentCacheEntry( const QString &path ) ;
62 
63  virtual ~QgsAbstractContentCacheEntry() = default;
64 
69 
73  QString path;
74 
76  QDateTime fileModified;
77 
80 
82  int mFileModifiedCheckTimeout = 30000;
83 
88  QgsAbstractContentCacheEntry *nextEntry = nullptr;
89 
94  QgsAbstractContentCacheEntry *previousEntry = nullptr;
95 
96  bool operator==( const QgsAbstractContentCacheEntry &other ) const
97  {
98  return other.path == path;
99  }
100 
104  virtual int dataSize() const = 0;
105 
109  virtual void dump() const = 0;
110 
111  protected:
112 
118  virtual bool isEqual( const QgsAbstractContentCacheEntry *other ) const = 0;
119 
120  private:
121 #ifdef SIP_RUN
123 #endif
124 
125 };
126 
137 class CORE_EXPORT QgsAbstractContentCacheBase: public QObject
138 {
139  Q_OBJECT
140 
141  public:
142 
146  QgsAbstractContentCacheBase( QObject *parent );
147 
148  signals:
149 
153  void remoteContentFetched( const QString &url );
154 
155  protected:
156 
161  virtual bool checkReply( QNetworkReply *reply, const QString &path ) const
162  {
163  Q_UNUSED( reply )
164  Q_UNUSED( path )
165  return true;
166  }
167 
168  protected slots:
169 
176  virtual void onRemoteContentFetched( const QString &url, bool success );
177 
178 };
179 
180 #ifndef SIP_RUN
181 
195 template<class T>
197 {
198 
199  public:
200 
212  QgsAbstractContentCache( QObject *parent SIP_TRANSFERTHIS = nullptr,
213  const QString &typeString = QString(),
214  long maxCacheSize = 20000000,
215  int fileModifiedCheckTimeout = 30000 )
216  : QgsAbstractContentCacheBase( parent )
217 #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0)
218  , mMutex( QMutex::Recursive )
219 #endif
220  , mMaxCacheSize( maxCacheSize )
221  , mFileModifiedCheckTimeout( fileModifiedCheckTimeout )
222  , mTypeString( typeString.isEmpty() ? QObject::tr( "Content" ) : typeString )
223  {
224  }
225 
227  {
228  qDeleteAll( mEntryLookup );
229  }
230 
231  protected:
232 
237  {
238  //only one entry in cache
239  if ( mLeastRecentEntry == mMostRecentEntry )
240  {
241  return;
242  }
243  T *entry = mLeastRecentEntry;
244  while ( entry && ( mTotalSize > mMaxCacheSize ) )
245  {
246  T *bkEntry = entry;
247  entry = static_cast< T * >( entry->nextEntry );
248 
249  takeEntryFromList( bkEntry );
250  mEntryLookup.remove( bkEntry->path, bkEntry );
251  mTotalSize -= bkEntry->dataSize();
252  delete bkEntry;
253  }
254  }
255 
269  QByteArray getContent( const QString &path, const QByteArray &missingContent, const QByteArray &fetchingContent, bool blocking = false ) const
270  {
271  // is it a path to local file?
272  QFile file( path );
273  if ( file.exists() )
274  {
275  if ( file.open( QIODevice::ReadOnly ) )
276  {
277  return file.readAll();
278  }
279  else
280  {
281  return missingContent;
282  }
283  }
284 
285  // maybe it's an embedded base64 string
286  if ( path.startsWith( QLatin1String( "base64:" ), Qt::CaseInsensitive ) )
287  {
288  const QByteArray base64 = path.mid( 7 ).toLocal8Bit(); // strip 'base64:' prefix
289  return QByteArray::fromBase64( base64, QByteArray::OmitTrailingEquals );
290  }
291 
292  // maybe it's a url...
293  if ( !path.contains( QLatin1String( "://" ) ) ) // otherwise short, relative SVG paths might be considered URLs
294  {
295  return missingContent;
296  }
297 
298  const QUrl url( path );
299  if ( !url.isValid() )
300  {
301  return missingContent;
302  }
303 
304  // check whether it's a url pointing to a local file
305  if ( url.scheme().compare( QLatin1String( "file" ), Qt::CaseInsensitive ) == 0 )
306  {
307  file.setFileName( url.toLocalFile() );
308  if ( file.exists() )
309  {
310  if ( file.open( QIODevice::ReadOnly ) )
311  {
312  return file.readAll();
313  }
314  }
315 
316  // not found...
317  return missingContent;
318  }
319 
320  const QMutexLocker locker( &mMutex );
321 
322  // already a request in progress for this url
323  if ( mPendingRemoteUrls.contains( path ) )
324  {
325  // it's a non blocking request so return fetching content
326  if ( !blocking )
327  {
328  return fetchingContent;
329  }
330 
331  // it's a blocking request so try to find the task and wait for task finished
332  const auto constActiveTasks = QgsApplication::taskManager()->activeTasks();
333  for ( QgsTask *task : constActiveTasks )
334  {
335  // the network content fetcher task's description ends with the path
336  if ( !task->description().endsWith( path ) )
337  {
338  continue;
339  }
340 
341  // cast task to network content fetcher task
342  QgsNetworkContentFetcherTask *ncfTask = qobject_cast<QgsNetworkContentFetcherTask *>( task );
343  if ( ncfTask )
344  {
345  // wait for task finished
346  if ( waitForTaskFinished( ncfTask ) )
347  {
348  if ( mRemoteContentCache.contains( path ) )
349  {
350  // We got the file!
351  return *mRemoteContentCache[ path ];
352  }
353  }
354  }
355  // task found, no needs to continue
356  break;
357  }
358  // if no content returns the content is probably in remote content cache
359  // or a new task will be created
360  }
361 
362  if ( mRemoteContentCache.contains( path ) )
363  {
364  // already fetched this content - phew. Just return what we already got.
365  return *mRemoteContentCache[ path ];
366  }
367 
368  mPendingRemoteUrls.insert( path );
369  //fire up task to fetch content in background
370  QNetworkRequest request( url );
371  QgsSetRequestInitiatorClass( request, QStringLiteral( "QgsAbstractContentCache<%1>" ).arg( mTypeString ) );
372  request.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
373  request.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );
374 
376  connect( task, &QgsNetworkContentFetcherTask::fetched, this, [this, task, path, missingContent]
377  {
378  const QMutexLocker locker( &mMutex );
379 
380  QNetworkReply *reply = task->reply();
381  if ( !reply )
382  {
383  // canceled
384  QMetaObject::invokeMethod( const_cast< QgsAbstractContentCacheBase * >( qobject_cast< const QgsAbstractContentCacheBase * >( this ) ), "onRemoteContentFetched", Qt::QueuedConnection, Q_ARG( QString, path ), Q_ARG( bool, false ) );
385  return;
386  }
387 
388  if ( reply->error() != QNetworkReply::NoError )
389  {
390  QgsMessageLog::logMessage( tr( "%3 request failed [error: %1 - url: %2]" ).arg( reply->errorString(), path, mTypeString ), mTypeString );
391  return;
392  }
393 
394  bool ok = true;
395 
396  const QVariant status = reply->attribute( QNetworkRequest::HttpStatusCodeAttribute );
397  if ( !status.isNull() && status.toInt() >= 400 )
398  {
399  const QVariant phrase = reply->attribute( QNetworkRequest::HttpReasonPhraseAttribute );
400  QgsMessageLog::logMessage( tr( "%4 request error [status: %1 - reason phrase: %2] for %3" ).arg( status.toInt() ).arg( phrase.toString(), path, mTypeString ), mTypeString );
401  mRemoteContentCache.insert( path, new QByteArray( missingContent ) );
402  ok = false;
403  }
404 
405  if ( !checkReply( reply, path ) )
406  {
407  mRemoteContentCache.insert( path, new QByteArray( missingContent ) );
408  ok = false;
409  }
410 
411  if ( ok )
412  {
413  // read the content data
414  const QByteArray ba = reply->readAll();
415 
416  // because of the fragility listed below in waitForTaskFinished, this slot may get called twice. In that case
417  // the second time will have an empty reply (we've already read it all...)
418  if ( !ba.isEmpty() )
419  mRemoteContentCache.insert( path, new QByteArray( ba ) );
420  }
421  QMetaObject::invokeMethod( const_cast< QgsAbstractContentCacheBase * >( qobject_cast< const QgsAbstractContentCacheBase * >( this ) ), "onRemoteContentFetched", Qt::QueuedConnection, Q_ARG( QString, path ), Q_ARG( bool, true ) );
422  } );
423 
425 
426  // if blocking, wait for finished
427  if ( blocking )
428  {
429  if ( waitForTaskFinished( task ) )
430  {
431  if ( mRemoteContentCache.contains( path ) )
432  {
433  // We got the file!
434  return *mRemoteContentCache[ path ];
435  }
436  }
437  }
438  return fetchingContent;
439  }
440 
441  void onRemoteContentFetched( const QString &url, bool success ) override
442  {
443  const QMutexLocker locker( &mMutex );
444  mPendingRemoteUrls.remove( url );
445 
446  T *nextEntry = mLeastRecentEntry;
447  while ( T *entry = nextEntry )
448  {
449  nextEntry = static_cast< T * >( entry->nextEntry );
450  if ( entry->path == url )
451  {
452  takeEntryFromList( entry );
453  mEntryLookup.remove( entry->path, entry );
454  mTotalSize -= entry->dataSize();
455  delete entry;
456  }
457  }
458 
459  if ( success )
460  emit remoteContentFetched( url );
461  }
462 
474  {
475  // Wait up to timeout seconds for task finished
477  {
478  // The wait did not time out
479  // Third step, check status as complete
480  if ( task->status() == QgsTask::Complete )
481  {
482  // Fourth step, force the signal fetched to be sure reply has been checked
483 
484  // ARGH this is BAD BAD BAD. The connection will get called twice as a result!!!
485  task->fetched();
486  return true;
487  }
488  }
489  return false;
490  }
491 
501  T *findExistingEntry( T *entryTemplate )
502  {
503  //search entries in mEntryLookup
504  const QString path = entryTemplate->path;
505  T *currentEntry = nullptr;
506  const QList<T *> entries = mEntryLookup.values( path );
507  QDateTime modified;
508  for ( T *cacheEntry : entries )
509  {
510  if ( cacheEntry->isEqual( entryTemplate ) )
511  {
512  if ( mFileModifiedCheckTimeout <= 0 || cacheEntry->fileModifiedLastCheckTimer.hasExpired( mFileModifiedCheckTimeout ) )
513  {
514  if ( !modified.isValid() )
515  modified = QFileInfo( path ).lastModified();
516 
517  if ( cacheEntry->fileModified != modified )
518  continue;
519  else
520  cacheEntry->fileModifiedLastCheckTimer.restart();
521  }
522  currentEntry = cacheEntry;
523  break;
524  }
525  }
526 
527  //if not found: insert entryTemplate as a new entry
528  if ( !currentEntry )
529  {
530  currentEntry = insertCacheEntry( entryTemplate );
531  }
532  else
533  {
534  delete entryTemplate;
535  entryTemplate = nullptr;
536  ( void )entryTemplate;
537  takeEntryFromList( currentEntry );
538  if ( !mMostRecentEntry ) //list is empty
539  {
540  mMostRecentEntry = currentEntry;
541  mLeastRecentEntry = currentEntry;
542  }
543  else
544  {
545  mMostRecentEntry->nextEntry = currentEntry;
546  currentEntry->previousEntry = mMostRecentEntry;
547  currentEntry->nextEntry = nullptr;
548  mMostRecentEntry = currentEntry;
549  }
550  }
551 
552  //debugging
553  //printEntryList();
554 
555  return currentEntry;
556  }
557 #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0)
558  mutable QMutex mMutex;
559 #else
560  mutable QRecursiveMutex mMutex;
561 #endif
563  long mTotalSize = 0;
564 
566  long mMaxCacheSize = 20000000;
567 
568  private:
569 
575  T *insertCacheEntry( T *entry )
576  {
577  entry->mFileModifiedCheckTimeout = mFileModifiedCheckTimeout;
578 
579  if ( !entry->path.startsWith( QLatin1String( "base64:" ) ) )
580  {
581  entry->fileModified = QFileInfo( entry->path ).lastModified();
582  entry->fileModifiedLastCheckTimer.start();
583  }
584 
585  mEntryLookup.insert( entry->path, entry );
586 
587  //insert to most recent place in entry list
588  if ( !mMostRecentEntry ) //inserting first entry
589  {
590  mLeastRecentEntry = entry;
591  mMostRecentEntry = entry;
592  entry->previousEntry = nullptr;
593  entry->nextEntry = nullptr;
594  }
595  else
596  {
597  entry->previousEntry = mMostRecentEntry;
598  entry->nextEntry = nullptr;
599  mMostRecentEntry->nextEntry = entry;
600  mMostRecentEntry = entry;
601  }
602 
603  trimToMaximumSize();
604  return entry;
605  }
606 
607 
611  void takeEntryFromList( T *entry )
612  {
613  if ( !entry )
614  {
615  return;
616  }
617 
618  if ( entry->previousEntry )
619  {
620  entry->previousEntry->nextEntry = entry->nextEntry;
621  }
622  else
623  {
624  mLeastRecentEntry = static_cast< T * >( entry->nextEntry );
625  }
626  if ( entry->nextEntry )
627  {
628  entry->nextEntry->previousEntry = entry->previousEntry;
629  }
630  else
631  {
632  mMostRecentEntry = static_cast< T * >( entry->previousEntry );
633  }
634  }
635 
639  void printEntryList()
640  {
641  QgsDebugMsg( QStringLiteral( "****************cache entry list*************************" ) );
642  QgsDebugMsg( "Cache size: " + QString::number( mTotalSize ) );
643  T *entry = mLeastRecentEntry;
644  while ( entry )
645  {
646  QgsDebugMsg( QStringLiteral( "***Entry:" ) );
647  entry->dump();
648  entry = entry->nextEntry;
649  }
650  }
651 
653  QMultiHash< QString, T * > mEntryLookup;
654 
656  int mFileModifiedCheckTimeout = 30000;
657 
658  //The content cache keeps the entries on a double connected list, moving the current entry to the front.
659  //That way, removing entries for more space can start with the least used objects.
660  T *mLeastRecentEntry = nullptr;
661  T *mMostRecentEntry = nullptr;
662 
663  mutable QCache< QString, QByteArray > mRemoteContentCache;
664  mutable QSet< QString > mPendingRemoteUrls;
665 
666  QString mTypeString;
667 
668  friend class TestQgsSvgCache;
669  friend class TestQgsImageCache;
670 };
671 
672 #endif
673 
674 #endif // QGSABSTRACTCONTENTCACHE_H
A QObject derived base class for QgsAbstractContentCache.
void remoteContentFetched(const QString &url)
Emitted when the cache has finished retrieving content from a remote url.
virtual bool checkReply(QNetworkReply *reply, const QString &path) const
Runs additional checks on a network reply to ensure that the reply content is consistent with that re...
Base class for entries in a QgsAbstractContentCache.
virtual int dataSize() const =0
Returns the memory usage in bytes for the entry.
virtual void dump() const =0
Dumps debugging strings containing the item's properties.
virtual ~QgsAbstractContentCacheEntry()=default
QElapsedTimer fileModifiedLastCheckTimer
Time since last check of file modified date.
QgsAbstractContentCacheEntry(const QgsAbstractContentCacheEntry &rh)=delete
QgsAbstractContentCacheEntry cannot be copied.
QgsAbstractContentCacheEntry & operator=(const QgsAbstractContentCacheEntry &rh)=delete
QgsAbstractContentCacheEntry cannot be copied.
QString path
Represents the absolute path to a file, a remote URL, or a base64 encoded string.
virtual bool isEqual(const QgsAbstractContentCacheEntry *other) const =0
Tests whether this entry matches another entry.
QDateTime fileModified
Timestamp when file was last modified.
bool operator==(const QgsAbstractContentCacheEntry &other) const
Abstract base class for file content caches, such as SVG or raster image caches.
T * findExistingEntry(T *entryTemplate)
Returns the existing entry from the cache which matches entryTemplate (deleting entryTemplate when do...
QByteArray getContent(const QString &path, const QByteArray &missingContent, const QByteArray &fetchingContent, bool blocking=false) const
Gets the file content corresponding to the given path.
void onRemoteContentFetched(const QString &url, bool success) override
Triggered after remote content (i.e.
QgsAbstractContentCache(QObject *parent=nullptr, const QString &typeString=QString(), long maxCacheSize=20000000, int fileModifiedCheckTimeout=30000)
Constructor for QgsAbstractContentCache, with the specified parent object.
void trimToMaximumSize()
Removes the least used cache entries until the maximum cache size is under the predefined size limit.
bool waitForTaskFinished(QgsNetworkContentFetcherTask *task) const
Blocks the current thread until the task finishes (or user's preset network timeout expires)
static QgsTaskManager * taskManager()
Returns the application's task manager, used for managing application wide background task handling.
static void logMessage(const QString &message, const QString &tag=QString(), Qgis::MessageLevel level=Qgis::MessageLevel::Warning, bool notifyUser=true)
Adds a message to the log instance (and creates it if necessary).
static int timeout()
Returns the network timeout length, in milliseconds.
Handles HTTP network content fetching in a background task.
void fetched()
Emitted when the network content has been fetched, regardless of whether the fetch was successful or ...
QNetworkReply * reply()
Returns the network reply.
QList< QgsTask * > activeTasks() const
Returns a list of the active (queued or running) tasks.
long addTask(QgsTask *task, int priority=0)
Adds a task to the manager.
Abstract base class for long running background tasks.
TaskStatus status() const
Returns the current task status.
@ Complete
Task successfully completed.
bool waitForFinished(int timeout=30000)
Blocks the current thread until the task finishes or a maximum of timeout milliseconds.
#define SIP_TRANSFERTHIS
Definition: qgis_sip.h:53
#define QgsDebugMsg(str)
Definition: qgslogger.h:38
#define QgsSetRequestInitiatorClass(request, _class)