QGIS API Documentation  3.21.0-Master (5b68dc587e)
qgsfileutils.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  qgsfileutils.cpp
3  ---------------------
4  begin : November 2017
5  copyright : (C) 2017 by Etienne Trimaille
6  email : etienne.trimaille at gmail dot com
7  ***************************************************************************
8  * *
9  * This program is free software; you can redistribute it and/or modify *
10  * it under the terms of the GNU General Public License as published by *
11  * the Free Software Foundation; either version 2 of the License, or *
12  * (at your option) any later version. *
13  * *
14  ***************************************************************************/
15 #include "qgsfileutils.h"
16 #include "qgis.h"
17 #include "qgsexception.h"
18 #include "qgsconfig.h"
19 #include "qgsproviderregistry.h"
20 #include "qgsprovidermetadata.h"
21 
22 #include <QObject>
23 #include <QRegularExpression>
24 #include <QFileInfo>
25 #include <QDir>
26 #include <QSet>
27 #include <QDirIterator>
28 
29 #ifdef Q_OS_UNIX
30 // For getrlimit()
31 #include <sys/resource.h>
32 #include <sys/time.h>
33 #endif
34 
35 #ifdef MSVC
36 #include <Windows.h>
37 #include <ShlObj.h>
38 #pragma comment(lib,"Shell32.lib")
39 #endif
40 
41 QString QgsFileUtils::representFileSize( qint64 bytes )
42 {
43  QStringList list;
44  list << QObject::tr( "KB" ) << QObject::tr( "MB" ) << QObject::tr( "GB" ) << QObject::tr( "TB" );
45 
46  QStringListIterator i( list );
47  QString unit = QObject::tr( "B" );
48 
49  double fileSize = bytes;
50  while ( fileSize >= 1024.0 && i.hasNext() )
51  {
52  fileSize /= 1024.0;
53  unit = i.next();
54  }
55  return QStringLiteral( "%1 %2" ).arg( QString::number( fileSize, 'f', bytes >= 1048576 ? 2 : 0 ), unit );
56 }
57 
58 QStringList QgsFileUtils::extensionsFromFilter( const QString &filter )
59 {
60  const QRegularExpression rx( QStringLiteral( "\\*\\.([a-zA-Z0-9]+)" ) );
61  QStringList extensions;
62  QRegularExpressionMatchIterator matches = rx.globalMatch( filter );
63 
64  while ( matches.hasNext() )
65  {
66  const QRegularExpressionMatch match = matches.next();
67  if ( match.hasMatch() )
68  {
69  QStringList newExtensions = match.capturedTexts();
70  newExtensions.pop_front(); // remove whole match
71  extensions.append( newExtensions );
72  }
73  }
74  return extensions;
75 }
76 
77 QString QgsFileUtils::wildcardsFromFilter( const QString &filter )
78 {
79  const QRegularExpression globPatternsRx( QStringLiteral( ".*\\((.*?)\\)$" ) );
80  const QRegularExpressionMatch matches = globPatternsRx.match( filter );
81  if ( matches.hasMatch() )
82  return matches.captured( 1 );
83  else
84  return QString();
85 }
86 
87 bool QgsFileUtils::fileMatchesFilter( const QString &fileName, const QString &filter )
88 {
89  QFileInfo fi( fileName );
90  const QString name = fi.fileName();
91  const QStringList parts = filter.split( QStringLiteral( ";;" ) );
92  for ( const QString &part : parts )
93  {
94 #if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
95  const QStringList globPatterns = wildcardsFromFilter( part ).split( ' ', QString::SkipEmptyParts );
96 #else
97  const QStringList globPatterns = wildcardsFromFilter( part ).split( ' ', Qt::SkipEmptyParts );
98 #endif
99  for ( const QString &glob : globPatterns )
100  {
101  const QString re = QRegularExpression::wildcardToRegularExpression( glob );
102 
103  const QRegularExpression globRx( re );
104  if ( globRx.match( name ).hasMatch() )
105  return true;
106  }
107  }
108  return false;
109 }
110 
111 QString QgsFileUtils::ensureFileNameHasExtension( const QString &f, const QStringList &extensions )
112 {
113  if ( extensions.empty() || f.isEmpty() )
114  return f;
115 
116  QString fileName = f;
117  bool hasExt = false;
118  for ( const QString &extension : std::as_const( extensions ) )
119  {
120  const QString extWithDot = extension.startsWith( '.' ) ? extension : '.' + extension;
121  if ( fileName.endsWith( extWithDot, Qt::CaseInsensitive ) )
122  {
123  hasExt = true;
124  break;
125  }
126  }
127 
128  if ( !hasExt )
129  {
130  const QString extension = extensions.at( 0 );
131  const QString extWithDot = extension.startsWith( '.' ) ? extension : '.' + extension;
132  fileName += extWithDot;
133  }
134 
135  return fileName;
136 }
137 
138 QString QgsFileUtils::addExtensionFromFilter( const QString &fileName, const QString &filter )
139 {
140  const QStringList extensions = extensionsFromFilter( filter );
141  return ensureFileNameHasExtension( fileName, extensions );
142 }
143 
144 QString QgsFileUtils::stringToSafeFilename( const QString &string )
145 {
146  QRegularExpression rx( QStringLiteral( "[/\\\\\\?%\\*\\:\\|\"<>]" ) );
147  QString s = string;
148  s.replace( rx, QStringLiteral( "_" ) );
149  return s;
150 }
151 
152 QString QgsFileUtils::findClosestExistingPath( const QString &path )
153 {
154  if ( path.isEmpty() )
155  return QString();
156 
157  QDir currentPath;
158  QFileInfo fi( path );
159  if ( fi.isFile() )
160  currentPath = fi.dir();
161  else
162  currentPath = QDir( path );
163 
164  QSet< QString > visited;
165  while ( !currentPath.exists() )
166  {
167  const QString parentPath = QDir::cleanPath( currentPath.absolutePath() + QStringLiteral( "/.." ) );
168  if ( visited.contains( parentPath ) )
169  return QString(); // break circular links
170 
171  if ( parentPath.isEmpty() || parentPath == QLatin1String( "." ) )
172  return QString();
173  currentPath = QDir( parentPath );
174  visited << parentPath;
175  }
176 
177  const QString res = QDir::cleanPath( currentPath.absolutePath() );
178 
179  if ( res == QDir::currentPath() )
180  return QString(); // avoid default to binary folder if a filename alone is specified
181 
182  return res == QLatin1String( "." ) ? QString() : res;
183 }
184 
185 QStringList QgsFileUtils::findFile( const QString &file, const QString &basePath, int maxClimbs, int searchCeilling, const QString &currentDir )
186 {
187  int depth = 0;
188  QString originalFolder;
189  QDir folder;
190  const QString fileName( basePath.isEmpty() ? QFileInfo( file ).fileName() : file );
191  const QString baseFolder( basePath.isEmpty() ? QFileInfo( file ).path() : basePath );
192 
193  if ( QFileInfo( baseFolder ).isDir() )
194  {
195  folder = QDir( baseFolder ) ;
196  originalFolder = folder.absolutePath();
197  }
198  else // invalid folder or file path
199  {
200  folder = QDir( QFileInfo( baseFolder ).absolutePath() );
201  originalFolder = folder.absolutePath();
202  }
203 
204  QStringList searchedFolder = QStringList();
205  QString existingBase;
206  QString backupDirectory = QDir::currentPath();
207  QStringList foundFiles;
208 
209  if ( !currentDir.isEmpty() && backupDirectory != currentDir && QDir( currentDir ).exists() )
210  QDir::setCurrent( currentDir );
211 
212  // find the nearest existing folder
213  while ( !folder.exists() && folder.absolutePath().count( '/' ) > searchCeilling )
214  {
215 
216  existingBase = folder.path();
217  if ( !folder.cdUp() )
218  folder = QFileInfo( existingBase ).absoluteDir(); // using fileinfo to move up one level
219 
220  depth += 1;
221 
222  if ( depth > ( maxClimbs + 4 ) ) //break early when no folders can be found
223  break;
224  }
225  bool folderExists = folder.exists();
226 
227  if ( depth > maxClimbs )
228  maxClimbs = depth;
229 
230  if ( folder.absolutePath().count( '/' ) < searchCeilling )
231  searchCeilling = folder.absolutePath().count( '/' ) - 1;
232 
233  while ( depth <= maxClimbs && folderExists && folder.absolutePath().count( '/' ) >= searchCeilling )
234  {
235 
236  QDirIterator localFinder( folder.path(), QStringList() << fileName, QDir::Files, QDirIterator::NoIteratorFlags );
237  searchedFolder.append( folder.absolutePath() );
238  if ( localFinder.hasNext() )
239  {
240  foundFiles << localFinder.next();
241  return foundFiles;
242  }
243 
244 
245  const QFileInfoList subdirs = folder.entryInfoList( QDir::AllDirs );
246  for ( const QFileInfo &subdir : subdirs )
247  {
248  if ( ! searchedFolder.contains( subdir.absolutePath() ) )
249  {
250  QDirIterator subDirFinder( subdir.path(), QStringList() << fileName, QDir::Files, QDirIterator::Subdirectories );
251  if ( subDirFinder.hasNext() )
252  {
253  QString possibleFile = subDirFinder.next();
254  if ( !subDirFinder.hasNext() )
255  {
256  foundFiles << possibleFile;
257  return foundFiles;
258  }
259 
260  foundFiles << possibleFile;
261  while ( subDirFinder.hasNext() )
262  {
263  foundFiles << subDirFinder.next();
264  }
265  return foundFiles;
266  }
267  }
268  }
269  depth += 1;
270 
271  if ( depth > maxClimbs )
272  break;
273 
274  folderExists = folder.cdUp();
275  }
276 
277  if ( QDir::currentPath() == currentDir && currentDir != backupDirectory )
278  QDir::setCurrent( backupDirectory );
279 
280  return foundFiles;
281 }
282 
283 #ifdef MSVC
284 std::unique_ptr< wchar_t[] > pathToWChar( const QString &path )
285 {
286  const QString nativePath = QDir::toNativeSeparators( path );
287 
288  std::unique_ptr< wchar_t[] > pathArray( new wchar_t[static_cast< uint>( nativePath.length() + 1 )] );
289  nativePath.toWCharArray( pathArray.get() );
290  pathArray[static_cast< size_t >( nativePath.length() )] = 0;
291  return pathArray;
292 }
293 #endif
294 
296 {
297 #ifdef MSVC
298  auto pathType = [ = ]( const QString & path ) -> DriveType
299  {
300  std::unique_ptr< wchar_t[] > pathArray = pathToWChar( path );
301  const UINT type = GetDriveTypeW( pathArray.get() );
302  switch ( type )
303  {
304  case DRIVE_UNKNOWN:
306 
307  case DRIVE_NO_ROOT_DIR:
309 
310  case DRIVE_REMOVABLE:
312 
313  case DRIVE_FIXED:
314  return Qgis::DriveType::Fixed;
315 
316  case DRIVE_REMOTE:
318 
319  case DRIVE_CDROM:
320  return Qgis::DriveType::CdRom;
321 
322  case DRIVE_RAMDISK:
324  }
325 
326  return Unknown;
327 
328  };
329 
330  const QString originalPath = QDir::cleanPath( path );
331  QString currentPath = originalPath;
332  QString prevPath;
333  while ( currentPath != prevPath )
334  {
335  prevPath = currentPath;
336  currentPath = QFileInfo( currentPath ).path();
337  const DriveType type = pathType( currentPath );
338  if ( type != Unknown && type != Invalid )
339  return type;
340  }
341  return Unknown;
342 
343 #else
344  ( void )path;
345  throw QgsNotSupportedException( QStringLiteral( "Determining drive type is not supported on this platform" ) );
346 #endif
347 }
348 
349 bool QgsFileUtils::pathIsSlowDevice( const QString &path )
350 {
351 #ifdef ENABLE_TESTS
352  if ( path.contains( QLatin1String( "fake_slow_path_for_unit_tests" ) ) )
353  return true;
354 #endif
355 
356  try
357  {
358  const Qgis::DriveType type = driveType( path );
359  switch ( type )
360  {
365  return false;
366 
370  return true;
371  }
372  }
373  catch ( QgsNotSupportedException & )
374  {
375 
376  }
377  return false;
378 }
379 
380 QSet<QString> QgsFileUtils::sidecarFilesForPath( const QString &path )
381 {
382  QSet< QString > res;
383  const QStringList providers = QgsProviderRegistry::instance()->providerList();
384  for ( const QString &provider : providers )
385  {
386  const QgsProviderMetadata *metadata = QgsProviderRegistry::instance()->providerMetadata( provider );
388  {
389  const QStringList possibleSidecars = metadata->sidecarFilesForUri( path );
390  for ( const QString &possibleSidecar : possibleSidecars )
391  {
392  if ( QFile::exists( possibleSidecar ) )
393  res.insert( possibleSidecar );
394  }
395  }
396  }
397  return res;
398 }
399 
400 bool QgsFileUtils::renameDataset( const QString &oldPath, const QString &newPath, QString &error, Qgis::FileOperationFlags flags )
401 {
402  if ( !QFile::exists( oldPath ) )
403  {
404  error = QObject::tr( "File does not exist" );
405  return false;
406  }
407 
408  const QFileInfo oldPathInfo( oldPath );
409  QSet< QString > sidecars = sidecarFilesForPath( oldPath );
411  {
412  const QString qmdPath = oldPathInfo.dir().filePath( oldPathInfo.completeBaseName() + QStringLiteral( ".qmd" ) );
413  if ( QFile::exists( qmdPath ) )
414  sidecars.insert( qmdPath );
415  }
417  {
418  const QString qmlPath = oldPathInfo.dir().filePath( oldPathInfo.completeBaseName() + QStringLiteral( ".qml" ) );
419  if ( QFile::exists( qmlPath ) )
420  sidecars.insert( qmlPath );
421  }
422 
423  const QFileInfo newPathInfo( newPath );
424 
425  bool res = true;
426  QStringList errors;
427  errors.reserve( sidecars.size() );
428  // first check if all sidecars CAN be renamed -- we don't want to get partly through the rename and then find a clash
429  for ( const QString &sidecar : std::as_const( sidecars ) )
430  {
431  const QFileInfo sidecarInfo( sidecar );
432  const QString newSidecarName = newPathInfo.dir().filePath( newPathInfo.completeBaseName() + '.' + sidecarInfo.suffix() );
433  if ( newSidecarName != sidecar && QFile::exists( newSidecarName ) )
434  {
435  res = false;
436  errors.append( QDir::toNativeSeparators( newSidecarName ) );
437  }
438  }
439  if ( !res )
440  {
441  error = QObject::tr( "Destination files already exist %1" ).arg( errors.join( QStringLiteral( ", " ) ) );
442  return false;
443  }
444 
445  if ( !QFile::rename( oldPath, newPath ) )
446  {
447  error = QObject::tr( "Could not rename %1" ).arg( QDir::toNativeSeparators( oldPath ) );
448  return false;
449  }
450 
451  for ( const QString &sidecar : std::as_const( sidecars ) )
452  {
453  const QFileInfo sidecarInfo( sidecar );
454  const QString newSidecarName = newPathInfo.dir().filePath( newPathInfo.completeBaseName() + '.' + sidecarInfo.suffix() );
455  if ( newSidecarName == sidecar )
456  continue;
457 
458  if ( !QFile::rename( sidecar, newSidecarName ) )
459  {
460  errors.append( QDir::toNativeSeparators( sidecar ) );
461  res = false;
462  }
463  }
464  if ( !res )
465  {
466  error = QObject::tr( "Could not rename %1" ).arg( errors.join( QStringLiteral( ", " ) ) );
467  }
468 
469  return res;
470 }
471 
473 {
474 #ifdef Q_OS_UNIX
475  struct rlimit rescLimit;
476  if ( getrlimit( RLIMIT_NOFILE, &rescLimit ) == 0 )
477  {
478  return rescLimit.rlim_cur;
479  }
480 #endif
481  return -1;
482 }
483 
485 {
486 #ifdef Q_OS_LINUX
487  int res = static_cast<int>( QDir( "/proc/self/fd" ).entryList().size() );
488  if ( res == 0 )
489  res = -1;
490  return res;
491 #else
492  return -1;
493 #endif
494 }
495 
496 bool QgsFileUtils::isCloseToLimitOfOpenedFiles( int filesToBeOpened )
497 {
498  const int nFileLimit = QgsFileUtils::openedFileLimit();
499  const int nFileCount = QgsFileUtils::openedFileCount();
500  // We need some margin as Qt will crash if it cannot create some file descriptors
501  constexpr int SOME_MARGIN = 20;
502  return nFileCount > 0 && nFileLimit > 0 && nFileCount + filesToBeOpened > nFileLimit - SOME_MARGIN;
503 }
DriveType
Drive types.
Definition: qgis.h:402
@ Fixed
Fixed drive.
@ Invalid
Invalid path.
@ Unknown
Unknown type.
@ RamDisk
RAM disk.
@ Removable
Removable drive.
@ Remote
Remote drive.
@ IncludeMetadataFile
Indicates that any associated .qmd metadata file should be included with the operation.
@ IncludeStyleFile
Indicates that any associated .qml styling file should be included with the operation.
static QString stringToSafeFilename(const QString &string)
Converts a string to a safe filename, replacing characters which are not safe for filenames with an '...
static QStringList findFile(const QString &file, const QString &basepath=QString(), int maxClimbs=4, int searchCeiling=4, const QString &currentDir=QString())
Will check basepath in an outward spiral up to maxClimbs levels to check if file exists.
static int openedFileCount()
Returns the number of currently opened files by the process.
static QString wildcardsFromFilter(const QString &filter)
Given a filter string like "GeoTIFF Files (*.tiff *.tif)", extracts the wildcard portion of this filt...
static bool renameDataset(const QString &oldPath, const QString &newPath, QString &error, Qgis::FileOperationFlags flags=Qgis::FileOperationFlag::IncludeMetadataFile|Qgis::FileOperationFlag::IncludeStyleFile)
Renames the dataset at oldPath to newPath, renaming both the file at oldPath and all associated sidec...
static QSet< QString > sidecarFilesForPath(const QString &path)
Returns a list of the sidecar files which exist for the dataset a the specified path.
static bool pathIsSlowDevice(const QString &path)
Returns true if the specified path is assumed to reside on a slow device, e.g.
static bool isCloseToLimitOfOpenedFiles(int filesToBeOpened=1)
Returns whether when opening new file(s) will reach, or nearly reach, the limit of simultaneously ope...
static bool fileMatchesFilter(const QString &fileName, const QString &filter)
Returns true if the given fileName matches a file filter string.
static Qgis::DriveType driveType(const QString &path) SIP_THROW(QgsNotSupportedException)
Returns the drive type for the given path.
static QString ensureFileNameHasExtension(const QString &fileName, const QStringList &extensions)
Ensures that a fileName ends with an extension from the provided list of extensions.
static QString representFileSize(qint64 bytes)
Returns the human size from bytes.
static QString addExtensionFromFilter(const QString &fileName, const QString &filter)
Ensures that a fileName ends with an extension from the specified filter string.
static int openedFileLimit()
Returns the limit of simultaneously opened files by the process.
static QString findClosestExistingPath(const QString &path)
Returns the top-most existing folder from path.
static QStringList extensionsFromFilter(const QString &filter)
Returns a list of the extensions contained within a file filter string.
Custom exception class which is raised when an operation is not supported.
Definition: qgsexception.h:118
Holds data provider key, description, and associated shared library file or function pointer informat...
virtual QgsProviderMetadata::ProviderCapabilities providerCapabilities() const
Returns the provider's capabilities.
@ FileBasedUris
Indicates that the provider can utilize URIs which are based on paths to files (as opposed to databas...
virtual QStringList sidecarFilesForUri(const QString &uri) const
Given a uri, returns any sidecar files which are associated with the URI and this provider.
static QgsProviderRegistry * instance(const QString &pluginPath=QString())
Means of accessing canonical single instance.
QStringList providerList() const
Returns list of available providers by their keys.
QgsProviderMetadata * providerMetadata(const QString &providerKey) const
Returns metadata of the provider or nullptr if not found.