Logo Search packages:      
Sourcecode: earth3d version File versions  Download package

fileCache.cpp

#include "fileCache.h"
#include "md5.h"
#include <qdir.h>
#include <qfile.h>
#include <qdom.h>
#include <iostream>
#include "globalsettings.h"

using namespace std;

// the first parts of the filenames are converted into directories
// e.g. 1a05e9c9260ca88ac39487091a96c2ee -> 1a/05/e9/c9260ca88ac39487091a96c2ee
// this helps to work with filesystems with per directory limitations
#define CACHE_DIRS_DEPTH 3

QMutex filesMutex(true);

FileCache::FileCache() {
}

/**
 * Sets the directory where the cache should be.
 */
00024 void FileCache::setCacheLocation(QString path) {
  cacheLocation = path;
  QDir().mkdir(path);

  loadCache();
}

/**
 * Loads the cache from the location set bei setCacheLocation.
 */
00034 void FileCache::loadCache() {
  QMutexLocker qml(&filesMutex);

  files.clear();
  filesizes.clear();

  QString filename = cacheLocation + QDir::separator() + QString("cache.index");
  QString alternativefilename = cacheLocation + QDir::separator() + QString("cache.index2");

  // try to load from index file
  loadCacheIndex(filename);

  // try to load from alternate index file (when the writing of the index file was interrupted)
  loadCacheIndex(alternativefilename);

  // start thread that saves the list every two seconds if something has changed
  if (!running()) {
    start();
  }
}

/**
 * Loads the cache from the given XML file and adds it to the cached files list.
 * So one can load several lists and they will be merged.
 */
00059 void FileCache::loadCacheIndex(QString filename) {
  // try to load from filename

#ifdef DEBUG
  cout << "Cache path: " << filename.latin1() << endl;
#endif
  cacheIndexFilename = filename;

  // parse the document
  QDomDocument doc;
  QFile *file = new QFile(cacheIndexFilename);
  if (doc.setContent(file)) {
    /* search for earth3dcache tag */
    QDomNode nform = doc.documentElement();
    while( !nform.isNull() ) {
      QDomElement e = nform.toElement(); // try to convert the node to an element.
      if( !e.isNull() && e.tagName() == QString("earth3dcache")) {
        /* iterate through attribute tags */
        QDomNode ninput = nform.firstChild();
        while( !ninput.isNull() ) {
          e = ninput.toElement(); // try to convert the node to an element.
          if( !e.isNull() && e.tagName() == QString("cacheentry")) {
#ifdef DEBUG
            cout << "filename: " << e.attribute("filename", "").latin1() << endl;
#endif
          
            /* insert into cache list  */
            QString newfilename = e.attribute("filename", "");
            if (newfilename != "" && !files.contains(newfilename)) {
              files.append(newfilename);
            }
          }
        
          ninput = ninput.nextSibling();
        }
      }
      nform = nform.nextSibling();      
    }
  }
  
  file->close();
  delete(file);
}

/**
 * Iterates through all files and checks if they exist. Removes file entries
 * from the cache that are invalid.
 */
00107 void FileCache::checkCacheIndex() {
  QMutexLocker qml(&filesMutex);

  QValueList<QString>::iterator it = files.begin();
  while(it != files.end()) {
    if (!QFile(cacheLocation + QDir::separator() + *it).exists()) {
      it = files.remove(it);
    }
    else {
      it++;
    }
  }
}

/**
 * Saves the cache to an XML file. It is first saved into cache.index2,
 * then cache.index is removed and cache.index2 is renamed. On startup both
 * files are loaded so there cannot be a moment where no file is valid. The
 * program can be stopped at any time.
 */
00127 void FileCache::saveCache() {
  // build XML document
  QDomDocument doc;

  QDomNode node = doc.createElement("earth3dcache");
  doc.appendChild(node);

  QValueList<QString>::iterator it = files.begin();
  while(it != files.end()) {
    QDomElement attrnode = doc.createElement("cacheentry");
    attrnode.setAttribute("filename", *it);

    node.appendChild(attrnode);

    it++;
  }

  // save to file cache.index2
  QDir().mkdir(cacheLocation);
  QFile file2(QDir(cacheLocation).filePath("cache.index2"));
  file2.open(IO_WriteOnly);
  QTextStream textstream(&file2);
  textstream << doc.toString().latin1();
  file2.close();
  
  // remove cache.index and rename cache.index2 to cache.index
  QFile file(QDir(cacheLocation).filePath("cache.index"));
  file.remove();
  QDir().rename(QDir(cacheLocation).filePath("cache.index2"), QDir(cacheLocation).filePath("cache.index"));
}

/**
 * Returns true if the cache contains a file for the given URL.
 */
00161 bool FileCache::contains(QString url) {
  QString filename = createFilename(url);
#ifdef DEBUG
  cout << "cache contains request for " << filename << endl;
#endif
  return(files.contains(filename));
}

/**
 * Builds a filename for the given url. It uses an md5 sum where the
 * first characters are used as directory names to distribute the files
 * over several directories.
 */
00174 QString FileCache::createFilename(QString url) {
  md5_context ctx;
  uint8 digest[16];

  // build md5 sum

  md5_starts(&ctx);
  md5_update(&ctx, (unsigned char *) url.latin1(), url.length());
  md5_finish(&ctx, digest);

  // build string. The first three hex numbers are treated as directories to support filesystems that have limits
  // on how many files are allowed in one directory.

  QString result;
  for(int i=0; i<16; i++) {
    result.append(hexString(digest[i]));
    if (i<CACHE_DIRS_DEPTH) {
      result.append(QDir::separator());
    }
  }

  return(result);
}

/**
 * Creates a hex string from the given data array. It is used to create
 * a hex string from the calculated md5 sum.
 */
00202 QString FileCache::hexString(uint8 value) {
  QString result = QString::number(value, 16);
  if (result.length()<2) {
    result = "0" + result;
  }

  return(result);
}

/**
 * Delivers the cached file for the given URL.
 */
00214 QByteArray FileCache::read(QString url) {
  filesMutex.lock();

  QString filename = createFilename(url);
#ifdef DEBUG
  cout << "Cache read: " << filename.latin1() << endl;
#endif
  int index = files.findIndex(filename);
  if (index>=0) {
    // move to top of the list
    files.remove(files.find(filename));
    files.prepend(filename);

    filesMutex.unlock();

    // read file from disk

    QFile file(QDir(cacheLocation).filePath(filename));
    file.open(IO_ReadOnly);
    QByteArray ba = file.readAll();
    file.close();

#ifdef DEBUG
  cout << "Cache hit: " << filename.latin1() << endl;
#endif

    return(ba);
  }

  filesMutex.unlock();

  QByteArray ba;
  return(ba);
}

/**
 * Adds the given data to the cache under the given url. It also checks the cache
 * size and removes files if necessary.
 */
00253 void FileCache::addFile(QString url, QByteArray &ba) {
  QString filename = createFilename(url);

#ifdef DEBUG
  cout << "Cache add: " << filename.latin1() << " size: " << ba.size() << endl;
#endif
  QString filepath = QDir(cacheLocation).filePath(filename);
  QString directory = QDir(filepath + QDir::separator() + "..").absPath();
#ifdef DEBUG
  cout << "Cache dir: " << directory.latin1() << endl;
  cout << "Filepath: " << filepath.latin1() << endl;
#endif
  mkpath(directory);

  QFile file(filepath);
  file.open(IO_WriteOnly);
  file.writeBlock(ba.data(), ba.size());
  file.close();

  QMutexLocker qml(&filesMutex);

  // insert the new file at the top of the list

  files.prepend(filename);
  dirty = true;

  checkCacheSize();

  saveCache();
}

/**
 * Checks if files from the cache must be deleted to fit the requested cache size.
 */
00287 void FileCache::checkCacheSize() {
  QMutexLocker qml(&filesMutex);

  // sum up all file sizes
  long size;
  size = getUsedSizeBytes();

  // check overall size

  long cachesize = getAttribute("cachesize", "50").toInt() * 1024 * 1024;
  if (cachesize<1) cachesize = 1;

  // remove entries from the end until the size is fit

  while(size>cachesize && !files.isEmpty()) {
      
      // get last entry
    QString lastfile = files.last();
    files.pop_back();

#ifdef DEBUG
    cout << "cache removing " << lastfile.latin1() << " current size: " << size << endl;
#endif

    // remove file
    removeFile(QDir(cacheLocation).filePath(lastfile));
    
    // the list has to be saved
    dirty = true;
    
    // calculate new filesize
    size -= filesizes[lastfile];
    filesizes.remove(lastfile);
  }
}

/** 
 * Remove the given file and try to remove its empty parent directories.
 */
00326 void FileCache::removeFile(QString filename) {
      // remove file
      QFile file(filename);
  file.remove();
      
      // remove directories
      rmpath(filename);
}

/**
 * Removes all empty directories.
 */
00338 void FileCache::rmpath(QString filename) {
  QString directory = QDir::convertSeparators(filename);
#ifdef DEBUG
  cout << "rmpath " << directory << endl;
  cout << "directory.findRev(QDir::separator()) " << directory.findRev(QDir::separator()) << " " << QDir::separator() << endl;
#endif
  while(directory.length()>0 && directory.findRev(QDir::separator())>0) {
      directory = directory.left(directory.findRev(QDir::separator()));

            bool removed = QDir().rmdir(directory);
#ifdef DEBUG
    cout << "removed " << directory << (removed ? " successful":" unsuccessful") << endl;
#endif
            if (!removed) break;
  }
}

/**
 * Returns the size of the given file either from the file system or
 * from the cache.
 */
00359 int FileCache::getFileSize(QString filename) {
  QMutexLocker qml(&filesMutex);

  if (!filesizes.contains(filename)) {
    filesizes[filename] = QFile(QDir(cacheLocation).filePath(filename)).size();
  }

  return(filesizes[filename]);
}

/**
 * Returns the sum of the size of all files in the cache.
 */
00372 long FileCache::getUsedSizeBytes() {
  long size = 0;

  QValueList<QString>::iterator it = files.begin();
  while(it != files.end()) {
    QString filename = *it;
    it++;

    size += getFileSize(filename);
  }

  return(size);
}

float FileCache::getUsedSizeMB() {
  long size = getUsedSizeBytes();

  // size in MB

  return(size/(1024.*1024.));
}

/**
 * Runs in a background thread. First loads the file sizes of all files and then
 * saves the cache every 2 seconds.
 */
00398 void FileCache::run() {
  // check entries and remove non existing
  checkCacheIndex();

      // load file sizes, this takes some time so do it in a thread
  getUsedSizeMB();
  
  while(true) {
    sleep(2);

    if (dirty) {
      QMutexLocker qml(&filesMutex);
      dirty = false;
      saveCache();
    }
  }
}

/**
 * Creates all directories from the given path.
 */
00419 void FileCache::mkpath(QString directory) {
  directory = QDir::convertSeparators(directory);

  QString mkdirsString = "";
  while(directory.length()>0 && directory.find(QDir::separator(), 1)>0) {
    QString dirpart = directory.left(directory.find(QDir::separator(), 1));
    directory = directory.mid(directory.find(QDir::separator(), 1));
    mkdirsString += dirpart;
    QDir().mkdir(mkdirsString);
  }
  
  mkdirsString += directory;
  QDir().mkdir(mkdirsString);
}


Generated by  Doxygen 1.6.0   Back to index