view mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManagerImpl.java @ 582:23abd5aa1115

TW-27233 bundle quartz
author Dmitry Neverov <dmitry.neverov@jetbrains.com>
date Mon, 08 Apr 2013 13:06:17 +0400
parents 3239780e4e8f
children 31a1aca3305c
line wrap: on
line source
package jetbrains.buildServer.buildTriggers.vcs.mercurial;

import com.intellij.openapi.diagnostic.Logger;
import jetbrains.buildServer.util.FileUtil;
import jetbrains.buildServer.util.Hash;
import org.jetbrains.annotations.NotNull;

import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import static jetbrains.buildServer.util.FileUtil.isEmptyDir;

/**
 * Manages local mirrors of remote repositories.
 * Each unique url get unique local mirror. Each mirror is used for one url only.
 * @author dmitry.neverov
 */
public final class MirrorManagerImpl implements MirrorManager {

  private static Logger LOG = Logger.getInstance(MirrorManagerImpl.class.getName());
  private static final String MIRROR_DIR_PREFIX = "hg_";
  private static final String MAPPING_FILE_NAME = "map";

  private final ReadWriteLock myLock = new ReentrantReadWriteLock();
  private final File myRootDir;
  /*Only one thread read or write to this file, it is protected by myLock.writeLock()*/
  private final File myMappingFile;
  /*Protected by myLock*/
  private final Map<String, File> myMirrors = new HashMap<String, File>();
  private HashCalculator myHash = new StandartHash();

  private final ConcurrentMap<String, Lock> myDirLocks = new ConcurrentHashMap<String, Lock>();

  public MirrorManagerImpl(@NotNull PluginConfig config) {
    myRootDir = config.getCachesDir();
    myMappingFile = new File(myRootDir, MAPPING_FILE_NAME);
    readMappingFromFile();
  }


  /**
   * Get directory of local mirror repository for specified url, if directory is not exists it is created
   * @param url url of interest
   * @return see above
   */
  @NotNull
  public File getMirrorDir(@NotNull final String url) {
    File result = getMirrorDirWithLock(url);
    if (result == null) {
      result = createDirFor(url);
    }
    updateLastUsedTime(result);
    return result;
  }


  /**
   * Get all local mirror repository dirs
   * @return see above
   */
  @NotNull
  public List<File> getMirrors() {
    myLock.readLock().lock();
    try {
      return new ArrayList<File>(myMirrors.values());
    } finally {
      myLock.readLock().unlock();
    }
  }


  @NotNull
  public Map<String, File> getMappings() {
    myLock.readLock().lock();
    try {
      return new HashMap<String, File>(myMirrors);
    } finally {
      myLock.readLock().unlock();
    }
  }

  public void lockDir(@NotNull final File dir) {
    lockFor(dir).lock();
  }

  public void unlockDir(@NotNull final File dir) {
    lockFor(dir).unlock();
  }

  private Lock lockFor(final File dir) {
    String path = dir.getAbsolutePath();
    Lock lock = myDirLocks.get(path);
    if (lock == null) {
      lock = new ReentrantLock();
      Lock curLock = myDirLocks.putIfAbsent(path, lock);
      if (curLock != null)
        lock = curLock;
    }
    return lock;
  }

  /**
   * Forget specified dir. After call to this method with non-empty dir,
   * all urls which were mapped to this dir will be mapped to another.
   * If dir is empty, subsequent call getMirrorDir(dir) will return the
   * same dir.
   *
   * @param dir dir of interest
   */
  public void forgetDir(@NotNull final File dir) {
    myLock.writeLock().lock();
    try {
      removeMappingsToDir(dir);
      saveMappingToFile();
    } finally {
      myLock.writeLock().unlock();
    }
  }

  private void removeMappingsToDir(@NotNull final File dir) {
    Set<String> keysToRemove = getUrlsMappedToDir(dir);
    for (String key : keysToRemove) {
      myMirrors.remove(key);
    }
  }

  private Set<String> getUrlsMappedToDir(@NotNull final File dir) {
    Set<String> urlsMappedToDir = new HashSet<String>();
    for (Map.Entry<String, File> entry : myMirrors.entrySet()) {
      File f = entry.getValue();
      if (f.equals(dir))
        urlsMappedToDir.add(entry.getKey());
    }
    return urlsMappedToDir;
  }


  //for tests only
  void setHashCalculator(HashCalculator hash) {
    myHash = hash;
  }


  private File createDirFor(String url) {
    File result;
    myLock.writeLock().lock();
    try {
      File mirrorDir = getUniqueDir(url);
      result = saveMappingIfAbsent(url, mirrorDir);
    } finally {
      myLock.writeLock().unlock();
    }
    if (!result.exists()) {
      result.mkdirs();
    }
    return result;
  }


  private File getMirrorDirWithLock(String url) {
    myLock.readLock().lock();
    try {
      return myMirrors.get(url);
    } finally {
      myLock.readLock().unlock();
    }
  }


  //should be called with myLock.writeLock() held
  private File saveMappingIfAbsent(String url, File mirrorDir) {
    File existing = myMirrors.get(url);
    if (existing != null) {
      return existing;
    } else {
      myMirrors.put(url, mirrorDir);
      saveMappingToFile();
      return mirrorDir;
    }
  }


  private File getUniqueDir(String url) {
    myLock.readLock().lock();
    try {
      String dirName = MIRROR_DIR_PREFIX + hash(normalize(url));
      File result = PathUtil.getCanonicalFile(new File(myRootDir, dirName));
      while (isUsedForOtherUrl(result, url) || !isEmptyDir(result)) {
        dirName = MIRROR_DIR_PREFIX + hash(result.getName());
        result = PathUtil.getCanonicalFile(new File(myRootDir, dirName));
      }
      return result;
    } finally {
      myLock.readLock().unlock();
    }
  }


  private boolean isUsedForOtherUrl(File repositoryDir, String url) {
    myLock.readLock().lock();
    try {
      for (Map.Entry<String, File> mirror : myMirrors.entrySet()) {
        String mirrorUrl = mirror.getKey();
        File mirrorDir = mirror.getValue();
        if (mirrorDir.equals(repositoryDir) && !mirrorUrl.equals(url)) {
          return true;
        }
      }
      return false;
    } finally {
      myLock.readLock().unlock();
    }
  }


  private String hash(String value) {
    return String.valueOf(myHash.calc(value));
  }


  private static String normalize(final String path) {
    String normalized = PathUtil.normalizeSeparator(path);
    if (path.endsWith("/")) {
      return normalized.substring(0, normalized.length()-1);
    }
    return normalized;
  }


  private void readMappingFromFile() {
    myLock.writeLock().lock();
    try {
      LOG.debug("Parse mapping file " + myMappingFile.getAbsolutePath());
      for (String line : readLines()) {
        int separatorIndex = line.lastIndexOf(" = ");
        if (separatorIndex == -1) {
          if (!line.equals(""))
            LOG.warn("Cannot parse mapping '" + line + "', skip it.");
        } else {
          String url = line.substring(0, separatorIndex);
          String dirName = line.substring(separatorIndex + 3);
          File repositoryDir = PathUtil.getCanonicalFile(new File(myRootDir, dirName));
          if (isUsedForOtherUrl(repositoryDir, url)) {
            LOG.error("Skip mapping " + line + ": " + dirName + " is used for url other than " + url);
          } else {
            myMirrors.put(url, PathUtil.getCanonicalFile(new File(myRootDir, dirName)));
          }
        }
      }
    } finally {
      myLock.writeLock().unlock();
    }
  }

  /*Should be called with myLock.writeLock() held*/
  private List<String> readLines() {
    if (myMappingFile.exists()) {
      try {
        return FileUtil.readFile(myMappingFile);
      } catch (IOException e) {
        LOG.error("Error while reading a mapping file at " + myMappingFile.getAbsolutePath() + " starting with empty mapping", e);
        return new ArrayList<String>();
      }
    } else {
      LOG.debug("No mapping file found at " + myMappingFile.getAbsolutePath() + " starting with empty mapping");
      File parentDir = myMappingFile.getParentFile();
      if (!parentDir.exists() && !parentDir.mkdirs()) {
        LOG.error("Cannot create local mirrors dir at " + parentDir.getAbsolutePath());
      } else {
        try {
          if (!myMappingFile.createNewFile())
            LOG.warn("Someone else creates a mapping file " + myMappingFile.getAbsolutePath() + ", will use it");
        } catch (IOException e) {
          LOG.error("Cannot create a mapping file at " + myMappingFile.getAbsolutePath(), e);
        }
      }
      return new ArrayList<String>();
    }
  }


  private void saveMappingToFile() {
    myLock.writeLock().lock();
    try {
      StringBuilder sb = new StringBuilder();
      for (Map.Entry<String, File> mirror : myMirrors.entrySet()) {
        String url = mirror.getKey();
        String dir = mirror.getValue().getName();
        sb.append(url).append(" = ").append(dir).append("\n");
      }
      FileUtil.writeFile(myMappingFile, sb.toString());
    } finally {
      myLock.writeLock().unlock();
    }
  }


  @Override
  public String toString() {
    StringBuilder sb = new StringBuilder();
    myLock.readLock().lock();
    try {
      Iterator<Map.Entry<String, File>> iter = myMirrors.entrySet().iterator();
      while (iter.hasNext()) {
        Map.Entry<String, File> entry = iter.next();
        sb.append("[").append(entry.getKey()).append("]").append("->").append(entry.getValue().getAbsolutePath());
        if (iter.hasNext())
          sb.append("\n");
      }
    } finally {
      myLock.readLock().unlock();
    }
    return sb.toString();
  }

  public long getLastUsedTime(@NotNull final File mirrorDir) {
    File dotHg = new File(mirrorDir, ".hg");
    File timestamp = new File(dotHg, "timestamp");
    if (timestamp.exists()) {
      try {
        List<String> lines = FileUtil.readFile(timestamp);
        if (lines.isEmpty())
          return mirrorDir.lastModified();
        else
          return Long.valueOf(lines.get(0));
      } catch (IOException e) {
        return mirrorDir.lastModified();
      }
    } else {
      return mirrorDir.lastModified();
    }
  }

  private void updateLastUsedTime(@NotNull final File dir) {
    File dotHg = new File(dir, ".hg");
    //create timestamp only if .hg exist, otherwise subsequent clone in this directory will
    //fail since directory is not empty
    if (!dotHg.exists())
      return;

    lockDir(dir);
    try {
      File timestamp = new File(dotHg, "timestamp");
      if (!timestamp.exists())
        timestamp.createNewFile();
      FileUtil.writeFileAndReportErrors(timestamp, String.valueOf(System.currentTimeMillis()));
    } catch (IOException e) {
      LOG.error("Error while updating timestamp in " + dir.getAbsolutePath(), e);
    } finally {
      unlockDir(dir);
    }
  }

  final static class StandartHash implements HashCalculator {
    public long calc(String value) {
      return Hash.calc(value);
    }
  }

  public static interface HashCalculator {
    long calc(String value);
  }
}