changeset 579:da75e774d237

Clean mirrors on the server relying on last used time
author Dmitry Neverov <dmitry.neverov@jetbrains.com>
date Wed, 27 Mar 2013 21:49:46 +0400
parents 40c085517770
children fa40f53ec019
files mercurial-server-tc/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCleaner.java mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/Cleanup.java mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerPluginConfig.java mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerPluginConfigImpl.java mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/CleanupTest.java mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerPluginConfigBuilder.java
diffstat 6 files changed, 63 insertions(+), 79 deletions(-) [+]
line wrap: on
line diff
--- a/mercurial-server-tc/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCleaner.java	Wed Mar 27 20:25:50 2013 +0400
+++ b/mercurial-server-tc/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCleaner.java	Wed Mar 27 21:49:46 2013 +0400
@@ -4,20 +4,18 @@
 import jetbrains.buildServer.serverSide.BuildServerListener;
 import jetbrains.buildServer.serverSide.SBuildServer;
 import jetbrains.buildServer.util.EventDispatcher;
-import jetbrains.buildServer.vcs.VcsManager;
 import org.jetbrains.annotations.NotNull;
 
 public class MercurialCleaner implements MercurialServerExtension {
 
   public MercurialCleaner(@NotNull final SBuildServer server,
-                          @NotNull final VcsManager vcsManager,
                           @NotNull final EventDispatcher<BuildServerListener> dispatcher,
                           @NotNull final ServerPluginConfig config,
                           @NotNull final MirrorManager mirrorManager) {
     dispatcher.addListener(new BuildServerAdapter() {
       @Override
       public void cleanupFinished() {
-        server.getExecutor().submit(new Cleanup(vcsManager, mirrorManager, config));
+        server.getExecutor().submit(new Cleanup(mirrorManager, config));
       }
     });
   }
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/Cleanup.java	Wed Mar 27 20:25:50 2013 +0400
+++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/Cleanup.java	Wed Mar 27 21:49:46 2013 +0400
@@ -1,11 +1,7 @@
 package jetbrains.buildServer.buildTriggers.vcs.mercurial;
 
 import com.intellij.openapi.diagnostic.Logger;
-import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.HgVcsRoot;
-import jetbrains.buildServer.serverSide.impl.LogUtil;
 import jetbrains.buildServer.util.FileUtil;
-import jetbrains.buildServer.vcs.VcsManager;
-import jetbrains.buildServer.vcs.VcsRoot;
 import org.jetbrains.annotations.NotNull;
 
 import java.io.File;
@@ -21,14 +17,11 @@
 
   private static Logger LOG = Logger.getInstance(Cleanup.class.getName());
 
-  private final VcsManager myVcsManager;
   private final MirrorManager myMirrorManager;
-  private final PluginConfig myConfig;
+  private final ServerPluginConfig myConfig;
 
-  public Cleanup(@NotNull final VcsManager vcsManager,
-                 @NotNull final MirrorManager mirrorManager,
-                 @NotNull final PluginConfig config) {
-    myVcsManager = vcsManager;
+  public Cleanup(@NotNull final MirrorManager mirrorManager,
+                 @NotNull final ServerPluginConfig config) {
     myMirrorManager = mirrorManager;
     myConfig = config;
   }
@@ -39,7 +32,7 @@
 
   private Collection<File> unusedDirs() {
     List<File> existingDirs = existingDirs();
-    List<File> mirrorsInUse = mirrorDirsOfRootsInUse();
+    List<File> mirrorsInUse = mirrorDirsOfRootsInUse(existingDirs);
     existingDirs.removeAll(mirrorsInUse);
     return existingDirs;
   }
@@ -61,32 +54,21 @@
     });
   }
 
-  private List<File> mirrorDirsOfRootsInUse() {
-    Map<String, File> mirrorMap = myMirrorManager.getMappings();
+  private List<File> mirrorDirsOfRootsInUse(@NotNull List<File> mirrors) {
     List<File> result = new ArrayList<File>();
-    for (VcsRoot root : mercurialVcsRoots()) {
-      HgVcsRoot hgRoot = new HgVcsRoot(root);
-      File mirrorDir = mirrorMap.get(hgRoot.getRepository());
-      if (mirrorDir != null)
-        result.add(mirrorDir);
+    long now = new Date().getTime();
+    for (File mirror : mirrors) {
+      File dotHg = new File(mirror, ".hg");
+      File timestamp = new File(dotHg, "timestamp");
+      if (!timestamp.exists())
+        continue;
+      long lastUsedTime = myMirrorManager.getLastUsedTime(mirror);
+      if (now - lastUsedTime < myConfig.getMirrorExpirationTimeoutMillis())
+        result.add(mirror);
     }
     return result;
   }
 
-  private Collection<VcsRoot> mercurialVcsRoots() {
-    List<VcsRoot> mercurialRoots = new ArrayList<VcsRoot>();
-    for (VcsRoot root : myVcsManager.getAllRegisteredVcsRoots()) {
-      if (isMercurialRoot(root))
-        mercurialRoots.add(root);
-    }
-    logRegisteredMercurialRoots(mercurialRoots);
-    return mercurialRoots;
-  }
-
-  private boolean isMercurialRoot(VcsRoot root) {
-    return Constants.VCS_NAME.equals(root.getVcsName());
-  }
-
   private void delete(Collection<File> dirs) {
     logUnusedLocalClones(dirs);
     for (File dir : dirs) {
@@ -100,18 +82,6 @@
     }
   }
 
-  private void logRegisteredMercurialRoots(@NotNull List<VcsRoot> roots) {
-    StringBuilder sb = new StringBuilder();
-    sb.append("Registered mercurial roots: ");
-    Iterator<VcsRoot> iter = roots.iterator();
-    while (iter.hasNext()) {
-      sb.append(LogUtil.describe(iter.next()));
-      if (iter.hasNext())
-        sb.append(", ");
-    }
-    LOG.debug(sb.toString());
-  }
-
   private void logUnusedLocalClones(@NotNull Collection<File> dirs) {
     if (dirs.isEmpty())
       return;
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerPluginConfig.java	Wed Mar 27 20:25:50 2013 +0400
+++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerPluginConfig.java	Wed Mar 27 21:49:46 2013 +0400
@@ -25,4 +25,6 @@
   public int getMaxDagNodesCount();
 
   public int getLogOutputLimit();
+
+  public long getMirrorExpirationTimeoutMillis();
 }
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerPluginConfigImpl.java	Wed Mar 27 20:25:50 2013 +0400
+++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerPluginConfigImpl.java	Wed Mar 27 21:49:46 2013 +0400
@@ -3,6 +3,7 @@
 import com.intellij.openapi.diagnostic.Logger;
 import jetbrains.buildServer.serverSide.CachePaths;
 import jetbrains.buildServer.serverSide.TeamCityProperties;
+import jetbrains.buildServer.util.Dates;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -61,4 +62,9 @@
   public boolean bookmarksEnabled() {
     return TeamCityProperties.getBooleanOrTrue("teamcity.hg.enableBookmarks");
   }
+
+  public long getMirrorExpirationTimeoutMillis() {
+    int days = TeamCityProperties.getInteger("teamcity.hg.mirrorExpirationTimeoutDays", 7);
+    return days * Dates.ONE_DAY;
+  }
 }
--- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/CleanupTest.java	Wed Mar 27 20:25:50 2013 +0400
+++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/CleanupTest.java	Wed Mar 27 21:49:46 2013 +0400
@@ -2,13 +2,9 @@
 
 import jetbrains.buildServer.TempFiles;
 import jetbrains.buildServer.util.FileUtil;
-import jetbrains.buildServer.vcs.SVcsRoot;
-import jetbrains.buildServer.vcs.VcsManager;
 import org.hamcrest.Description;
 import org.hamcrest.TypeSafeMatcher;
 import org.jetbrains.annotations.NotNull;
-import org.jmock.Expectations;
-import org.jmock.Mockery;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
@@ -18,11 +14,13 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Date;
 import java.util.List;
 
 import static java.util.Arrays.asList;
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
 
 /**
  * @author dmitry.neverov
@@ -30,25 +28,21 @@
 @Test
 public class CleanupTest {
 
-  private Mockery myContext;
   private TempFiles myTempFiles;
   private File myCachesDir;
   private Cleanup myCleanup;
   private MirrorManager myMirrorManager;
-  private VcsManager myVcsManager;
-  private long myRootId = 0;
+  private long myMirrorExpirationTimeout = 2000L;
 
   @BeforeMethod
   public void setUp() throws Exception {
-    myContext = new Mockery();
     myTempFiles = new TempFiles();
     myCachesDir = myTempFiles.createTempDir();
 
-    ServerPluginConfig config = new ServerPluginConfigBuilder().cachesDir(myCachesDir).build();
+    ServerPluginConfig config = new ServerPluginConfigBuilder().cachesDir(myCachesDir).withMirrorExpirationTimeout(myMirrorExpirationTimeout).build();
     myMirrorManager = new MirrorManagerImpl(config);
 
-    myVcsManager = myContext.mock(VcsManager.class);
-    myCleanup = new Cleanup(myVcsManager, myMirrorManager, config);
+    myCleanup = new Cleanup(myMirrorManager, config);
   }
 
   @AfterMethod
@@ -57,43 +51,47 @@
   }
 
 
-  public void cleanup_should_remove_all_directories_that_are_not_mirrors() throws IOException {
+  public void cleanup_should_remove_directories_by_timestamp() throws Exception {
     final String url1 = "http://some.org/repository1";
     final String url2 = "http://some.org/repository2";
     final String url3 = "http://some.org/repository3";
-    createDirFor(url1);
-    createDirFor(url2);
-    createDirFor(url3);
-    createUnusedDir();
-    myContext.checking(new Expectations() {{
-      atLeast(1).of(myVcsManager).getAllRegisteredVcsRoots();
-      will(returnValue(asList(build(vcsRoot().withUrl(url1)), build(vcsRoot().withUrl(url2)))));
-    }});
+    File mirror1 = createDirFor(url1);
+    File mirror2 = createDirFor(url2);
+    File mirror3 = createDirFor(url3);
+
+    updateTimestamp(mirror1, new Date().getTime());
+    updateTimestamp(mirror2, new Date().getTime() - myMirrorExpirationTimeout);
+    deleteTimestamp(mirror3);
 
     myCleanup.run();
 
-    myContext.assertIsSatisfied();
-    assertEquals(2, directoriesInside(myCachesDir).size());
+    assertTrue(mirror1.exists());
+    assertFalse(mirror2.exists());
+    assertFalse(mirror3.exists());
+
     assertThat(myMirrorManager, knowsAboutAll(directoriesInside(myCachesDir)));
     assertThat(mappingsFile(), containsOnly(directoriesInside(myCachesDir)));
   }
 
-  private void createUnusedDir() {
-    new File(myCachesDir, "some unused dir").mkdirs();
+
+  private void updateTimestamp(@NotNull File mirror, @NotNull long time) throws IOException {
+    File dotHg = new File(mirror, ".hg");
+    dotHg.mkdirs();
+    File timestamp = new File(dotHg, "timestamp");
+    timestamp.createNewFile();
+    FileUtil.writeFileAndReportErrors(timestamp, String.valueOf(time));
   }
 
 
-  private void createDirFor(@NotNull String url) {
-    myMirrorManager.getMirrorDir(url);
+  private void deleteTimestamp(@NotNull File mirror) {
+    FileUtil.delete(new File(new File(mirror, ".hg"), "timestamp"));
   }
 
-  private VcsRootBuilder vcsRoot() {
-    return new VcsRootBuilder().withId(myRootId++);
+
+  private File createDirFor(@NotNull String url) {
+    return myMirrorManager.getMirrorDir(url);
   }
 
-  private SVcsRoot build(@NotNull VcsRootBuilder builder) {
-    return builder.build(myContext);
-  }
 
   private List<File> directoriesInside(@NotNull File dir) {
     return asList(dir.listFiles(new FileFilter() {
--- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerPluginConfigBuilder.java	Wed Mar 27 20:25:50 2013 +0400
+++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerPluginConfigBuilder.java	Wed Mar 27 21:49:46 2013 +0400
@@ -14,6 +14,7 @@
   private File myCachesDir;
   private boolean myDontUseRevsets = false;
   private boolean myDetectSubrepoChanges = false;
+  private long myMirrorExpirationTimeout;
 
   @NotNull
   public ServerPluginConfig build() {
@@ -57,6 +58,10 @@
       public int getLogOutputLimit() {
         return 0;
       }
+
+      public long getMirrorExpirationTimeoutMillis() {
+        return myMirrorExpirationTimeout;
+      }
     };
   }
 
@@ -89,4 +94,9 @@
     myDetectSubrepoChanges = doDetect;
     return this;
   }
+
+  public ServerPluginConfigBuilder withMirrorExpirationTimeout(long timeoutMillis) {
+    myMirrorExpirationTimeout = timeoutMillis;
+    return this;
+  }
 }