changeset 424:3239780e4e8f

Implement cleaners for mirrors on the agents
author Dmitry Neverov <dmitry.neverov@jetbrains.com>
date Fri, 11 May 2012 12:10:16 +0400
parents 010d8663ac4d
children e33c3e4918f5
files mercurial-agent/src/META-INF/build-agent-plugin-mercurial.xml mercurial-agent/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentMirrorCleaner.java mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManager.java mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManagerImpl.java mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentMirrorCleanerTest.java mercurial-tests/src/testng.xml mercurial.ipr mercurial.xml
diffstat 8 files changed, 229 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/mercurial-agent/src/META-INF/build-agent-plugin-mercurial.xml	Thu May 10 15:10:35 2012 +0400
+++ b/mercurial-agent/src/META-INF/build-agent-plugin-mercurial.xml	Fri May 11 12:10:16 2012 +0400
@@ -7,4 +7,5 @@
   <bean id="hgDetector" class="jetbrains.buildServer.buildTriggers.vcs.mercurial.HgDetector" />
   <bean id="pluginConfig" class="jetbrains.buildServer.buildTriggers.vcs.mercurial.AgentPluginConfigImpl"/>
   <bean id="mirrorManager" class="jetbrains.buildServer.buildTriggers.vcs.mercurial.MirrorManagerImpl" />
+  <bean id="mirrorCleaner" class="jetbrains.buildServer.buildTriggers.vcs.mercurial.AgentMirrorCleaner" />
 </beans>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial-agent/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentMirrorCleaner.java	Fri May 11 12:10:16 2012 +0400
@@ -0,0 +1,64 @@
+package jetbrains.buildServer.buildTriggers.vcs.mercurial;
+
+import com.intellij.openapi.diagnostic.Logger;
+import jetbrains.buildServer.agent.DirectoryCleanersProvider;
+import jetbrains.buildServer.agent.DirectoryCleanersProviderContext;
+import jetbrains.buildServer.agent.DirectoryCleanersRegistry;
+import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.AuthSettings;
+import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.Settings;
+import jetbrains.buildServer.vcs.VcsRoot;
+import jetbrains.buildServer.vcs.VcsRootEntry;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * @author dmitry.neverov
+ */
+public class AgentMirrorCleaner implements DirectoryCleanersProvider {
+
+  private final static Logger ourLog = Logger.getInstance(AgentMirrorCleaner.class.getName());
+  private final MirrorManager myMirrorManager;
+  private final HgPathProvider myHgPathProvider;
+
+  public AgentMirrorCleaner(@NotNull final MirrorManager mirrorManager,
+                            @NotNull final HgPathProvider hgPathProvider) {
+    myMirrorManager = mirrorManager;
+    myHgPathProvider = hgPathProvider;
+  }
+
+  @NotNull
+  public String getCleanerName() {
+    return "Mercurial mirrors clean";
+  }
+
+  public void registerDirectoryCleaners(@NotNull DirectoryCleanersProviderContext context,
+                                        @NotNull DirectoryCleanersRegistry registry) {
+    Set<String> repositoriesUsedInBuild = getRunningBuildRepositories(context);
+    for (Map.Entry<String, File> entry : myMirrorManager.getMappings().entrySet()) {
+      String repository = entry.getKey();
+      File mirror = entry.getValue();
+      if (!repositoriesUsedInBuild.contains(repository)) {
+        ourLog.debug("Register cleaner for mirror " + mirror.getAbsolutePath());
+        registry.addCleaner(mirror, new Date(myMirrorManager.getLastUsedTime(mirror)));
+      }
+    }
+  }
+
+  private Set<String> getRunningBuildRepositories(@NotNull DirectoryCleanersProviderContext context) {
+    Set<String> repositories = new HashSet<String>();
+    for (VcsRootEntry entry : context.getRunningBuild().getVcsRootEntries()) {
+      VcsRoot root = entry.getVcsRoot();
+      Settings s = new Settings(myHgPathProvider, root);
+      AuthSettings auth = s.getAuthSettings();
+      ourLog.debug("Repository " + auth.getRepositoryUrlWithHiddenPassword(s.getRepository()) +
+              " is used in the build, its mirror won't be cleaned");
+      repositories.add(s.getRepository());
+    }
+    return repositories;
+  }
+}
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManager.java	Thu May 10 15:10:35 2012 +0400
+++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManager.java	Fri May 11 12:10:16 2012 +0400
@@ -26,6 +26,8 @@
   @NotNull
   public List<File> getMirrors();
 
+  public long getLastUsedTime(@NotNull final File mirrorDir);
+
   /**
    * Forget specified dir. After call to this method with non-empty dir,
    * all urls which were mapped to this dir will be mapped to another.
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManagerImpl.java	Thu May 10 15:10:35 2012 +0400
+++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManagerImpl.java	Fri May 11 12:10:16 2012 +0400
@@ -56,6 +56,7 @@
     if (result == null) {
       result = createDirFor(url);
     }
+    updateLastUsedTime(result);
     return result;
   }
 
@@ -319,6 +320,44 @@
     return sb.toString();
   }
 
+  public long getLastUsedTime(@NotNull final File mirrorDir) {
+    File dotHg = new File(mirrorDir, ".hg");
+    File timestamp = new File(dotHg, "timestamp");
+    if (timestamp.exists()) {
+      try {
+        List<String> lines = FileUtil.readFile(timestamp);
+        if (lines.isEmpty())
+          return mirrorDir.lastModified();
+        else
+          return Long.valueOf(lines.get(0));
+      } catch (IOException e) {
+        return mirrorDir.lastModified();
+      }
+    } else {
+      return mirrorDir.lastModified();
+    }
+  }
+
+  private void updateLastUsedTime(@NotNull final File dir) {
+    File dotHg = new File(dir, ".hg");
+    //create timestamp only if .hg exist, otherwise subsequent clone in this directory will
+    //fail since directory is not empty
+    if (!dotHg.exists())
+      return;
+
+    lockDir(dir);
+    try {
+      File timestamp = new File(dotHg, "timestamp");
+      if (!timestamp.exists())
+        timestamp.createNewFile();
+      FileUtil.writeFileAndReportErrors(timestamp, String.valueOf(System.currentTimeMillis()));
+    } catch (IOException e) {
+      LOG.error("Error while updating timestamp in " + dir.getAbsolutePath(), e);
+    } finally {
+      unlockDir(dir);
+    }
+  }
+
   final static class StandartHash implements HashCalculator {
     public long calc(String value) {
       return Hash.calc(value);
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentMirrorCleanerTest.java	Fri May 11 12:10:16 2012 +0400
@@ -0,0 +1,120 @@
+package jetbrains.buildServer.buildTriggers.vcs.mercurial;
+
+import com.intellij.openapi.diagnostic.Logger;
+import jetbrains.buildServer.TempFiles;
+import jetbrains.buildServer.agent.*;
+import jetbrains.buildServer.log.Log4jFactory;
+import jetbrains.buildServer.vcs.*;
+import org.jetbrains.annotations.NotNull;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.util.Date;
+import java.util.HashMap;
+
+import static java.util.Arrays.asList;
+import static jetbrains.buildServer.buildTriggers.vcs.mercurial.LocalRepositoryUtil.copyRepository;
+import static jetbrains.buildServer.buildTriggers.vcs.mercurial.VcsRootBuilder.vcsRoot;
+
+/**
+ * @author dmitry.neverov
+ */
+@Test
+public class AgentMirrorCleanerTest {
+
+  static {
+    Logger.setFactory(new Log4jFactory());
+  }
+
+  private TempFiles myTempFiles = new TempFiles();
+  private Mockery myContext;
+  private MercurialAgentSideVcsSupport myVcsSupport;
+  private AgentMirrorCleaner myCleaner;
+  private BuildProgressLogger myLogger;
+  private File myWorkDir;
+  private int myBuildCounter;
+  private MirrorManager myMirrorManager;
+
+  @BeforeMethod
+  public void setUp() throws Exception {
+    myContext = new Mockery();
+    myWorkDir = myTempFiles.createTempDir();
+
+    final BuildAgentConfiguration agentConfig = myContext.mock(BuildAgentConfiguration.class);
+    myContext.checking(new Expectations() {{
+      allowing(agentConfig).getCacheDirectory("mercurial"); will(returnValue(myTempFiles.createTempDir()));
+      allowing(agentConfig).getParametersResolver(); will(returnValue(new HgPathResolver()));
+    }});
+
+    AgentPluginConfigImpl pluginConfig = new AgentPluginConfigImpl(agentConfig);
+    AgentHgPathProvider hgPathProvider = new AgentHgPathProvider(agentConfig);
+    myMirrorManager = new MirrorManagerImpl(pluginConfig);
+    myVcsSupport = new MercurialAgentSideVcsSupport(pluginConfig, hgPathProvider, myMirrorManager);
+    myCleaner = new AgentMirrorCleaner(myMirrorManager, hgPathProvider);
+    myLogger = myContext.mock(BuildProgressLogger.class);
+    myContext.checking(new Expectations() {{
+      allowing(myLogger).message(with(any(String.class)));
+      allowing(myLogger).warning(with(any(String.class)));
+    }});
+  }
+
+  @AfterMethod
+  public void tearDown() {
+    myTempFiles.cleanup();
+  }
+
+
+  public void should_add_cleaners_only_for_roots_not_used_in_build() throws Exception {
+    //setup mirrors for 2 roots on an agent:
+    final File repo1 = myTempFiles.createTempDir();
+    final File repo2 = myTempFiles.createTempDir();
+    copyRepository(new File("mercurial-tests/testData/rep1"), repo1);
+    copyRepository(new File("mercurial-tests/testData/rep2"), repo2);
+
+    final VcsRoot root1 = vcsRoot().withUrl(repo1.getAbsolutePath()).build();
+    final VcsRoot root2 = vcsRoot().withUrl(repo2.getAbsolutePath()).build();
+    //update will initialize mirrors
+    doUpdate(root1, "4:b06a290a363b", myWorkDir);
+    doUpdate(root2, "8:b6e2d176fe8e", new File(myWorkDir, "subdir"));
+
+    final File mirrorDir1 = myMirrorManager.getMirrorDir(repo1.getAbsolutePath());
+    final File mirrorDir2 = myMirrorManager.getMirrorDir(repo2.getAbsolutePath());
+    final Date mirrorDir1LastUsedTime = new Date(myMirrorManager.getLastUsedTime(mirrorDir1));
+    final Date mirrorDir2LastUsedTime = new Date(myMirrorManager.getLastUsedTime(mirrorDir2));
+
+    //run build which uses root1:
+    final DirectoryCleanersProviderContext ctx = myContext.mock(DirectoryCleanersProviderContext.class);
+    myContext.checking(new Expectations(){{
+      AgentRunningBuild build = myContext.mock(AgentRunningBuild.class, "build" + myBuildCounter++);
+      allowing(build).getVcsRootEntries(); will(returnValue(asList(new VcsRootEntry(root1, CheckoutRules.DEFAULT))));
+      allowing(ctx).getRunningBuild(); will(returnValue(build));
+    }});
+
+    //cleaner should add cleaners only for roots which are not used in the running build
+    final DirectoryCleanersRegistry registry = myContext.mock(DirectoryCleanersRegistry.class);
+    myContext.checking(new Expectations() {{
+      never(registry).addCleaner(with(mirrorDir1), with(mirrorDir1LastUsedTime));
+      one(registry).addCleaner(with(mirrorDir2), with(mirrorDir2LastUsedTime));
+    }});
+
+    myCleaner.registerDirectoryCleaners(ctx, registry);
+    myContext.assertIsSatisfied();
+  }
+
+
+  private void doUpdate(@NotNull VcsRoot vcsRoot, @NotNull String toVersion, @NotNull File workDir) throws VcsException {
+    final AgentRunningBuild build = myContext.mock(AgentRunningBuild.class, "build" + myBuildCounter++);
+    myContext.checking(new Expectations() {{
+      allowing(build).getBuildLogger(); will(returnValue(myLogger));
+      allowing(build).getSharedConfigParameters(); will(returnValue(new HashMap<String, String>() {{
+        put("teamcity.hg.use.local.mirrors", "true");
+      }}));
+    }});
+    myVcsSupport.getUpdater(vcsRoot, CheckoutRules.DEFAULT, toVersion, workDir, build, false).process(IncludeRule.createDefaultInstance(), workDir);
+  }
+
+}
--- a/mercurial-tests/src/testng.xml	Thu May 10 15:10:35 2012 +0400
+++ b/mercurial-tests/src/testng.xml	Fri May 11 12:10:16 2012 +0400
@@ -22,6 +22,7 @@
       <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.UnrelatedResitoriesTest"/>
       <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.CleanupTest"/>
       <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.MercurialResetCacheHandlerTest"/>
+      <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.AgentMirrorCleanerTest"/>
     </classes>
   </test>
 </suite>
--- a/mercurial.ipr	Thu May 10 15:10:35 2012 +0400
+++ b/mercurial.ipr	Fri May 11 12:10:16 2012 +0400
@@ -444,6 +444,7 @@
     <library name="TeamCityAPI-agent">
       <CLASSES>
         <root url="jar://$TeamCityDistribution$/devPackage/agent-api.jar!/" />
+        <root url="jar://$TeamCityDistribution$/buildAgent/lib/agent.jar!/" />
       </CLASSES>
       <JAVADOC />
       <SOURCES>
--- a/mercurial.xml	Thu May 10 15:10:35 2012 +0400
+++ b/mercurial.xml	Fri May 11 12:10:16 2012 +0400
@@ -128,6 +128,7 @@
   </path>
   
   <path id="library.teamcityapi-agent.classpath">
+    <pathelement location="${path.variable.teamcitydistribution}/buildAgent/lib/agent.jar"/>
     <pathelement location="${path.variable.teamcitydistribution}/devPackage/agent-api.jar"/>
   </path>