Mercurial > hg > mercurial
changeset 424:3239780e4e8f
Implement cleaners for mirrors on the agents
author | Dmitry Neverov <dmitry.neverov@jetbrains.com> |
---|---|
date | Fri, 11 May 2012 12:10:16 +0400 |
parents | 010d8663ac4d |
children | e33c3e4918f5 |
files | mercurial-agent/src/META-INF/build-agent-plugin-mercurial.xml mercurial-agent/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentMirrorCleaner.java mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManager.java mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManagerImpl.java mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentMirrorCleanerTest.java mercurial-tests/src/testng.xml mercurial.ipr mercurial.xml |
diffstat | 8 files changed, 229 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/mercurial-agent/src/META-INF/build-agent-plugin-mercurial.xml Thu May 10 15:10:35 2012 +0400 +++ b/mercurial-agent/src/META-INF/build-agent-plugin-mercurial.xml Fri May 11 12:10:16 2012 +0400 @@ -7,4 +7,5 @@ <bean id="hgDetector" class="jetbrains.buildServer.buildTriggers.vcs.mercurial.HgDetector" /> <bean id="pluginConfig" class="jetbrains.buildServer.buildTriggers.vcs.mercurial.AgentPluginConfigImpl"/> <bean id="mirrorManager" class="jetbrains.buildServer.buildTriggers.vcs.mercurial.MirrorManagerImpl" /> + <bean id="mirrorCleaner" class="jetbrains.buildServer.buildTriggers.vcs.mercurial.AgentMirrorCleaner" /> </beans>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial-agent/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentMirrorCleaner.java Fri May 11 12:10:16 2012 +0400 @@ -0,0 +1,64 @@ +package jetbrains.buildServer.buildTriggers.vcs.mercurial; + +import com.intellij.openapi.diagnostic.Logger; +import jetbrains.buildServer.agent.DirectoryCleanersProvider; +import jetbrains.buildServer.agent.DirectoryCleanersProviderContext; +import jetbrains.buildServer.agent.DirectoryCleanersRegistry; +import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.AuthSettings; +import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.Settings; +import jetbrains.buildServer.vcs.VcsRoot; +import jetbrains.buildServer.vcs.VcsRootEntry; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.util.Date; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * @author dmitry.neverov + */ +public class AgentMirrorCleaner implements DirectoryCleanersProvider { + + private final static Logger ourLog = Logger.getInstance(AgentMirrorCleaner.class.getName()); + private final MirrorManager myMirrorManager; + private final HgPathProvider myHgPathProvider; + + public AgentMirrorCleaner(@NotNull final MirrorManager mirrorManager, + @NotNull final HgPathProvider hgPathProvider) { + myMirrorManager = mirrorManager; + myHgPathProvider = hgPathProvider; + } + + @NotNull + public String getCleanerName() { + return "Mercurial mirrors clean"; + } + + public void registerDirectoryCleaners(@NotNull DirectoryCleanersProviderContext context, + @NotNull DirectoryCleanersRegistry registry) { + Set<String> repositoriesUsedInBuild = getRunningBuildRepositories(context); + for (Map.Entry<String, File> entry : myMirrorManager.getMappings().entrySet()) { + String repository = entry.getKey(); + File mirror = entry.getValue(); + if (!repositoriesUsedInBuild.contains(repository)) { + ourLog.debug("Register cleaner for mirror " + mirror.getAbsolutePath()); + registry.addCleaner(mirror, new Date(myMirrorManager.getLastUsedTime(mirror))); + } + } + } + + private Set<String> getRunningBuildRepositories(@NotNull DirectoryCleanersProviderContext context) { + Set<String> repositories = new HashSet<String>(); + for (VcsRootEntry entry : context.getRunningBuild().getVcsRootEntries()) { + VcsRoot root = entry.getVcsRoot(); + Settings s = new Settings(myHgPathProvider, root); + AuthSettings auth = s.getAuthSettings(); + ourLog.debug("Repository " + auth.getRepositoryUrlWithHiddenPassword(s.getRepository()) + + " is used in the build, its mirror won't be cleaned"); + repositories.add(s.getRepository()); + } + return repositories; + } +}
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManager.java Thu May 10 15:10:35 2012 +0400 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManager.java Fri May 11 12:10:16 2012 +0400 @@ -26,6 +26,8 @@ @NotNull public List<File> getMirrors(); + public long getLastUsedTime(@NotNull final File mirrorDir); + /** * Forget specified dir. After call to this method with non-empty dir, * all urls which were mapped to this dir will be mapped to another.
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManagerImpl.java Thu May 10 15:10:35 2012 +0400 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManagerImpl.java Fri May 11 12:10:16 2012 +0400 @@ -56,6 +56,7 @@ if (result == null) { result = createDirFor(url); } + updateLastUsedTime(result); return result; } @@ -319,6 +320,44 @@ return sb.toString(); } + public long getLastUsedTime(@NotNull final File mirrorDir) { + File dotHg = new File(mirrorDir, ".hg"); + File timestamp = new File(dotHg, "timestamp"); + if (timestamp.exists()) { + try { + List<String> lines = FileUtil.readFile(timestamp); + if (lines.isEmpty()) + return mirrorDir.lastModified(); + else + return Long.valueOf(lines.get(0)); + } catch (IOException e) { + return mirrorDir.lastModified(); + } + } else { + return mirrorDir.lastModified(); + } + } + + private void updateLastUsedTime(@NotNull final File dir) { + File dotHg = new File(dir, ".hg"); + //create timestamp only if .hg exist, otherwise subsequent clone in this directory will + //fail since directory is not empty + if (!dotHg.exists()) + return; + + lockDir(dir); + try { + File timestamp = new File(dotHg, "timestamp"); + if (!timestamp.exists()) + timestamp.createNewFile(); + FileUtil.writeFileAndReportErrors(timestamp, String.valueOf(System.currentTimeMillis())); + } catch (IOException e) { + LOG.error("Error while updating timestamp in " + dir.getAbsolutePath(), e); + } finally { + unlockDir(dir); + } + } + final static class StandartHash implements HashCalculator { public long calc(String value) { return Hash.calc(value);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentMirrorCleanerTest.java Fri May 11 12:10:16 2012 +0400 @@ -0,0 +1,120 @@ +package jetbrains.buildServer.buildTriggers.vcs.mercurial; + +import com.intellij.openapi.diagnostic.Logger; +import jetbrains.buildServer.TempFiles; +import jetbrains.buildServer.agent.*; +import jetbrains.buildServer.log.Log4jFactory; +import jetbrains.buildServer.vcs.*; +import org.jetbrains.annotations.NotNull; +import org.jmock.Expectations; +import org.jmock.Mockery; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.Date; +import java.util.HashMap; + +import static java.util.Arrays.asList; +import static jetbrains.buildServer.buildTriggers.vcs.mercurial.LocalRepositoryUtil.copyRepository; +import static jetbrains.buildServer.buildTriggers.vcs.mercurial.VcsRootBuilder.vcsRoot; + +/** + * @author dmitry.neverov + */ +@Test +public class AgentMirrorCleanerTest { + + static { + Logger.setFactory(new Log4jFactory()); + } + + private TempFiles myTempFiles = new TempFiles(); + private Mockery myContext; + private MercurialAgentSideVcsSupport myVcsSupport; + private AgentMirrorCleaner myCleaner; + private BuildProgressLogger myLogger; + private File myWorkDir; + private int myBuildCounter; + private MirrorManager myMirrorManager; + + @BeforeMethod + public void setUp() throws Exception { + myContext = new Mockery(); + myWorkDir = myTempFiles.createTempDir(); + + final BuildAgentConfiguration agentConfig = myContext.mock(BuildAgentConfiguration.class); + myContext.checking(new Expectations() {{ + allowing(agentConfig).getCacheDirectory("mercurial"); will(returnValue(myTempFiles.createTempDir())); + allowing(agentConfig).getParametersResolver(); will(returnValue(new HgPathResolver())); + }}); + + AgentPluginConfigImpl pluginConfig = new AgentPluginConfigImpl(agentConfig); + AgentHgPathProvider hgPathProvider = new AgentHgPathProvider(agentConfig); + myMirrorManager = new MirrorManagerImpl(pluginConfig); + myVcsSupport = new MercurialAgentSideVcsSupport(pluginConfig, hgPathProvider, myMirrorManager); + myCleaner = new AgentMirrorCleaner(myMirrorManager, hgPathProvider); + myLogger = myContext.mock(BuildProgressLogger.class); + myContext.checking(new Expectations() {{ + allowing(myLogger).message(with(any(String.class))); + allowing(myLogger).warning(with(any(String.class))); + }}); + } + + @AfterMethod + public void tearDown() { + myTempFiles.cleanup(); + } + + + public void should_add_cleaners_only_for_roots_not_used_in_build() throws Exception { + //setup mirrors for 2 roots on an agent: + final File repo1 = myTempFiles.createTempDir(); + final File repo2 = myTempFiles.createTempDir(); + copyRepository(new File("mercurial-tests/testData/rep1"), repo1); + copyRepository(new File("mercurial-tests/testData/rep2"), repo2); + + final VcsRoot root1 = vcsRoot().withUrl(repo1.getAbsolutePath()).build(); + final VcsRoot root2 = vcsRoot().withUrl(repo2.getAbsolutePath()).build(); + //update will initialize mirrors + doUpdate(root1, "4:b06a290a363b", myWorkDir); + doUpdate(root2, "8:b6e2d176fe8e", new File(myWorkDir, "subdir")); + + final File mirrorDir1 = myMirrorManager.getMirrorDir(repo1.getAbsolutePath()); + final File mirrorDir2 = myMirrorManager.getMirrorDir(repo2.getAbsolutePath()); + final Date mirrorDir1LastUsedTime = new Date(myMirrorManager.getLastUsedTime(mirrorDir1)); + final Date mirrorDir2LastUsedTime = new Date(myMirrorManager.getLastUsedTime(mirrorDir2)); + + //run build which uses root1: + final DirectoryCleanersProviderContext ctx = myContext.mock(DirectoryCleanersProviderContext.class); + myContext.checking(new Expectations(){{ + AgentRunningBuild build = myContext.mock(AgentRunningBuild.class, "build" + myBuildCounter++); + allowing(build).getVcsRootEntries(); will(returnValue(asList(new VcsRootEntry(root1, CheckoutRules.DEFAULT)))); + allowing(ctx).getRunningBuild(); will(returnValue(build)); + }}); + + //cleaner should add cleaners only for roots which are not used in the running build + final DirectoryCleanersRegistry registry = myContext.mock(DirectoryCleanersRegistry.class); + myContext.checking(new Expectations() {{ + never(registry).addCleaner(with(mirrorDir1), with(mirrorDir1LastUsedTime)); + one(registry).addCleaner(with(mirrorDir2), with(mirrorDir2LastUsedTime)); + }}); + + myCleaner.registerDirectoryCleaners(ctx, registry); + myContext.assertIsSatisfied(); + } + + + private void doUpdate(@NotNull VcsRoot vcsRoot, @NotNull String toVersion, @NotNull File workDir) throws VcsException { + final AgentRunningBuild build = myContext.mock(AgentRunningBuild.class, "build" + myBuildCounter++); + myContext.checking(new Expectations() {{ + allowing(build).getBuildLogger(); will(returnValue(myLogger)); + allowing(build).getSharedConfigParameters(); will(returnValue(new HashMap<String, String>() {{ + put("teamcity.hg.use.local.mirrors", "true"); + }})); + }}); + myVcsSupport.getUpdater(vcsRoot, CheckoutRules.DEFAULT, toVersion, workDir, build, false).process(IncludeRule.createDefaultInstance(), workDir); + } + +}
--- a/mercurial-tests/src/testng.xml Thu May 10 15:10:35 2012 +0400 +++ b/mercurial-tests/src/testng.xml Fri May 11 12:10:16 2012 +0400 @@ -22,6 +22,7 @@ <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.UnrelatedResitoriesTest"/> <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.CleanupTest"/> <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.MercurialResetCacheHandlerTest"/> + <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.AgentMirrorCleanerTest"/> </classes> </test> </suite>
--- a/mercurial.ipr Thu May 10 15:10:35 2012 +0400 +++ b/mercurial.ipr Fri May 11 12:10:16 2012 +0400 @@ -444,6 +444,7 @@ <library name="TeamCityAPI-agent"> <CLASSES> <root url="jar://$TeamCityDistribution$/devPackage/agent-api.jar!/" /> + <root url="jar://$TeamCityDistribution$/buildAgent/lib/agent.jar!/" /> </CLASSES> <JAVADOC /> <SOURCES>
--- a/mercurial.xml Thu May 10 15:10:35 2012 +0400 +++ b/mercurial.xml Fri May 11 12:10:16 2012 +0400 @@ -128,6 +128,7 @@ </path> <path id="library.teamcityapi-agent.classpath"> + <pathelement location="${path.variable.teamcitydistribution}/buildAgent/lib/agent.jar"/> <pathelement location="${path.variable.teamcitydistribution}/devPackage/agent-api.jar"/> </path>