Mercurial > hg > mercurial
changeset 177:ed8ac2b85eba remote-run/dmitry.neverov/agent_caches2
Add local mirrors on agent
line wrap: on
line diff
--- a/mercurial-agent/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialAgentSideVcsSupport.java Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-agent/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialAgentSideVcsSupport.java Wed Feb 23 14:13:20 2011 +0300 @@ -15,6 +15,7 @@ */ package jetbrains.buildServer.buildTriggers.vcs.mercurial; +import jetbrains.buildServer.agent.BuildAgentConfiguration; import jetbrains.buildServer.agent.BuildProgressLogger; import jetbrains.buildServer.agent.vcs.AgentVcsSupport; import jetbrains.buildServer.agent.vcs.IncludeRuleUpdater; @@ -29,9 +30,15 @@ import org.jetbrains.annotations.NotNull; import java.io.File; -import java.io.IOException; public class MercurialAgentSideVcsSupport extends AgentVcsSupport implements UpdateByIncludeRules { + + private final MirrorManager myMirrorManager; + + public MercurialAgentSideVcsSupport(BuildAgentConfiguration agentConfiguration) { + myMirrorManager = new MirrorManager(agentConfiguration.getCacheDirectory("mercurial")); + } + private void updateWorkingDir(final Settings settings, final String version, final BuildProgressLogger logger) throws VcsException { logger.message("Updating working directory from the local repository copy"); UpdateCommand uc = new UpdateCommand(settings); @@ -41,51 +48,16 @@ logger.message("Working directory updated successfully"); } - private File cloneRepository(final Settings settings) throws VcsException { - File tempDir; + private void cloneFromLocalMirror(final Settings settings, File workingDir) throws VcsException { + File mirrorDir = settings.getLocalMirrorDir(); try { - File parent = FileUtil.createTempDirectory("hg", "clone"); - parent.deleteOnExit(); - tempDir = new File(parent, "hg"); - } catch (IOException e) { - throw new VcsException("Failed to create temp directory: " + e.getLocalizedMessage()); + CloneCommand cc = new CloneCommand(settings, mirrorDir.getCanonicalPath()); + cc.setDestDir(workingDir.getAbsolutePath()); + cc.setUpdateWorkingDir(false); + cc.execute(); + } catch (Exception e) { + throw new VcsException("Failed to clone from local mirror at " + mirrorDir.getAbsolutePath(), e); } - - CloneCommand cc = new CloneCommand(settings); - cc.setDestDir(tempDir.getAbsolutePath()); - - cc.setUpdateWorkingDir(false); - cc.execute(); - return tempDir; - } - - /** - * Moves files from one directory to another with all subdirectories. - * Removes old directory if it became empty. - */ - private static boolean moveDir(File oldDir, File newDir) { - // both old and new directories exist - boolean moveSuccessful = true; - final File[] files = oldDir.listFiles(); - if (files != null) { - for (File file: files) { - if (file.isFile()) { - File destFile = new File(newDir, file.getName()); - destFile.getParentFile().mkdirs(); - if (!file.renameTo(destFile)) { - moveSuccessful = false; - } - } else if (!moveDir(file, new File(newDir, file.getName()))) { - moveSuccessful = false; - } - } - } - - if (moveSuccessful) { - FileUtil.deleteIfEmpty(oldDir); - } - - return moveSuccessful; } @NotNull @@ -103,35 +75,25 @@ public IncludeRuleUpdater getUpdater(@NotNull final VcsRoot vcsRoot, @NotNull final CheckoutRules checkoutRules, @NotNull final String toVersion, @NotNull final File checkoutDirectory, @NotNull final BuildProgressLogger logger) throws VcsException { return new IncludeRuleUpdater() { public void process(@NotNull final IncludeRule includeRule, @NotNull final File workingDir) throws VcsException { - if (includeRule.getTo() != null && includeRule.getTo().length() > 0) { - if (!".".equals(includeRule.getFrom()) && includeRule.getFrom().length() != 0) { - throw new VcsException("Invalid include rule: " + includeRule.toString() + ", Mercurial plugin supports mapping of the form: +:.=>dir only."); - } - } + checkRuleIsValid(includeRule); - Settings settings = new Settings(workingDir, vcsRoot); + Settings settings = new Settings(myMirrorManager, workingDir, vcsRoot); settings.setWorkingDir(workingDir); + updateLocalMirror(settings, logger); if (settings.hasCopyOfRepository()) { - // execute pull command - logger.message("Repository in working directory found, start pulling changes"); - PullCommand pc = new PullCommand(settings); - pc.execute(); - logger.message("Changes successfully pulled"); + if (isClonedFromLocalMirror(settings)) { + logger.message("Repository in working directory found, start pulling changes"); + new PullCommand(settings).execute(); + logger.message("Changes successfully pulled"); + } else { + logger.message("Repository in working directory is cloned from remote repository, clone it from local mirror"); + FileUtil.delete(workingDir); + cloneFromLocalMirror(settings, workingDir); + } } else { - // execute clone command - logger.message("No repository in working directory found, start cloning repository to temporary folder"); - File parentDir = cloneRepository(settings); - logger.message("Repository successfully cloned to: " + parentDir.getAbsolutePath()); - logger.message("Moving repository to working directory: " + workingDir.getAbsolutePath()); - if (!moveDir(parentDir, workingDir)) { - File hgDir = new File(workingDir, ".hg"); - if (hgDir.isDirectory()) { - FileUtil.delete(hgDir); - } - throw new VcsException("Failed to move directory content: " + parentDir.getAbsolutePath() + " to: " + workingDir.getAbsolutePath()); - } - - logger.message("Repository successfully moved to working directory: " + workingDir.getAbsolutePath()); + logger.message("No repository in working directory found, start cloning from local mirror"); + cloneFromLocalMirror(settings, workingDir); + logger.message("Repository successfully cloned to working directory: " + workingDir.getAbsolutePath()); } updateWorkingDir(settings, toVersion, logger); } @@ -140,4 +102,42 @@ } }; } + + private void updateLocalMirror(Settings settings, BuildProgressLogger logger) throws VcsException { + if (settings.hasMirrorRepository()) { + logger.message("Start pulling changes to local mirror at " + settings.getLocalMirrorDir()); + PullCommand pc = new PullCommand(settings, settings.getLocalMirrorDir()); + pc.execute(); + logger.message("Local mirror changes successfully pulled"); + } else { + File mirrorDir = settings.getLocalMirrorDir(); + logger.message("No local mirror found for " + settings.getRepositoryUrl() + ", create mirror at " + mirrorDir.getAbsolutePath()); + logger.message("Clone local mirror at " + mirrorDir); + CloneCommand cc = new CloneCommand(settings); + cc.setDestDir(mirrorDir.getAbsolutePath()); + cc.setUpdateWorkingDir(false); + cc.execute(); + logger.message("Local mirror successfully cloned to " + mirrorDir); + } + } + + private void checkRuleIsValid(IncludeRule includeRule) throws VcsException { + if (includeRule.getTo() != null && includeRule.getTo().length() > 0) { + if (!".".equals(includeRule.getFrom()) && includeRule.getFrom().length() != 0) { + throw new VcsException("Invalid include rule: " + includeRule.toString() + ", Mercurial plugin supports mapping of the form: +:.=>dir only."); + } + } + } + + public boolean isClonedFromLocalMirror(Settings settings) { + try { + File mirrorDir = settings.getLocalMirrorDir(); + File workingDir = settings.getLocalRepositoryDir(); + File hgrc = new File(workingDir, ".hg" + File.separator + "hgrc"); + String config = FileUtil.readText(hgrc); + return config.contains("default = " + mirrorDir.getCanonicalPath()); + } catch (Exception e) { + return false; + } + } }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManager.java Wed Feb 23 14:13:20 2011 +0300 @@ -0,0 +1,233 @@ +package jetbrains.buildServer.buildTriggers.vcs.mercurial; + +import com.intellij.openapi.diagnostic.Logger; +import jetbrains.buildServer.util.FileUtil; +import jetbrains.buildServer.util.Hash; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Manages local mirrors of remote repositories. + * Each unique url get unique local mirror. Each mirror is used for one url only. + * @author dmitry.neverov + */ +public final class MirrorManager { + + private static Logger LOG = Logger.getInstance(MirrorManager.class.getName()); + private static final String MIRROR_DIR_PREFIX = "hg_"; + private static final String MAPPING_FILE_NAME = "map"; + + private final ReadWriteLock myLock = new ReentrantReadWriteLock(); + private final File myRootDir; + /*Only one thread read or write to this file, it is protected by myLock.writeLock()*/ + private final File myMappingFile; + /*Protected by myLock*/ + private final Map<String, File> myMirrors = new HashMap<String, File>(); + private HashCalculator myHash = new StandartHash(); + + /** + * @param rootDir root directory where all mirrors are stored + */ + public MirrorManager(File rootDir) { + myRootDir = rootDir; + myMappingFile = new File(myRootDir, MAPPING_FILE_NAME); + readMappingFromFile(); + } + + + /** + * Get directory of local mirror repository for specified url, if directory is not exists it is created + * @param url url of interest + * @return see above + */ + @NotNull + public File getMirrorDir(@NotNull final String url) { + File result = getMirrorDirWithLock(url); + if (result == null) { + result = createDirFor(url); + } + return result; + } + + + /** + * Get all local mirror repository dirs + * @return see above + */ + @NotNull + public List<File> getMirrors() { + myLock.readLock().lock(); + try { + return new ArrayList<File>(myMirrors.values()); + } finally { + myLock.readLock().unlock(); + } + } + + + //for tests only + void setHashCalculator(HashCalculator hash) { + myHash = hash; + } + + + private File createDirFor(String url) { + File result; + myLock.writeLock().lock(); + try { + File mirrorDir = getUniqueDir(url); + result = saveMappingIfAbsent(url, mirrorDir); + } finally { + myLock.writeLock().unlock(); + } + if (!result.exists()) { + result.mkdirs(); + } + return result; + } + + + private File getMirrorDirWithLock(String url) { + myLock.readLock().lock(); + try { + return myMirrors.get(url); + } finally { + myLock.readLock().unlock(); + } + } + + + //should be called with myLock.writeLock() held + private File saveMappingIfAbsent(String url, File mirrorDir) { + File existing = myMirrors.get(url); + if (existing != null) { + return existing; + } else { + myMirrors.put(url, mirrorDir); + saveMappingToFile(); + return mirrorDir; + } + } + + + private File getUniqueDir(String url) { + myLock.readLock().lock(); + try { + String dirName = MIRROR_DIR_PREFIX + hash(normalize(url)); + File result = PathUtil.getCanonicalFile(new File(myRootDir, dirName)); + while (isUsedForOtherUrl(result, url)) { + dirName = MIRROR_DIR_PREFIX + hash(result.getName()); + result = PathUtil.getCanonicalFile(new File(myRootDir, dirName)); + } + return result; + } finally { + myLock.readLock().unlock(); + } + } + + + private boolean isUsedForOtherUrl(File repositoryDir, String url) { + myLock.readLock().lock(); + try { + for (Map.Entry<String, File> mirror : myMirrors.entrySet()) { + String mirrorUrl = mirror.getKey(); + File mirrorDir = mirror.getValue(); + if (mirrorDir.equals(repositoryDir) && !mirrorUrl.equals(url)) { + return true; + } + } + return false; + } finally { + myLock.readLock().unlock(); + } + } + + + private String hash(String value) { + return String.valueOf(myHash.calc(value)); + } + + + private static String normalize(final String path) { + String normalized = PathUtil.normalizeSeparator(path); + if (path.endsWith("/")) { + return normalized.substring(0, normalized.length()-1); + } + return normalized; + } + + + private void readMappingFromFile() { + myLock.writeLock().lock(); + try { + LOG.debug("Parse mapping file " + myMappingFile.getAbsolutePath()); + for (String line : readLines()) { + int separatorIndex = line.lastIndexOf(" = "); + if (separatorIndex == -1) { + LOG.error("Cannot parse mapping '" + line + "', skip it."); + } else { + String url = line.substring(0, separatorIndex); + String dirName = line.substring(separatorIndex + 3); + File repositoryDir = PathUtil.getCanonicalFile(new File(myRootDir, dirName)); + if (isUsedForOtherUrl(repositoryDir, url)) { + LOG.error("Skip mapping " + line + ": " + dirName + " is used for url other than " + url); + } else { + myMirrors.put(url, PathUtil.getCanonicalFile(new File(myRootDir, dirName))); + } + } + } + } finally { + myLock.writeLock().unlock(); + } + } + + /*Should be called with myLock.writeLock() held*/ + private List<String> readLines() { + if (myMappingFile.exists()) { + try { + return FileUtil.readFile(myMappingFile); + } catch (IOException e) { + LOG.error("Error while reading a mapping file at " + myMappingFile.getAbsolutePath() + " starting with empty mapping", e); + return new ArrayList<String>(); + } + } else { + LOG.info("No mapping file found at " + myMappingFile.getAbsolutePath() + " starting with empty mapping"); + return new ArrayList<String>(); + } + } + + + private void saveMappingToFile() { + myLock.writeLock().lock(); + try { + StringBuilder sb = new StringBuilder(); + for (Map.Entry<String, File> mirror : myMirrors.entrySet()) { + String url = mirror.getKey(); + String dir = mirror.getValue().getName(); + sb.append(url).append(" = ").append(dir).append("\n"); + } + FileUtil.writeFile(myMappingFile, sb.toString()); + } finally { + myLock.writeLock().unlock(); + } + } + + + final static class StandartHash implements HashCalculator { + public long calc(String value) { + return Hash.calc(value); + } + } + + public static interface HashCalculator { + long calc(String value); + } +}
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CloneCommand.java Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CloneCommand.java Wed Feb 23 14:13:20 2011 +0300 @@ -33,6 +33,11 @@ myRepository = getSettings().getRepositoryUrl(); } + public CloneCommand(@NotNull final Settings settings, String localMirror) { + super(settings); + myRepository = localMirror; + } + /** * Sets repository to clone, by default uses repository from the specified settings * @param repo repository path
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/PullCommand.java Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/PullCommand.java Wed Feb 23 14:13:20 2011 +0300 @@ -19,15 +19,23 @@ import jetbrains.buildServer.vcs.VcsException; import org.jetbrains.annotations.NotNull; +import java.io.File; + /** * @author Pavel.Sher * Date: 14.07.2008 */ public class PullCommand extends BaseCommand { + public PullCommand(@NotNull final Settings settings) { super(settings); } + public PullCommand(@NotNull final Settings settings, File repositoryDir) { + this(settings); + setWorkDirectory(repositoryDir.getAbsolutePath()); + } + public void execute() throws VcsException { GeneralCommandLine cli = createCommandLine(); cli.addParameter("pull");
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/Settings.java Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/Settings.java Wed Feb 23 14:13:20 2011 +0300 @@ -16,9 +16,9 @@ package jetbrains.buildServer.buildTriggers.vcs.mercurial.command; import jetbrains.buildServer.buildTriggers.vcs.mercurial.Constants; +import jetbrains.buildServer.buildTriggers.vcs.mercurial.MirrorManager; import jetbrains.buildServer.buildTriggers.vcs.mercurial.PathUtil; import jetbrains.buildServer.log.Loggers; -import jetbrains.buildServer.util.Hash; import jetbrains.buildServer.util.StringUtil; import jetbrains.buildServer.vcs.VcsRoot; import org.jetbrains.annotations.NotNull; @@ -43,12 +43,14 @@ private String myPassword; private String myBranchName; private boolean myUncompressedTransfer = false; + private final MirrorManager myMirrorManager; private static final String DEFAULT_BRANCH_NAME = "default"; - public Settings(@NotNull File workFolderParentDir, @NotNull VcsRoot vcsRoot) { + public Settings(@NotNull MirrorManager mirrorManager, @NotNull File workFolderParentDir, @NotNull VcsRoot vcsRoot) { + myMirrorManager = mirrorManager; myWorkFolderParentDir = workFolderParentDir; - setRepository(vcsRoot.getProperty(Constants.REPOSITORY_PROP)); - setHgCommandPath(vcsRoot.getProperty(Constants.HG_COMMAND_PATH_PROP)); + myRepository = vcsRoot.getProperty(Constants.REPOSITORY_PROP); + myHgCommandPath = vcsRoot.getProperty(Constants.HG_COMMAND_PATH_PROP); myBranchName = vcsRoot.getProperty(Constants.BRANCH_NAME_PROP); myUsername = vcsRoot.getProperty(Constants.USERNAME); @@ -56,13 +58,6 @@ myUncompressedTransfer = "true".equals(vcsRoot.getProperty(Constants.UNCOMPRESSED_TRANSFER)); } - public Settings() { - } - - public void setRepository(@NotNull final String repository) { - myRepository = repository; - } - /** * Returns name of the branch to use (returns 'default' if no branch specified) * @return see above @@ -184,10 +179,6 @@ return userInfo; } - public void setHgCommandPath(@NotNull final String hgCommandPath) { - myHgCommandPath = hgCommandPath; - } - public void setWorkingDir(@NotNull final File workingDir) { myWorkingDir = PathUtil.getCanonicalFile(workingDir); } @@ -202,7 +193,11 @@ return myWorkingDir; } - return getDefaultWorkDir(myWorkFolderParentDir, myRepository); + return myMirrorManager.getMirrorDir(myRepository); + } + + public File getLocalMirrorDir() { + return myMirrorManager.getMirrorDir(myRepository); } /** @@ -210,22 +205,15 @@ * @return see above */ public boolean hasCopyOfRepository() { - // need better way to check that repository copy is ok - return getLocalRepositoryDir().isDirectory() && new File(getLocalRepositoryDir(), ".hg").isDirectory(); + return isValidRepository(getLocalRepositoryDir()); } - public static String DEFAULT_WORK_DIR_PREFIX = "hg_"; - - private static File getDefaultWorkDir(@NotNull File workFolderParentDir, @NotNull String repPath) { - String workingDirname = DEFAULT_WORK_DIR_PREFIX + String.valueOf(Hash.calc(normalize(repPath))); - return PathUtil.getCanonicalFile(new File(workFolderParentDir, workingDirname)); + public boolean hasMirrorRepository() { + return isValidRepository(getLocalMirrorDir()); } - private static String normalize(final String path) { - String normalized = PathUtil.normalizeSeparator(path); - if (path.endsWith("/")) { - return normalized.substring(0, normalized.length()-1); - } - return normalized; + private boolean isValidRepository(File dir) { + // need better way to check that repository copy is ok + return dir.isDirectory() && new File(dir, ".hg").isDirectory(); } }
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupport.java Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupport.java Wed Feb 23 14:13:20 2011 +0300 @@ -56,6 +56,7 @@ private ConcurrentMap<String, Lock> myWorkDirLocks= new ConcurrentHashMap<String, Lock>(); private VcsManager myVcsManager; private File myDefaultWorkFolderParent; + private MirrorManager myMirrorManager; public MercurialVcsSupport(@NotNull final VcsManager vcsManager, @NotNull ServerPaths paths, @@ -63,6 +64,7 @@ @NotNull EventDispatcher<BuildServerListener> dispatcher) { myVcsManager = vcsManager; myDefaultWorkFolderParent = new File(paths.getCachesDir(), "mercurial"); + myMirrorManager = new MirrorManager(myDefaultWorkFolderParent); dispatcher.addListener(new BuildServerAdapter() { @Override public void cleanupFinished() { @@ -79,22 +81,28 @@ super.sourcesVersionReleased(agent); server.getExecutor().submit(new Runnable() { public void run() { - Set<File> clonedRepos = getAllClonedRepos(); - if (clonedRepos == null) return; - for (File f: clonedRepos) { - lockWorkDir(f); - try { - FileUtil.delete(f); - } finally { - unlockWorkDir(f); - } - } + deleteWithLocking(myMirrorManager.getMirrors()); } }); } }); } + public MirrorManager getMirrorManager() { + return myMirrorManager; + } + + private void deleteWithLocking(Collection<File> filesForDelete) { + for (File f : filesForDelete) { + lockWorkDir(f); + try { + FileUtil.delete(f); + } finally { + unlockWorkDir(f); + } + } + } + private Collection<ModifiedFile> computeModifiedFilesForMergeCommit(final Settings settings, final ChangeSet cur) throws VcsException { ChangedFilesCommand cfc = new ChangedFilesCommand(settings); cfc.setRevId(cur.getId()); @@ -563,8 +571,7 @@ } private void removeOldWorkFolders() { - Set<File> workDirs = getAllClonedRepos(); - if (workDirs == null) return; + Set<File> workDirs = new HashSet<File>(myMirrorManager.getMirrors()); for (VcsRoot vcsRoot: getMercurialVcsRoots()) { try { @@ -575,14 +582,7 @@ } } - for (File f: workDirs) { - lockWorkDir(f); - try { - FileUtil.delete(f); - } finally { - unlockWorkDir(f); - } - } + deleteWithLocking(workDirs); } private Collection<VcsRoot> getMercurialVcsRoots() { @@ -595,25 +595,6 @@ return res; } - @Nullable - private Set<File> getAllClonedRepos() { - File workFoldersParent = myDefaultWorkFolderParent; - if (!workFoldersParent.isDirectory()) return null; - - Set<File> workDirs = new HashSet<File>(); - File[] files = workFoldersParent.listFiles(new FileFilter() { - public boolean accept(final File file) { - return file.isDirectory() && file.getName().startsWith(Settings.DEFAULT_WORK_DIR_PREFIX); - } - }); - if (files != null) { - for (File f: files) { - workDirs.add(PathUtil.getCanonicalFile(f)); - } - } - return workDirs; - } - public String label(@NotNull String label, @NotNull String version, @NotNull VcsRoot root, @NotNull CheckoutRules checkoutRules) throws VcsException { syncClonedRepository(root); @@ -649,7 +630,7 @@ } private Settings createSettings(final VcsRoot root) throws VcsException { - Settings settings = new Settings(myDefaultWorkFolderParent, root); + Settings settings = new Settings(myMirrorManager, myDefaultWorkFolderParent, root); String customClonePath = root.getProperty(Constants.SERVER_CLONE_PATH_PROP); if (!StringUtil.isEmptyOrSpaces(customClonePath) && !myDefaultWorkFolderParent.equals(new File(customClonePath).getAbsoluteFile())) { File parentDir = new File(customClonePath);
--- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentSideCheckoutTest.java Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/AgentSideCheckoutTest.java Wed Feb 23 14:13:20 2011 +0300 @@ -15,6 +15,7 @@ */ package jetbrains.buildServer.buildTriggers.vcs.mercurial; +import jetbrains.buildServer.agent.BuildAgentConfiguration; import jetbrains.buildServer.agent.BuildProgressLogger; import jetbrains.buildServer.util.FileUtil; import jetbrains.buildServer.vcs.CheckoutRules; @@ -22,11 +23,13 @@ import jetbrains.buildServer.vcs.VcsException; import jetbrains.buildServer.vcs.VcsRoot; import org.jmock.Mock; +import org.jmock.core.stub.ReturnStub; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.io.File; import java.io.IOException; +import java.util.List; /** * @author Pavel.Sher @@ -37,13 +40,18 @@ private MercurialAgentSideVcsSupport myVcsSupport; private Mock myProgressLoggerMock; private File myWorkDir; + private File myMirrorsRootDir; @Override @BeforeMethod protected void setUp() throws Exception { super.setUp(); - myVcsSupport = new MercurialAgentSideVcsSupport(); + Mock agentConfigMock = new Mock(BuildAgentConfiguration.class); + myMirrorsRootDir = myTempFiles.createTempDir(); + agentConfigMock.stubs().method("getCacheDirectory").will(new ReturnStub(myMirrorsRootDir)); + + myVcsSupport = new MercurialAgentSideVcsSupport((BuildAgentConfiguration) agentConfigMock.proxy()); myProgressLoggerMock = new Mock(BuildProgressLogger.class); myWorkDir = myTempFiles.createTempDir(); myProgressLoggerMock.stubs().method("message"); @@ -105,6 +113,51 @@ checkWorkingDir("patch4/after", workDir); } + public void local_mirror_is_created() throws IOException, VcsException { + List<File> mirrors = FileUtil.getSubDirectories(myMirrorsRootDir); + assertTrue(mirrors.isEmpty()); + VcsRoot root = createVcsRoot(simpleRepo()); + doUpdate(root, "3:9522278aa38d", new IncludeRule(".", ".", null)); + mirrors = FileUtil.getSubDirectories(myMirrorsRootDir); + assertEquals(1, mirrors.size()); + File mirror = mirrors.get(0); + File dotHg = new File(mirror, ".hg"); + assertTrue(dotHg.exists()); + File hgrc = new File(dotHg, "hgrc"); + String hgrcContent = FileUtil.readText(hgrc); + assertTrue(hgrcContent.contains("default = " + root.getProperty(Constants.REPOSITORY_PROP))); + } + + public void new_repository_is_cloned_from_local_mirror() throws IOException, VcsException { + VcsRoot root = createVcsRoot(simpleRepo()); + File workingDir = doUpdate(root, "3:9522278aa38d", new IncludeRule(".", ".", null)); + File mirrorDir = FileUtil.getSubDirectories(myMirrorsRootDir).get(0); + File hgrc = new File(workingDir, ".hg" + File.separator + "hgrc"); + String hgrcContent = FileUtil.readText(hgrc); + assertTrue(hgrcContent.contains("default = " + mirrorDir.getCanonicalPath())); + } + + public void repository_cloned_from_remote_start_cloning_from_local_mirror() throws IOException, VcsException { + VcsRoot root = createVcsRoot(simpleRepo()); + File workingDir = doUpdate(root, "3:9522278aa38d", new IncludeRule(".", ".", null)); + String hgrcContent = FileUtil.readText(new File(workingDir, ".hg" + File.separator + "hgrc")); + + //at this moment repository is cloned from local mirror, if we change mirrorsRootDir and do update + //this repository should start cloning from new local mirror + + //create vcsSupport that use mirrorManager with changed mirrorsRootDir + Mock agentConfigMock = new Mock(BuildAgentConfiguration.class); + myMirrorsRootDir = myTempFiles.createTempDir(); + agentConfigMock.stubs().method("getCacheDirectory").will(new ReturnStub(myMirrorsRootDir)); + myVcsSupport = new MercurialAgentSideVcsSupport((BuildAgentConfiguration) agentConfigMock.proxy()); + + File workingDir2 = doUpdate(root, "3:9522278aa38d", new IncludeRule(".", ".", null)); + File newMirrorDir = FileUtil.getSubDirectories(myMirrorsRootDir).get(0); + String hgrcContent2 = FileUtil.readText(new File(workingDir2, ".hg" + File.separator + "hgrc")); + assertFalse(hgrcContent2.equals(hgrcContent));//repository settings are changed + assertTrue(hgrcContent2.contains("default = " + newMirrorDir.getCanonicalPath()));//now it clones from different local mirror + } + protected String getTestDataPath() { return "mercurial-tests/testData"; }
--- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupportTest.java Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupportTest.java Wed Feb 23 14:13:20 2011 +0300 @@ -312,15 +312,15 @@ VcsRootImpl vcsRoot = createVcsRoot(simpleRepo()); String repPath = vcsRoot.getProperty(Constants.REPOSITORY_PROP); vcsRoot.addProperty(Constants.REPOSITORY_PROP, repPath + "#test_branch"); - Settings settings = new Settings(new File(myServerPaths.getCachesDir()), vcsRoot); + Settings settings = new Settings(myVcs.getMirrorManager(), new File(myServerPaths.getCachesDir()), vcsRoot); assertEquals("test_branch", settings.getBranchName()); vcsRoot.addProperty(Constants.REPOSITORY_PROP, repPath + "#"); - settings = new Settings(new File(myServerPaths.getCachesDir()), vcsRoot); + settings = new Settings(myVcs.getMirrorManager(), new File(myServerPaths.getCachesDir()), vcsRoot); assertEquals("default", settings.getBranchName()); vcsRoot.addProperty(Constants.REPOSITORY_PROP, repPath); - settings = new Settings(new File(myServerPaths.getCachesDir()), vcsRoot); + settings = new Settings(myVcs.getMirrorManager(), new File(myServerPaths.getCachesDir()), vcsRoot); assertEquals("default", settings.getBranchName()); } @@ -404,7 +404,7 @@ VcsRootImpl root = new VcsRootImpl(1, Constants.VCS_NAME); root.addAllProperties(myVcs.getDefaultVcsProperties()); root.addProperty(Constants.REPOSITORY_PROP, "http://host.com/path"); - Settings settings = new Settings(new File("."), root); + Settings settings = new Settings(myVcs.getMirrorManager(), new File("."), root); assertFalse(settings.isUncompressedTransfer()); }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MirrorManagerTest.java Wed Feb 23 14:13:20 2011 +0300 @@ -0,0 +1,106 @@ +package jetbrains.buildServer.buildTriggers.vcs.mercurial; + +import jetbrains.buildServer.TempFiles; +import jetbrains.buildServer.util.Hash; +import junit.framework.TestCase; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * @author dmitry.neverov + */ +@Test +public class MirrorManagerTest extends TestCase { + + private TempFiles myTempFiles; + private File myRootDir; + private MirrorManager myManager; + + + @BeforeMethod + public void setUp() throws Exception { + myTempFiles = new TempFiles(); + myRootDir = myTempFiles.createTempDir(); + myManager = new MirrorManager(myRootDir); + } + + @AfterMethod + public void tearDown() { + myTempFiles.cleanup(); + } + + + public void getMirrorDir_returns_dir_under_root() { + File mirrorDir = myManager.getMirrorDir("hg://some.com/repository.hg"); + assertEquals(myRootDir, mirrorDir.getParentFile()); + assertTrue(mirrorDir.exists()); + } + + + public void getMirrorDir_returns_same_result_for_same_url() { + String url = "hg://some.com/repository.hg"; + assertEquals(myManager.getMirrorDir(url), myManager.getMirrorDir(url)); + } + + + public void getMirrors_remember_created_repositories() { + File mirrorDir1 = myManager.getMirrorDir("hg://some.com/repository.hg"); + File mirrorDir2 = myManager.getMirrorDir("hg://other.com/repository.hg"); + List<File> mirrors = myManager.getMirrors(); + assertEquals(2, mirrors.size()); + assertTrue(mirrors.contains(mirrorDir1)); + assertTrue(mirrors.contains(mirrorDir2)); + } + + + public void should_handle_url_collisions() throws IOException { + final String url1 = "hg://some.com/repository.hg"; + final String url2 = "hg://other.com/repository.hg"; + + MirrorManager.HashCalculator hashWithCollision = new MirrorManager.HashCalculator() { + public long calc(String value) { + if (value.equals(url1) || value.equals(url2)) { + return 0;//emulate collision + } else { + return Hash.calc(value); + } + } + }; + + //alone they get dir with the same name: + MirrorManager mm1 = new MirrorManager(myTempFiles.createTempDir()); + mm1.setHashCalculator(hashWithCollision); + File separateMirrorDir1 = mm1.getMirrorDir(url1); + + MirrorManager mm2 = new MirrorManager(myTempFiles.createTempDir()); + mm2.setHashCalculator(hashWithCollision); + File separateMirrorDir2 = mm2.getMirrorDir(url2); + + assertEquals(separateMirrorDir1.getName(), separateMirrorDir2.getName()); + + myManager.setHashCalculator(hashWithCollision); + File mirrorDir1 = myManager.getMirrorDir(url1); + File mirrorDir2 = myManager.getMirrorDir(url2); + assertFalse(mirrorDir1.equals(mirrorDir2)); + } + + + public void should_survive_restart() throws IOException { + String url1 = "hg://some.com/repository.hg"; + String url2 = "hg://other.com/repository.hg?param = 1"; + File mirrorDir1 = myManager.getMirrorDir(url1); + File mirrorDir2 = myManager.getMirrorDir(url2); + + //emulate restart by creating a new manager for the same rootDir + MirrorManager manager = new MirrorManager(myRootDir); + + assertEquals(2, manager.getMirrors().size()); + assertTrue(manager.getMirrors().contains(mirrorDir1)); + assertTrue(manager.getMirrors().contains(mirrorDir2)); + } +}
--- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/SettingsTest.java Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/SettingsTest.java Wed Feb 23 14:13:20 2011 +0300 @@ -15,9 +15,12 @@ */ package jetbrains.buildServer.buildTriggers.vcs.mercurial; +import jetbrains.buildServer.TempFiles; import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.Settings; import jetbrains.buildServer.vcs.impl.VcsRootImpl; -import junit.framework.Assert; +import junit.framework.TestCase; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.io.File; @@ -26,67 +29,83 @@ * @author Pavel.Sher */ @Test -public class SettingsTest extends Assert { +public class SettingsTest extends TestCase { + + private TempFiles myTempFiles = new TempFiles(); + private MirrorManager myMirrorManager; + + @Override + @BeforeMethod + public void setUp() throws Exception { + myMirrorManager = new MirrorManager(myTempFiles.createTempDir()); + } + + @Override + @AfterMethod + public void tearDown() throws Exception { + myTempFiles.cleanup(); + } + public void test_url_without_credentials() { VcsRootImpl vcsRoot = createVcsRoot("http://host.com/path"); - Settings settings = new Settings(new File("."), vcsRoot); + Settings settings = new Settings(myMirrorManager, new File("."), vcsRoot); assertEquals("http://user:pwd@host.com/path", settings.getRepositoryUrl()); } public void test_url_with_credentials() { VcsRootImpl vcsRoot = createVcsRoot("http://user:pwd@host.com/path"); - Settings settings = new Settings(new File("."), vcsRoot); + Settings settings = new Settings(myMirrorManager, new File("."), vcsRoot); assertEquals("http://user:pwd@host.com/path", settings.getRepositoryUrl()); } public void test_url_with_username() { VcsRootImpl vcsRoot = createVcsRoot("http://user@host.com/path"); - Settings settings = new Settings(new File("."), vcsRoot); + Settings settings = new Settings(myMirrorManager, new File("."), vcsRoot); assertEquals("http://user:pwd@host.com/path", settings.getRepositoryUrl()); } public void test_url_with_at_after_slash() { VcsRootImpl vcsRoot = createVcsRoot("http://host.com/path@"); - Settings settings = new Settings(new File("."), vcsRoot); + Settings settings = new Settings(myMirrorManager, new File("."), vcsRoot); assertEquals("http://user:pwd@host.com/path@", settings.getRepositoryUrl()); } public void test_url_with_at_in_username() { VcsRootImpl vcsRoot = createVcsRoot("http://host.com/path", "my.name@gmail.com", "1234"); - Settings settings = new Settings(new File("."), vcsRoot); + Settings settings = new Settings(myMirrorManager, new File("."), vcsRoot); assertEquals("http://my.name%40gmail.com:1234@host.com/path", settings.getRepositoryUrl()); } /** TW-13768 */ public void test_underscore_in_host() { VcsRootImpl vcsRoot = createVcsRoot("http://Klekovkin.SDK_GARANT:8000/", "my.name@gmail.com", "1234"); - Settings settings = new Settings(new File("."), vcsRoot); + Settings settings = new Settings(myMirrorManager, new File("."), vcsRoot); assertEquals("http://my.name%40gmail.com:1234@Klekovkin.SDK_GARANT:8000/", settings.getRepositoryUrl()); } /** TW-13768 */ public void test_underscore_in_host_with_credentials_in_url() { VcsRootImpl vcsRoot = createVcsRoot("http://me:mypass@Klekovkin.SDK_GARANT:8000/"); - Settings settings = new Settings(new File("."), vcsRoot); + Settings settings = new Settings(myMirrorManager, new File("."), vcsRoot); assertEquals("http://me:mypass@Klekovkin.SDK_GARANT:8000/", settings.getRepositoryUrl()); } public void test_windows_path() throws Exception { VcsRootImpl vcsRoot = createVcsRoot("c:\\windows\\path"); - Settings settings = new Settings(new File("."), vcsRoot); + Settings settings = new Settings(myMirrorManager, new File("."), vcsRoot); assertEquals("c:\\windows\\path", settings.getRepositoryUrl()); } public void test_file_scheme_has_no_credentials() { VcsRootImpl vcsRoot = createVcsRoot("file:///path/to/repo", "my.name@gmail.com", "1234"); - Settings settings = new Settings(new File("."), vcsRoot); + Settings settings = new Settings(myMirrorManager, new File("."), vcsRoot); assertEquals("file:///path/to/repo", settings.getRepositoryUrl()); } public void uncompressed_transfer() { VcsRootImpl root = createVcsRoot("http://host.com/path"); root.addProperty(Constants.UNCOMPRESSED_TRANSFER, "true"); - Settings settings = new Settings(new File("."), root); + Settings settings = new Settings(myMirrorManager, new File("."), root); assertTrue(settings.isUncompressedTransfer()); }
--- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/BaseCommandTestCase.java Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/BaseCommandTestCase.java Wed Feb 23 14:13:20 2011 +0300 @@ -19,6 +19,7 @@ import jetbrains.buildServer.TempFiles; import jetbrains.buildServer.buildTriggers.vcs.mercurial.Constants; import jetbrains.buildServer.buildTriggers.vcs.mercurial.LocalRepositoryUtil; +import jetbrains.buildServer.buildTriggers.vcs.mercurial.MirrorManager; import jetbrains.buildServer.buildTriggers.vcs.mercurial.Util; import jetbrains.buildServer.vcs.VcsException; import jetbrains.buildServer.vcs.VcsRoot; @@ -35,6 +36,11 @@ private String myUsername; private String myPassword; private boolean myCloneRequired; + private MirrorManager myManager; + + public BaseCommandTestCase() { + myManager = new MirrorManager(new File(".")); + } protected void setRepository(final String repository, boolean cloneRequired) { myRepository = repository; @@ -72,7 +78,7 @@ final File workingDir = new File(parentDir, "rep").getAbsoluteFile(); VcsRoot vcsRoot = new VcsRootImpl(1, vcsRootProps); - Settings settings = new Settings(workingDir.getParentFile(), vcsRoot); + Settings settings = new Settings(myManager, workingDir.getParentFile(), vcsRoot); settings.setWorkingDir(workingDir); try { if (myCloneRequired) {
--- a/mercurial-tests/src/testng.xml Wed Feb 16 13:35:57 2011 +0300 +++ b/mercurial-tests/src/testng.xml Wed Feb 23 14:13:20 2011 +0300 @@ -8,6 +8,7 @@ <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.MercurialVcsSupportTest"/> <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.AgentSideCheckoutTest"/> <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.SettingsTest"/> + <class name="jetbrains.buildServer.buildTriggers.vcs.mercurial.MirrorManagerTest"/> </classes> </test> </suite>