Mercurial > hg > mercurial
changeset 557:2b8299ba321d
Speed up changes collecting
line wrap: on
line diff
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommandUtil.java Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommandUtil.java Fri Mar 01 16:03:00 2013 +0400 @@ -49,6 +49,7 @@ @Override public void onProcessFinished(Process ps) { long duration = System.currentTimeMillis() - start; + System.out.println("Command " + command + " took " + duration + "ms"); Loggers.VCS.debug("Command " + command + " took " + duration + "ms"); } }); @@ -77,5 +78,6 @@ private static void logCommandOutput(@NotNull String command, @NotNull CommandResult result) { Loggers.VCS.debug("Command " + command + " output:\n" + result.getStdout()); +// System.out.println("Command " + command + " output:\n " + result.getStdout()); } }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/LoadDagCommand.java Fri Mar 01 16:03:00 2013 +0400 @@ -0,0 +1,52 @@ +package jetbrains.buildServer.buildTriggers.vcs.mercurial.command; + +import com.intellij.openapi.util.Pair; +import jetbrains.buildServer.util.StringUtil; +import jetbrains.buildServer.vcs.VcsException; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +public class LoadDagCommand extends VcsRootCommand { + + private final File myDagLogTemplate; + + public LoadDagCommand(@NotNull CommandSettings commandSettings, + @NotNull String hgPath, + @NotNull File workingDir, + @NotNull AuthSettings authSettings, + @NotNull File dagLogTemplate) { + super(commandSettings, hgPath, workingDir, authSettings); + myDagLogTemplate = dagLogTemplate; + } + + @NotNull + public List<Pair<String, String>> call() throws VcsException { + List<Pair<String, String>> edges = new ArrayList<Pair<String, String>>(); + MercurialCommandLine cli = createCommandLine(); + cli.addParameter("log"); + cli.addParameter("--style=" + myDagLogTemplate.getAbsolutePath()); + CommandResult res = runCommand(cli); + String output = res.getStdout(); + String fromNode = null; + for (String line : StringUtil.splitByLines(output)) { + String[] revs = line.split(" "); + if (revs.length == 0) + continue; + if (fromNode != null) { + edges.add(Pair.create(fromNode, revs[0])); + fromNode = null; + } + if (revs.length == 1) { + fromNode = revs[0]; + } else { + edges.add(Pair.create(revs[0], revs[1])); + if (revs.length == 3) + edges.add(Pair.create(revs[0], revs[2])); + } + } + return edges; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial-server/resources/buildServerResources/dag.template Fri Mar 01 16:03:00 2013 +0400 @@ -0,0 +1,2 @@ +changeset = '{node|short} {parents}\n' +parent = '{node|short} ' \ No newline at end of file
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupport.java Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupport.java Fri Mar 01 16:03:00 2013 +0400 @@ -488,7 +488,7 @@ if (toRevision.equals(fromRevision)) continue; - List<String> fromRevisions = ctx.getFromRevisionsForBranch(hgRoot, fromRevision, toRevision); + Collection<String> fromRevisions = ctx.getFromRevisionsForBranch(hgRoot, toRevision); List<ModificationData> branchChanges = collectChanges(ctx, root, fromRevisions, toRevision, rules); for (ModificationData change : branchChanges) { if (!ctx.isReportedModification(change)) { @@ -558,7 +558,7 @@ private List<ModificationData> collectChanges(@NotNull OperationContext ctx, @NotNull VcsRoot root, - @NotNull List<String> fromVersion, + @NotNull Collection<String> fromVersion, @Nullable String currentVersion, @NotNull CheckoutRules checkoutRules) throws VcsException { HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root); @@ -657,7 +657,7 @@ @NotNull - private List<ChangeSet> getChangesets(@NotNull OperationContext ctx, @NotNull final HgVcsRoot root, @NotNull final List<String> fromVersions, @Nullable final String toVersion) throws VcsException { + private List<ChangeSet> getChangesets(@NotNull OperationContext ctx, @NotNull final HgVcsRoot root, @NotNull final Collection<String> fromVersions, @Nullable final String toVersion) throws VcsException { if (toVersion == null) return Collections.emptyList(); List<String> fromCommits = new ArrayList<String>(); @@ -667,10 +667,17 @@ String toCommit = new ChangeSetRevision(toVersion).getId(); try { ServerHgRepo repo = createRepo(ctx, root); - return repo.collectChanges(root) + List<ChangeSet> csets = repo.collectChanges(root) .fromRevision(fromCommits) .toRevision(toCommit) .call(); + Iterator<ChangeSet> iter = csets.iterator(); + while (iter.hasNext()) { + ChangeSet cset = iter.next(); + if (fromVersions.contains(cset.getId())) + iter.remove(); + } + return csets; } catch (UnknownRevisionException e) { Loggers.VCS.warn("Revision '" + e.getRevision() + "' is unknown, will return no changes"); return Collections.emptyList();
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/OperationContext.java Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/OperationContext.java Fri Mar 01 16:03:00 2013 +0400 @@ -2,7 +2,9 @@ import com.intellij.openapi.util.Pair; import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.AuthSettings; +import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.ChangeSetRevision; import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.HgVcsRoot; +import jetbrains.buildServer.util.graph.*; import jetbrains.buildServer.vcs.ModificationData; import jetbrains.buildServer.vcs.RepositoryStateData; import jetbrains.buildServer.vcs.VcsException; @@ -20,6 +22,8 @@ private final HgPathProvider myHgPathProvider; private final RepositoryStateData myFromState; private final RepositoryStateData myToState; +// private final Map<VcsRootKey, DAG<ChangeSetRevision>> myDags = new HashMap<VcsRootKey, DAG<ChangeSetRevision>>(); + private final Map<VcsRootKey, DAG<String>> myDags = new HashMap<VcsRootKey, DAG<String>>(); private Set<File> mySyncedDirs = new HashSet<File>(); private Map<String, HgVersion> myHgVersions = new HashMap<String, HgVersion>(); private Map<String, Set<SubrepoChangesInterval>> myProcessedSubrepoChanges = new HashMap<String, Set<SubrepoChangesInterval>>();//subrepo url -> processed changes intervals @@ -142,20 +146,24 @@ * This method detects such a case and returns [99, 100] as fromRevisions for 101. */ @NotNull - public List<String> getFromRevisionsForBranch(@NotNull HgVcsRoot root, - @NotNull String fromRevision, - @NotNull String toRevision) throws VcsException { - List<String> fromRevisions = new ArrayList<String>(); - fromRevisions.add(fromRevision); - + public Collection<String> getFromRevisionsForBranch(@NotNull HgVcsRoot root, @NotNull String toRevision) throws VcsException { + Set<String> fromRevisions = new HashSet<String>(); syncRepository(root); + ServerHgRepo repo = createRepo(myVcs.getWorkingDir(root), myHgPathProvider.getHgPath(root), root.getAuthSettings()); + if (myToState.getBranchRevisions().size() > 1) { + VcsRootKey rootKey = VcsRootKey.create(root); + DAG<String> dag = myDags.get(rootKey); + if (dag == null) { + dag = repo.loadDag(); + myDags.put(rootKey, dag); + } + FindIntervalVisitor visitor = new FindIntervalVisitor(dag, myFromState.getBranchRevisions().values()); + dag.breadthFirstSearch(toRevision, visitor); + fromRevisions.addAll(visitor.getEndpoints()); + } - ServerHgRepo repo = createRepo(myVcs.getWorkingDir(root), myHgPathProvider.getHgPath(root), root.getAuthSettings()); - List<String> parentRevisions = repo.parents().ofRevision(toRevision).call(); - for (String parent : parentRevisions) { - if (!fromRevision.equals(parent) && myFromState.getBranchRevisions().values().contains(parent)) - fromRevisions.add(parent); - } + if (fromRevisions.isEmpty()) + fromRevisions.add(toRevision); return fromRevisions; } @@ -175,4 +183,60 @@ return new VcsRootKey(root.getProperty(Constants.REPOSITORY_PROP), root.getProperty("teamcity.internal.subrepo.path")); } } + + + + private final static class FindIntervalVisitor extends BFSVisitorAdapter<String> { + + private final DAG<String> myDag; + private final Collection<String> myUninterestingRevisions; + private final Set<String> myVisitedUninterestingRevisions = new HashSet<String>(); + private final Set<String> myEndpoints = new HashSet<String>(); + + private FindIntervalVisitor(@NotNull DAG<String> dag, @NotNull Collection<String> uninteresting) { + myDag = dag; + myUninterestingRevisions = uninteresting; + } + + @Override + public boolean discover(@NotNull String revision) { + markUninteresting(); + if (!isInteresting(revision)) { + addEndpoint(revision); + return false; + } + return true; + } + + private void markUninteresting() { + if (!myVisitedUninterestingRevisions.isEmpty()) + return; + for (String rev : myUninterestingRevisions) { + myDag.breadthFirstSearch(rev, new MarkUninterestingRevisions()); + } + } + + private boolean isInteresting(@NotNull String revision) { + return !myVisitedUninterestingRevisions.contains(revision); + } + + private void addEndpoint(@NotNull String revision) { + myEndpoints.add(revision); + } + + @NotNull + public Set<String> getEndpoints() { + return myEndpoints; + } + + private class MarkUninterestingRevisions extends BFSVisitorAdapter<String> { + @Override + public boolean discover(@NotNull String node) { + if (myVisitedUninterestingRevisions.contains(node)) + return false; + myVisitedUninterestingRevisions.add(node); + return true; + } + } + } }
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/RepoFactory.java Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/RepoFactory.java Fri Mar 01 16:03:00 2013 +0400 @@ -21,6 +21,7 @@ private final CommandSettingsFactory myCommandSettingsFactory; private File myLogTemplate; private File myLogNoFilesTemplate; + private File myDagTemplate; public RepoFactory(@NotNull ServerPluginConfig config, @NotNull CommandSettingsFactory commandSettingsFactory) throws IOException { @@ -28,13 +29,15 @@ myCommandSettingsFactory = commandSettingsFactory; myLogTemplate = createLogTemplate(); myLogNoFilesTemplate = createLogNoFilesTemplate(); + myDagTemplate = createDagTemplate(); } @NotNull public ServerHgRepo create(@NotNull File workingDir, @NotNull String hgPath, @NotNull AuthSettings authSettings) throws VcsException { - return new ServerHgRepo(myCommandSettingsFactory, myConfig, workingDir, hgPath, authSettings).withLogTemplates(getTemplate(), getLogNoFilesTemplate()); + return new ServerHgRepo(myCommandSettingsFactory, myConfig, workingDir, hgPath, authSettings) + .withLogTemplates(getTemplate(), getLogNoFilesTemplate(), getDagTemplate()); } public void dispose() { @@ -63,15 +66,32 @@ } } + private File getDagTemplate() throws VcsException { + if (myDagTemplate.isFile() && myDagTemplate.exists()) + return myDagTemplate; + try { + myDagTemplate = createLogNoFilesTemplate(); + return myDagTemplate; + } catch (IOException e) { + throw new VcsException("Cannot create mercurial log template", e); + } + } + private File createLogTemplate() throws IOException { - File template = createTempFile("teamcity", "hg.log.template"); - FileUtil.copyResource(RepoFactory.class, "/buildServerResources/log.template", template); - return template; + return createTmpTemplateFile("hg.log.template", "/buildServerResources/log.template"); } private File createLogNoFilesTemplate() throws IOException { - File template = createTempFile("teamcity", "hg.short.log.template"); - FileUtil.copyResource(RepoFactory.class, "/buildServerResources/log.no.files.template", template); + return createTmpTemplateFile("hg.short.log.template", "/buildServerResources/log.no.files.template"); + } + + private File createDagTemplate() throws IOException { + return createTmpTemplateFile("hg.dag.template", "/buildServerResources/dag.template"); + } + + private File createTmpTemplateFile(@NotNull String tmpFileSuffix, @NotNull String resourcePath) throws IOException { + File template = createTempFile("teamcity", tmpFileSuffix); + FileUtil.copyResource(RepoFactory.class, resourcePath, template); return template; } }
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerHgRepo.java Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/ServerHgRepo.java Fri Mar 01 16:03:00 2013 +0400 @@ -1,12 +1,16 @@ package jetbrains.buildServer.buildTriggers.vcs.mercurial; +import com.intellij.openapi.util.Pair; import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.*; +import jetbrains.buildServer.util.graph.DAG; +import jetbrains.buildServer.util.graph.DAGs; import jetbrains.buildServer.vcs.VcsException; import jetbrains.buildServer.vcs.VcsRoot; import jetbrains.buildServer.vcs.VcsRootInstance; import org.jetbrains.annotations.NotNull; import java.io.File; +import java.util.List; /** * @author dmitry.neverov @@ -18,6 +22,7 @@ private final ServerPluginConfig myConfig; private File myLogTemplate; private File myLogNoFilesTemplate; + private File myDagTemplate; private OperationContext myContext; public ServerHgRepo(@NotNull CommandSettingsFactory commandSettingsFactory, @@ -34,9 +39,12 @@ myContext = context; } - public ServerHgRepo withLogTemplates(@NotNull File logTemplate, @NotNull File logNoFilesTemplate) { + public ServerHgRepo withLogTemplates(@NotNull File logTemplate, + @NotNull File logNoFilesTemplate, + @NotNull File dagTemplate) { myLogTemplate = logTemplate; myLogNoFilesTemplate = logNoFilesTemplate; + myDagTemplate = dagTemplate; return this; } @@ -80,4 +88,11 @@ long parentId = ((VcsRootInstance) root).getParentId(); return myConfig.getRevsetParentRootIds().contains(parentId); } + + @NotNull + public DAG<String> loadDag() throws VcsException { +// List<Pair<ChangeSetRevision, ChangeSetRevision>> edges = new LoadDagCommand(myCommandSettingsFactory.create(), myHgPath, myWorkingDir, myAuthSettings, myDagTemplate).call(); + List<Pair<String, String>> edges = new LoadDagCommand(myCommandSettingsFactory.create(), myHgPath, myWorkingDir, myAuthSettings, myDagTemplate).call(); + return DAGs.createFromEdges(edges); + } }
--- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupportTest.java Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupportTest.java Fri Mar 01 16:03:00 2013 +0400 @@ -39,6 +39,7 @@ import static jetbrains.buildServer.util.Util.map; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.not; @Test public class MercurialVcsSupportTest extends BaseMercurialTestCase { @@ -609,7 +610,27 @@ RepositoryStateData.createVersionState("default", map("default", "505c5b9d01e6", "personal-branch", "96b78d73081d")), RepositoryStateData.createVersionState("default", map("default", "df04faa7575a", "personal-branch", "9ec402c74298")), CheckoutRules.DEFAULT); - assertEquals(changes.size(), 4); + assertThat(changes, not(hasItem(modificationData().withVersion("dec47d2d49bf")))); + } + + + public void should_not_report_duplicate_changes2() throws Exception { + VcsRootImpl root = createVcsRoot(myRep2Path); + List<ModificationData> changes = myVcs.collectChanges(root, + RepositoryStateData.createVersionState("default", map("default", "528572bbf77b", "personal-branch", "27184c50d7ef")), + RepositoryStateData.createVersionState("default", map("default", "4780519e01aa", "personal-branch", "fd50e4842211")), + CheckoutRules.DEFAULT); + assertThat(changes, not(hasItem(modificationData().withVersion("4dbb87c381be")))); + } + + + public void should_not_report_all_changes_in_repository_if_default_branch_is_unrelated() throws Exception { + VcsRootImpl root = createVcsRoot(myRep2Path); + List<ModificationData> changes = myVcs.collectChanges(root, + RepositoryStateData.createVersionState("NULL", map("default", "1f355761350e")), + RepositoryStateData.createVersionState("NULL", map("default", "1f355761350e", "personal-branch", "fd50e4842211")), + CheckoutRules.DEFAULT); + assertEquals(0, changes.size()); }
--- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/SubrepoChangesTest.java Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/SubrepoChangesTest.java Fri Mar 01 16:03:00 2013 +0400 @@ -20,6 +20,7 @@ import static jetbrains.buildServer.buildTriggers.vcs.mercurial.ModificationDataMatcher.modificationData; import static jetbrains.buildServer.buildTriggers.vcs.mercurial.Util.copyRepository; import static jetbrains.buildServer.buildTriggers.vcs.mercurial.VcsRootBuilder.vcsRoot; +import static jetbrains.buildServer.util.Util.map; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; import static org.testng.AssertJUnit.assertEquals; @@ -158,4 +159,28 @@ assertThat(changes, hasItem(modificationData().withVersion("1f9eb39a3921").withVcsRootProperties(subrepoRootProperties))); } + + + private void slow(@NotNull HgVersion _) throws IOException, VcsException { + RepositoryStateData from = RepositoryStateData.createVersionState("default", map("default", "bd3cd9555f51")); + + RepositoryStateData to = RepositoryStateData.createVersionState("default", map("default", "ae2ba34d0b2c")); + + ServerPluginConfig pluginConfig = new ServerPluginConfigBuilder() + .cachesDir(new File("/home/nd/sandbox/tc/8.0/.BuildServer/system/caches/mercurial")) + .detectSubrepoChanges(true) + .build(); + myVcs = mercurialSupport().withConfig(pluginConfig).build(); + + VcsRoot root = vcsRoot().withUrl("http://hg.labs.intellij.net/ReSharper") + .withUserName("hg-ro") + .withPassword("9ZO463dVyo") + .withSubrepoChanges(true).build(); + + List<ModificationData> changes = myVcs.collectChanges(root, from, to, CheckoutRules.DEFAULT); + + boolean pause = true; + } + + }
--- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/UnrelatedResitoriesTest.java Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/UnrelatedResitoriesTest.java Fri Mar 01 16:03:00 2013 +0400 @@ -25,7 +25,7 @@ @Test public class UnrelatedResitoriesTest { - private final static String CURRENT_VERSION_OF_NEW_REPO = "a2672ee13212"; + private final static String CURRENT_VERSION_OF_NEW_REPO = "4780519e01aa"; private MercurialVcsSupport myVcs; private TempFiles myTempFiles; private File myRepositoryLocation;
--- a/mercurial-tests/testData/README Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-tests/testData/README Fri Mar 01 16:03:00 2013 +0400 @@ -24,6 +24,26 @@ rep2 history: +o 31:1f355761350e create branch NULL (NULL) tip + +o 30:4780519e01aa continue default tip +| +| o 29:fd50e4842211 continue topic (topic) +| | +o | 28:737c6f57ef84 merge topic +|\| +| o 27:2a368008e4d9 topic2 (topic) +| | +| o 26:27184c50d7ef topic1 (topic) +| | +| o 25:4dbb87c381be start topic (topic) +| | +o | 24:528572bbf77b default3 +| | +o | 23:029040d32471 default2 +| | +o | 22:32f2afc19d75 default1 +|/ o 21:a2672ee13212 Tag for test //add tag t1 on 43023ea3f13b | o 20:43023ea3f13b Merge closed personal-branch
--- a/mercurial-tests/testData/rep2/hg/store/fncache Thu Feb 28 20:38:04 2013 +0400 +++ b/mercurial-tests/testData/rep2/hg/store/fncache Fri Mar 01 16:03:00 2013 +0400 @@ -1,11 +1,12 @@ data/dir4/file4.txt.i +data/dir1/file1.txt.i data/dir3/file3.txt.i -data/dir1/file1.txt.i -data/file.txt.i data/.hgtags.i data/dir4/file42.txt.i data/dir5/file5.txt.i -data/dir2/file2.txt.i +data/file.txt.i +data/a.i data/dir4/file43.txt.i +data/dir6/file6.txt.i data/dir4/file41.txt.i -data/dir6/file6.txt.i +data/dir2/file2.txt.i