view mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCollectChangesPolicy.java @ 875:b06ea5c379cc

more progress for 'hg pull' commands
author Dmitry Neverov <dmitry.neverov@jetbrains.com>
date Wed, 01 Oct 2014 14:21:31 +0200
parents 894b7e0c9dd6
children 45311425ee3c
line wrap: on
line source
/*
 * Copyright 2000-2014 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package jetbrains.buildServer.buildTriggers.vcs.mercurial;

import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.*;
import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.exception.UnknownRevisionException;
import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.exception.WrongSubrepoUrlException;
import jetbrains.buildServer.log.Loggers;
import jetbrains.buildServer.vcs.*;
import jetbrains.buildServer.vcs.impl.VcsRootImpl;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.*;

import static java.util.Arrays.asList;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;

public class MercurialCollectChangesPolicy implements CollectChangesBetweenRoots, CollectChangesBetweenRepositories {

  private final static AscendingRevNums ASCENDING_REV_NUMS = new AscendingRevNums();

  private final VcsOperationProgressProvider myProgressProvider;
  private final MercurialVcsSupport myVcs;
  private final ServerPluginConfig myConfig;
  private final HgVcsRootFactory myHgVcsRootFactory;
  private final RepoFactory myRepoFactory;
  private final HgPathProvider myHgPathProvider;

  public MercurialCollectChangesPolicy(@NotNull VcsOperationProgressProvider progressProvider,
                                       @NotNull MercurialVcsSupport vcs,
                                       @NotNull ServerPluginConfig config,
                                       @NotNull HgVcsRootFactory hgVcsRootFactory,
                                       @NotNull RepoFactory repoFactory,
                                       @NotNull HgPathProvider hgPathProvider) {
    myProgressProvider = progressProvider;
    myVcs = vcs;
    myConfig = config;
    myHgVcsRootFactory = hgVcsRootFactory;
    myRepoFactory = repoFactory;
    myHgPathProvider = hgPathProvider;
  }


  @NotNull
  public RepositoryStateData getCurrentState(@NotNull VcsRoot root) throws VcsException {
    final HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root);
    VcsCallable<Map<String, String>> cmd = new VcsCallable<Map<String, String>>() {
      public Map<String, String> call() throws VcsException {
        return getHeads(hgRoot);
      }
    };
    SyncSettings<Map<String, String>> settings = new SyncSettings<Map<String, String>>(cmd);
    settings.setProgressConsumer(createProgressConsumer());
    Map<String, String> revisions = myVcs.syncRepository(hgRoot, settings);
    String defaultBranchName = hgRoot.getBranchName();
    if (revisions.get(defaultBranchName) == null && !hgRoot.isIgnoreMissingDefaultBranch()) {
      throw new VcsException("Cannot find revision of the default branch '" +
              defaultBranchName + "' of vcs root " + myVcs.describeVcsRoot(root));
    }
    return RepositoryStateData.createVersionState(defaultBranchName, revisions);
  }

  @NotNull
  public Map<String, String> getHeads(@NotNull final HgVcsRoot hgRoot) throws VcsException {
    boolean includeTags = myConfig.useTagsAsBranches() && hgRoot.useTagsAsBranches();
    return myVcs.createRepo(hgRoot).getBranchRevisions(myConfig.bookmarksEnabled(), includeTags);
  }

  @NotNull
  public List<ModificationData> collectChanges(@NotNull VcsRoot fromRoot,
                                               @NotNull RepositoryStateData fromState,
                                               @NotNull VcsRoot toRoot,
                                               @NotNull RepositoryStateData toState,
                                               @NotNull CheckoutRules rules) throws VcsException {
    return collectChanges(toRoot, fromState, toState, rules);
  }


  @NotNull
  public List<ModificationData> collectChanges(@NotNull VcsRoot root,
                                               @NotNull RepositoryStateData fromState,
                                               @NotNull RepositoryStateData toState,
                                               @NotNull CheckoutRules rules) throws VcsException {
    List<ModificationData> changes = new ArrayList<ModificationData>();
    HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root);
    OperationContext ctx = new OperationContext(myVcs, myRepoFactory, myHgPathProvider, fromState, toState);
    ctx.setProgressConsumer(createProgressConsumer());
    for (Map.Entry<String, String> entry : toState.getBranchRevisions().entrySet()) {
      String branch = entry.getKey();
      String toRevision = entry.getValue();
      String fromRevision = fromState.getBranchRevisions().get(branch);
      if (fromRevision == null)
        fromRevision = fromState.getBranchRevisions().get(fromState.getDefaultBranchName());
      if (toRevision.equals(fromRevision) || fromRevision == null)
        continue;

      Collection<String> fromRevisions = ctx.getFromRevisionsForBranch(hgRoot, fromRevision, toRevision, ctx.getProgressConsumer());
      List<ModificationData> branchChanges = collectChanges(ctx, root, fromRevisions, toRevision, rules);
      for (ModificationData change : branchChanges) {
        if (!ctx.isReportedModification(change)) {
          changes.add(change);
          ctx.markAsReported(change);
        }
      }
    }
    changes.addAll(getSubrepoChanges(ctx, root, changes));
    return changes;
  }


  @NotNull
  public List<ModificationData> collectChanges(@NotNull VcsRoot fromRoot,
                                               @NotNull String fromRootRevision,
                                               @NotNull VcsRoot toRoot,
                                               @Nullable String toRootRevision,
                                               @NotNull CheckoutRules checkoutRules) throws VcsException {
    HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(toRoot);
    SyncSettings<Void> settings = new SyncSettings<Void>(VcsCallable.NO_OP);
    settings.setProgressConsumer(createProgressConsumer());
    myVcs.syncRepository(hgRoot, settings);
    String toRevision = toRootRevision;
    if (toRevision == null) {
      RepositoryStateData state = myVcs.getCollectChangesPolicy().getCurrentState(toRoot);
      toRevision = state.getBranchRevisions().get(state.getDefaultBranchName());
    }
    String mergeBase = getMergeBase(hgRoot, fromRootRevision, toRevision);
    if (mergeBase == null)
      return Collections.emptyList();
    return collectChanges(toRoot, mergeBase, toRootRevision, checkoutRules);
  }


  public List<ModificationData> collectChanges(@NotNull VcsRoot root,
                                               @NotNull String fromVersion,
                                               @Nullable String currentVersion,
                                               @NotNull CheckoutRules checkoutRules) throws VcsException {
    if (currentVersion == null)
      return emptyList();
    OperationContext ctx = new OperationContext(myVcs, myRepoFactory, myHgPathProvider, fromVersion, currentVersion);
    ctx.setProgressConsumer(createProgressConsumer());
    List<ModificationData> changes = collectChanges(ctx, root, asList(fromVersion), currentVersion, checkoutRules);
    changes.addAll(getSubrepoChanges(ctx, root, changes));
    return changes;
  }


  @Nullable
  private String getMergeBase(@NotNull HgVcsRoot root, @NotNull String revision1, @NotNull String revision2) throws VcsException {
    String result = myVcs.createRepo(root).mergeBase()
            .revision1(revision1)
            .revision2(revision2)
            .call();
    if (result == null)
      result = getMinusNthCommit(root, 10);
    return result;
  }


  @Nullable
  private String getMinusNthCommit(@NotNull HgVcsRoot root, int n) throws VcsException {
    LogCommand log = myVcs.createRepo(root).log()
            .inBranch(root.getBranchName())
            .toNamedRevision(root.getBranchName());
    if (n > 0)
      log.setLimit(n);
    List<ChangeSet> changeSets = log.call();
    if (changeSets.isEmpty())
      return null;
    return changeSets.get(0).getId();
  }


  @NotNull
  private List<ModificationData> collectChanges(@NotNull OperationContext ctx,
                                                @NotNull VcsRoot root,
                                                @NotNull Collection<String> fromVersion,
                                                @Nullable String currentVersion,
                                                @NotNull CheckoutRules checkoutRules) throws VcsException {
    HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root);
    ctx.syncRepository(hgRoot, ctx.getProgressConsumer());
    List<ModificationData> result = new ArrayList<ModificationData>();
    List<ChangeSet> csets = getChangesets(ctx, hgRoot, fromVersion, currentVersion);
    //When commit has no changes in subrepo configuration we can reuse
    //subrepo revision table calculated for its parent commit(s). To do
    //that parents should be processed before children:
    Collections.sort(csets, ASCENDING_REV_NUMS);
    for (ChangeSet cset : csets) {
      result.add(createModificationData(ctx, cset, root, checkoutRules));
    }
    return result;
  }


  @NotNull
  private List<ChangeSet> getChangesets(@NotNull OperationContext ctx,
                                        @NotNull final HgVcsRoot root,
                                        @NotNull final Collection<String> fromVersions,
                                        @Nullable final String toVersion) throws VcsException {
    if (toVersion == null)
      return emptyList();
    List<String> fromCommits = new ArrayList<String>();
    for (String fromVersion : fromVersions) {
      fromCommits.add(new ChangeSetRevision(fromVersion).getId());
    }
    String toCommit = new ChangeSetRevision(toVersion).getId();
    try {
      ServerHgRepo repo = myVcs.createRepo(ctx, root);
      List<ChangeSet> csets = repo.collectChanges(root)
              .fromRevision(fromCommits)
              .toRevision(toCommit)
              .includeFromRevision(ctx.includeFromRevisions())
              .withProgressConsumer(ctx.getProgressConsumer())
              .call();
      if (!ctx.includeFromRevisions()) {
        Iterator<ChangeSet> iter = csets.iterator();
        while (iter.hasNext()) {
          ChangeSet cset = iter.next();
          if (fromVersions.contains(cset.getId()))
            iter.remove();
        }
      }
      return csets;
    } catch (UnknownRevisionException e) {
      Loggers.VCS.warn("Revision '" + e.getRevision() + "' is unknown, will return no changes");
      return emptyList();
    }
  }

  private ModificationData createModificationData(@NotNull final OperationContext ctx,
                                                  @NotNull final ChangeSet cset,
                                                  @NotNull final VcsRoot root,
                                                  @NotNull final CheckoutRules checkoutRules) throws VcsException {
    final ModificationData result = ModificationDataFactory.createModificationData(ctx, cset, root, checkoutRules);
    result.setAttributes(getAttributes(ctx, root, result));
    return result;
  }

  @NotNull
  private List<ModificationData> getSubrepoChanges(@NotNull OperationContext ctx, @NotNull VcsRoot root, @NotNull List<ModificationData> changes) throws VcsException {
    if (changes.isEmpty())
      return emptyList();

    ctx.setIncludeFromRevisions(true);

    HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root);
    if (!detectSubrepoChanges(hgRoot))
      return emptyList();

    List<HgSubrepoConfigChange> subrepoConfigChanges = getSubrepoConfigChanges(ctx, changes);
    List<ModificationData> subrepoChanges = new ArrayList<ModificationData>();

    for (HgSubrepoConfigChange configChange : subrepoConfigChanges) {
      SubRepo current = configChange.getCurrent();
      assert current != null;

      String subrepoUrl = current.url();
      String curRevision = current.revision();
      List<String> prevRevisions = new ArrayList<String>(0);
      for (SubRepo prevSubrepo : configChange.getPrevious()) {
        prevRevisions.add(prevSubrepo.revision());
      }
      String path = configChange.getPath();

      Map<String, String> subrepoParams = new HashMap<String, String>(hgRoot.getProperties());
      subrepoParams.put(Constants.REPOSITORY_PROP, subrepoUrl);
      subrepoParams.put("teamcity.internal.subrepo", "true");
      subrepoParams.put("teamcity.internal.subrepo.path", path);

      VcsRootImpl subrepo = new VcsRootImpl(root.getId(), subrepoParams);
      if (ctx.isProcessedSubrepoChanges(subrepo, prevRevisions, curRevision))
        continue;
      try {
        List<ModificationData> subChanges = collectChanges(ctx, subrepo, prevRevisions, curRevision, CheckoutRules.DEFAULT);
        for (ModificationData m : subChanges) {
          if (!ctx.isReportedModification(m)) {
            subrepoChanges.add(m);
            ctx.markAsReported(m);
          }
        }
        ctx.markProcessedSubrepoChanges(subrepo, prevRevisions, curRevision);
      } catch (VcsException e) {
        Loggers.VCS.warn("Error while collecting subrepo changes, repository: " + hgRoot.getRepository() +
                ", revision: " + configChange.getMainRepoRevision() +
                ", subrepo: " + subrepoUrl + " at " + configChange.getPath() +
                ", subrepo revisions interval: [" + prevRevisions + ", " + curRevision + "] "+
                "skip collecting subrepo changes in this interval", e);
      }
    }

    List<ModificationData> subSubrepoChanges = getSubrepoChanges(ctx, root, subrepoChanges);
    subrepoChanges.addAll(subSubrepoChanges);
    return subrepoChanges;
  }


  private boolean detectSubrepoChanges(@NotNull HgVcsRoot root) {
    return myConfig.detectSubrepoChanges() && root.detectSubrepoChanges();
  }


  private List<HgSubrepoConfigChange> getSubrepoConfigChanges(@NotNull OperationContext ctx, @NotNull List<ModificationData> mainRootChanges) throws VcsException {
    List<HgSubrepoConfigChange> subrepoConfigChanges = new ArrayList<HgSubrepoConfigChange>();
    for (ModificationData m : mainRootChanges) {
      subrepoConfigChanges.addAll(getSubrepoConfigChanges(ctx, m));
    }
    return subrepoConfigChanges;
  }


  @NotNull
  private List<HgSubrepoConfigChange> getSubrepoConfigChanges(@NotNull OperationContext ctx, @NotNull ModificationData m) throws VcsException {
    List<HgSubrepoConfigChange> configChanges = new ArrayList<HgSubrepoConfigChange>();

    HgVcsRoot mainRoot = myHgVcsRootFactory.createHgRoot(m.getVcsRoot());
    ServerHgRepo repo = myVcs.createRepo(ctx, mainRoot);
    for (HgSubrepoConfigChange c : repo.getSubrepoConfigChanges(m)) {
      if (!(c.subrepoUrlChanged() || c.subrepoAdded() || c.subrepoRemoved())) {//report only changes in revisions, because we collect changes only for such changes
        //map url and path, relative to the main repository
        try {
          SubRepo currentSubrepo = c.getCurrent();
          assert currentSubrepo != null;
          String subrepoUrl = ctx.getStringFromPool(currentSubrepo.resolveUrl(mainRoot.getRepository()));
          String curRevision = ctx.getStringFromPool(currentSubrepo.revision());
          List<SubRepo> prevSubrepos = new ArrayList<SubRepo>(0);
          String path = ctx.getStringFromPool(mainRoot.expandSubrepoPath(c.getPath()));
          for (SubRepo prevSubrepo : c.getPrevious()) {
            prevSubrepos.add(new SubRepo(path, subrepoUrl, ctx.getStringFromPool(prevSubrepo.revision())));
          }
          configChanges.add(new HgSubrepoConfigChange(m.getVersion(), path, prevSubrepos, new SubRepo(path, subrepoUrl, curRevision)));
        } catch (WrongSubrepoUrlException e) {
          Loggers.VCS.warn("Error while collecting subrepo config changes, repository: " + mainRoot.getRepository()
                  +", commit: " + m.getVersion() + ", skip subrepo changes in this commit", e);
        }
      }
    }
    return configChanges;
  }


  @NotNull
  private Map<String, String> getAttributes(@NotNull OperationContext ctx, @NotNull VcsRoot mainRoot, @NotNull ModificationData m) throws VcsException {
    Map<String, String> attributes = new HashMap<String, String>();
    HgVcsRoot root = myHgVcsRootFactory.createHgRoot(mainRoot);
    if (detectSubrepoChanges(root)) {
      try {
        ServerHgRepo repo = myVcs.createRepo(ctx, root);
        SubrepoRevisionAttributesBuilder attrBuilder = new SubrepoRevisionAttributesBuilder();
        for (SubRepo s : repo.getSubrepositories(m).values()) {
          attrBuilder.addSubrepo(new SubrepoConfig(mainRoot)
                  .setSubrepoPath(ctx.getStringFromPool(root.expandSubrepoPath(s.path())))
                  .setSubrepoRootParamDiff(Constants.REPOSITORY_PROP, ctx.getStringFromPool(s.resolveUrl(root.getRepository())))
                  .setSubrepoRootParamDiff("teamcity.internal.subrepo", "true")
                  .setSubrepoRootParamDiff("teamcity.internal.subrepo.path", ctx.getStringFromPool(root.expandSubrepoPath(s.path())))
                  .setSubrepoRevision(ctx.getStringFromPool(s.revision())));
        }
        attributes.putAll(attrBuilder.buildAttributes());
      } catch (Exception e) {
        Loggers.VCS.warn("Error while reporting subrepo config changes", e);
        if (e instanceof VcsException)
          throw (VcsException) e;
        throw new VcsException(e);
      }
    }
    if (attributes.isEmpty())
      return emptyMap();
    return attributes;
  }

  private final static class AscendingRevNums implements Comparator<ChangeSet> {
    public int compare(ChangeSet o1, ChangeSet o2) {
      int revnum1 = o1.getRevNumber();
      int revnum2 = o2.getRevNumber();
      if (revnum1 < revnum2)
        return -1;
      if (revnum1 > revnum2)
        return 1;
      return 0;
    }
  }


  @Nullable
  private ProgressParser.ProgressConsumer createProgressConsumer() {
    try {
      final VcsOperationProgress progress = myProgressProvider.getProgress();
      return new VcsOperationProgressConsumer(progress);
    } catch (IllegalStateException e) {
      return null;
    }
  }

  private static class VcsOperationProgressConsumer implements ProgressParser.ProgressConsumer {
    private final VcsOperationProgress myProgress;
    private String myPrevMessage;
    private int myPrevPercents;
    public VcsOperationProgressConsumer(@NotNull VcsOperationProgress progress) {
      myProgress = progress;
    }

    public void consume(float progressPercents, @NotNull String stage) {
      if (progressPercents < 0) {
        resetPrevProgress();
        myProgress.reportProgress(stage);
      } else {
        int percents = (int) Math.floor(progressPercents * 100);
        if (!isDuplicate(stage, percents)) {
          myProgress.reportProgress(stage + " " + percents + "%");
          updatePrevProgress(stage, percents);
        }
      }
    }

    private void resetPrevProgress() {
      myPrevMessage = null;
      myPrevPercents = -1;
    }

    private boolean isDuplicate(@NotNull String message, int percents) {
      return message.equals(myPrevMessage) && percents == myPrevPercents;
    }

    private void updatePrevProgress(@NotNull String message, int percents) {
      myPrevMessage = message;
      myPrevPercents = percents;
    }
  }
}