view mercurial-agent/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialIncludeRuleUpdater.java @ 886:bf83331f51f0

Report agent-side checkout progress to build log This also fixes 'hg pull' timeouts problem. In order to support custom configs we started to use 'hg init', 'hg pull' sequence instead of 'hg clone'. Need that in order to modify repository configs after 'hg init', 'hg clone' doesn't give such an opportunity. The default pull timeout wasn't enough to pull a big repository from scratch. With progress reporting the default pull timeout (1 hour) should be enough, because it is an idle timeout.
author Dmitry Neverov <dmitry.neverov@jetbrains.com>
date Wed, 15 Oct 2014 06:29:39 +0200
parents 6df89e185a3c
children d1a5fc6c1d6e
line wrap: on
line source
/*
 * Copyright 2000-2014 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package jetbrains.buildServer.buildTriggers.vcs.mercurial;

import jetbrains.buildServer.agent.AgentRunningBuild;
import jetbrains.buildServer.agent.BuildProgressLogger;
import jetbrains.buildServer.agent.vcs.IncludeRuleUpdater;
import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.AuthSettings;
import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.HgVcsRoot;
import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.exception.UnrelatedRepositoryException;
import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.exception.WrongSubrepoUrlException;
import jetbrains.buildServer.buildTriggers.vcs.mercurial.ext.BeforeWorkingDirUpdateExtension;
import jetbrains.buildServer.buildTriggers.vcs.mercurial.ext.MercurialExtension;
import jetbrains.buildServer.log.Loggers;
import jetbrains.buildServer.util.FileUtil;
import jetbrains.buildServer.vcs.IncludeRule;
import jetbrains.buildServer.vcs.VcsException;
import jetbrains.buildServer.vcs.VcsRoot;
import org.jetbrains.annotations.NotNull;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import static com.intellij.openapi.util.io.FileUtil.delete;

/**
 * @author dmitry.neverov
 */
public class MercurialIncludeRuleUpdater implements IncludeRuleUpdater {

  protected final MirrorManager myMirrorManager;
  protected final AgentRepoFactory myRepoFactory;
  protected final HgVcsRoot myRoot;
  private final AuthSettings myAuthSettings;
  private final String myToVersion;
  private final BuildProgressLogger myLogger;
  private final boolean myUseLocalMirrors;
  private int myPullTimeout;
  private final boolean myUseTraceback;
  private final boolean myProfile;
  private final List<MercurialExtension> myExtensions = new ArrayList<MercurialExtension>();
  protected final MercurialProgress myProgress;

  public MercurialIncludeRuleUpdater(@NotNull AgentPluginConfig pluginConfig,
                                     @NotNull MirrorManager mirrorManager,
                                     @NotNull AgentRepoFactory repoFactory,
                                     @NotNull VcsRoot root,
                                     @NotNull String toVersion,
                                     @NotNull AgentRunningBuild build) {
    myMirrorManager = mirrorManager;
    myRepoFactory = repoFactory;
    myRoot = new HgVcsRoot(root);
    myAuthSettings = myRoot.getAuthSettings();
    myToVersion = toVersion;
    myLogger = build.getBuildLogger();
    myUseLocalMirrors = pluginConfig.isUseLocalMirrors(build);
    myPullTimeout = pluginConfig.getPullTimeout(build);
    myUseTraceback = pluginConfig.runWithTraceback(build);
    myProfile = pluginConfig.runWithProfile(build);
    myProgress = new MercurialBuildLogProgress(build.getBuildLogger());
  }


  public void process(@NotNull IncludeRule rule, @NotNull File workingDir) throws VcsException {
    try {
      checkRuleIsValid(rule);
      if (myUseLocalMirrors)
        updateLocalMirror(myRoot.getRepository(), myToVersion);
      updateRepository(workingDir);
      updateWorkingDir(workingDir, myToVersion, myRoot.getRepository());
    } catch (Exception e) {
      throwVcsException(e);
    }
  }


  public void dispose() throws VcsException {
  }


  public void registerExtension(@NotNull MercurialExtension extention) {
    myExtensions.add(extention);
  }


  @NotNull
  protected <T extends MercurialExtension> List<T> getExtensions(@NotNull Class<T> extensionClass) {
    List<T> extentions = new ArrayList<T>();
    for (MercurialExtension e : myExtensions) {
      if (extensionClass.isInstance(e))
        extentions.add(extensionClass.cast(e));
    }
    return extentions;
  }


  protected void updateLocalMirror(@NotNull String repositoryUrl, @NotNull String revision) throws VcsException, IOException {
    File mirrorDir = myMirrorManager.getMirrorDir(repositoryUrl);
    HgRepo mirrorRepo = myRepoFactory.createRepo(myRoot, mirrorDir, myProgress);
    if (!mirrorRepo.isValidRepository()) {
      delete(mirrorDir);
      mirrorRepo.init().call();
    }
    mirrorRepo.setDefaultPath(myRoot.getRepository());
    mirrorRepo.setTeamCityConfig(myRoot.getCustomHgConfig());
    myLogger.message("Update local mirror of " + myAuthSettings.getRepositoryUrlWithHiddenPassword(repositoryUrl) + " at " + mirrorDir);
    if (mirrorRepo.containsRevision(revision)) {
      myLogger.message("Local mirror is already up-to-date");
    } else {
      myLogger.message("Start pulling changes from " + myAuthSettings.getRepositoryUrlWithHiddenPassword(repositoryUrl));
      mirrorRepo.pull().fromRepository(repositoryUrl)
              .withTraceback(myUseTraceback)
              .withProfile(myProfile)
              .withTimeout(myPullTimeout)
              .call();
      myLogger.message("Local mirror changes successfully pulled");
    }
  }


  protected void updateRepository(@NotNull File workingDir) throws VcsException, IOException {
    String repositoryUrl = getDefaultPullUrl(myRoot, myUseLocalMirrors);
    HgRepo repo = myRepoFactory.createRepo(myRoot, workingDir, myProgress);
    myLogger.message("Update repository " + workingDir.getAbsolutePath());
    disableSharing(workingDir);
    if (!repo.isValidRepository())
      repo.init().call();
    repo.setDefaultPath(myRoot.getRepository());
    repo.setTeamCityConfig(myRoot.getCustomHgConfig());
    if (repo.containsRevision(myToVersion)) {
      myLogger.message("Repository already contains revision " + myToVersion);
    } else {
      myLogger.message("Start pulling changes from " + (myUseLocalMirrors ? "local mirror " : "") + myAuthSettings.getRepositoryUrlWithHiddenPassword(repositoryUrl));
      try {
        repo.pull().fromRepository(repositoryUrl)
                .withTraceback(myUseTraceback)
                .withProfile(myProfile)
                .withTimeout(myPullTimeout)
                .call();
      } catch (UnrelatedRepositoryException e) {
        throw new UnrelatedRepositoryException(myAuthSettings.getRepositoryUrlWithHiddenPassword(repositoryUrl), workingDir);
      }
      myLogger.message("Changes successfully pulled");
    }
  }


  private void disableSharing(@NotNull File workingDir) {
    File dotHg = new File(workingDir, ".hg");
    File sharedpath = new File(dotHg, "sharedpath");
    if (sharedpath.exists())
      FileUtil.delete(sharedpath);
  }


  private void updateWorkingDir(@NotNull File workingDir, @NotNull String toVersion, @NotNull String repositoryUrl) throws VcsException, IOException {
    HgRepo repo = myRepoFactory.createRepo(myRoot, workingDir, myProgress);
    List<File> repos = new ArrayList<File>();
    updateSubrepositories(repo, toVersion, repositoryUrl, repos);
    doUpdateWorkingDir(repo, toVersion);
    purge(repos);
  }

  private void purge(@NotNull List<File> dirs) throws VcsException {
    HgVcsRoot.PurgePolicy purgePolicy = myRoot.getPurgePolicy();
    if (purgePolicy == HgVcsRoot.PurgePolicy.DONT_RUN)
      return;
    for (File dir : dirs) {
      myLogger.message("Run purge in " + dir.getAbsolutePath());
      HgRepo repo = myRepoFactory.createRepo(myRoot, dir, myProgress);
      repo.purge().withPolicy(purgePolicy).call();
      myLogger.message("Purge in " + dir.getAbsolutePath() + " is finished");
    }
  }

  private void updateSubrepositories(@NotNull HgRepo repo,
                                     @NotNull String toVersion,
                                     @NotNull String parentRepositoryUrl,
                                     @NotNull List<File> repoAccumulator) throws VcsException, IOException {
    repoAccumulator.add(repo.getWorkingDir());
    if (!repo.hasSubreposAtRevision(toVersion))
      return;
    myLogger.message("Process subrepos of " + parentRepositoryUrl);
    String workingDirRevision = repo.getWorkingDirRevision();
    Map<String, SubRepo> workingDirSubrepos = repo.getSubrepositories(workingDirRevision);
    Map<String, SubRepo> subrepos = repo.getSubrepositories(toVersion);
    for (Map.Entry<String, SubRepo> entry : subrepos.entrySet()) {
      String path = entry.getKey();
      SubRepo subrepoConfig = entry.getValue();
      myLogger.message("Process subrepoConfig at path " + path + " (url: " + subrepoConfig.url() + ")");
      SubRepo workingDirSubrepo = workingDirSubrepos.get(path);
      if (workingDirSubrepo != null && subrepoConfig.hasDifferentUrlThan(workingDirSubrepo)) {
        myLogger.message("The url of subrepoConfig was changed between revisions " + workingDirRevision + " and " + toVersion + " , delete the subrepoConfig");
        delete(subrepoConfigDir(repo, subrepoConfig));
      }
      HgRepo subrepository = myRepoFactory.createRepo(myRoot, subrepoConfigDir(repo, subrepoConfig), myProgress);
      String subrepoUrl;
      try {
        subrepoUrl = subrepoConfig.resolveUrl(parentRepositoryUrl);
        if (myUseLocalMirrors && subrepoConfig.vcsType() == SubRepo.VcsType.hg && !isRelativeUrl(subrepoUrl))
          syncSubrepo(subrepository, subrepoUrl, subrepoConfig.revision());
      } catch (WrongSubrepoUrlException e) {
        myLogger.warning("Failed to resolve subrepo url '" + subrepoConfig.url() + "': " + e.getMessage());
        Loggers.VCS.warn("Failed to resolve subrepo url '" + subrepoConfig.url() + "'", e);
        subrepoUrl = subrepoConfig.url();
      }
      updateSubrepositories(subrepository, subrepoConfig.revision(), subrepoUrl, repoAccumulator);
    }
  }

  private boolean isRelativeUrl(@NotNull String url) {
    return url.startsWith(".");
  }

  protected void syncSubrepo(@NotNull HgRepo subrepository, @NotNull String subrepoUrl, @NotNull String subrepoRevision) throws VcsException, IOException {
    disableSharing(subrepository.getWorkingDir());
    if (!subrepository.isValidRepository() || !subrepository.containsRevision(subrepoRevision)) {
      updateLocalMirror(subrepoUrl, subrepoRevision);
      File mirrorDir = myMirrorManager.getMirrorDir(subrepoUrl);
      if (!subrepository.isValidRepository())
        subrepository.init().call();
      subrepository.setDefaultPath(subrepoUrl);
      subrepository.setTeamCityConfig(myRoot.getCustomHgConfig());
      myLogger.message("Pull from local mirror");
      subrepository.pull().fromRepository(mirrorDir)
              .withTraceback(myUseTraceback)
              .withProfile(myProfile)
              .withTimeout(myPullTimeout)
              .call();
      myLogger.message("done");
    }
  }


  private void doUpdateWorkingDir(@NotNull HgRepo repo, @NotNull String revision) throws VcsException {
    for (BeforeWorkingDirUpdateExtension e : getExtensions(BeforeWorkingDirUpdateExtension.class)) {
      e.call(repo, revision);
    }
    myLogger.message("Updating working dir " + repo.path() + " to revision " + revision);
    repo.update().withTraceback(myUseTraceback).toRevision(revision).call();
    myLogger.message("Working dir updated");
  }


  protected String getDefaultPullUrl(HgVcsRoot root, boolean useLocalMirror) throws IOException {
    if (useLocalMirror) {
      File mirrorDir = myMirrorManager.getMirrorDir(root.getRepository());
      return mirrorDir.getCanonicalPath();
    } else {
      return root.getRepository();
    }
  }


  private void checkRuleIsValid(IncludeRule includeRule) throws VcsException {
    if (includeRule.getTo() != null && includeRule.getTo().length() > 0) {
      if (!".".equals(includeRule.getFrom()) && includeRule.getFrom().length() != 0)
        throw new VcsException("Invalid include rule: " + includeRule.toString() + ", Mercurial plugin supports mapping of the form: +:.=>dir only.");
    }
  }


  private void throwVcsException(Exception e) throws VcsException {
    if (e instanceof VcsException)
      throw (VcsException) e;
    else
      throw new VcsException(e);
  }

  private File subrepoConfigDir(@NotNull HgRepo parentRepo, @NotNull SubRepo subrepo) {
    return new File(parentRepo.getWorkingDir(), subrepo.path());
  }
}