changeset 799:b130c7d27c02

Report clone/pull progress
author Dmitry Neverov <dmitry.neverov@jetbrains.com>
date Sat, 24 May 2014 16:45:14 +0200
parents ea26bc72db7f
children ea4a75bf2324
files mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CloneCommand.java mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommandSettings.java mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommandUtil.java mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/ProgressParser.java mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/PullCommand.java mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCollectChangesPolicy.java mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCommitsInfoBuilderSupport.java mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialFetchService.java mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupport.java mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/SyncSettings.java
diffstat 10 files changed, 187 insertions(+), 21 deletions(-) [+]
line wrap: on
line diff
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CloneCommand.java	Sat May 24 14:29:00 2014 +0200
+++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CloneCommand.java	Sat May 24 16:45:14 2014 +0200
@@ -29,6 +29,7 @@
   private boolean myUsePullProtocol = true;
   private boolean myUseUncompressedTransfer = false;
   private boolean myTraceback;
+  private ProgressParser.ProgressConsumer myProgressConsumer;
 
   public CloneCommand(@NotNull CommandSettings commandSettings,
                       @NotNull String hgPath,
@@ -76,6 +77,11 @@
     return this;
   }
 
+  public CloneCommand withProgressConsumer(ProgressParser.ProgressConsumer progressConsumer) {
+    myProgressConsumer = progressConsumer;
+    return this;
+  }
+
   public void call() throws VcsException {
     myWorkingDir.mkdirs();
     MercurialCommandLine cli = createCommandLine();
@@ -97,15 +103,19 @@
       cli.addParameter("--uncompressed");
     }
 
-    cli.addParameters("--config", "extensions.progress=");
-    cli.addParameters("--config", "progress.format=topic number");
-    cli.addParameters("--config", "progress.delay=0");
-    cli.addParameters("--config", "progress.assume-tty=True");
+    CommandSettings settings = myCommandSettings.setTimeout(24 * 3600); // some repositories are quite large, we set timeout to 24 hours
+    if (myProgressConsumer != null) {
+      cli.addParameters("--config", "extensions.progress=");
+      cli.addParameters("--config", "progress.format=topic number");
+      cli.addParameters("--config", "progress.delay=0");
+      cli.addParameters("--config", "progress.assume-tty=True");
+      settings.setProgressConsumer(myProgressConsumer);
+    }
 
     String repositoryUrl = myAuthSettings.getRepositoryUrlWithCredentials(myRepository);
     cli.addParameter(repositoryUrl);
     cli.addParameter(myWorkingDir.getName());
 
-    runCommand(cli, myCommandSettings.setTimeout(24 * 3600)); // some repositories are quite large, we set timeout to 24 hours
+    runCommand(cli, settings);
   }
 }
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommandSettings.java	Sat May 24 14:29:00 2014 +0200
+++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommandSettings.java	Sat May 24 16:45:14 2014 +0200
@@ -34,6 +34,7 @@
   private int myLogOutputLimit = -1;
   private int myExceptionOutputLimit = 5000;
   private List<String> myGlobalArguments = new ArrayList<String>(0);
+  private ProgressParser.ProgressConsumer myProgressConsumer;
 
   public CommandSettings setTimeout(int timeout) {
     myTimeout = timeout;
@@ -117,4 +118,12 @@
   public void setExceptionOutputLimit(int limit) {
     myExceptionOutputLimit = limit;
   }
+
+  public ProgressParser.ProgressConsumer getProgressConsumer() {
+    return myProgressConsumer;
+  }
+
+  public void setProgressConsumer(ProgressParser.ProgressConsumer progressConsumer) {
+    myProgressConsumer = progressConsumer;
+  }
 }
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommandUtil.java	Sat May 24 14:29:00 2014 +0200
+++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommandUtil.java	Sat May 24 16:45:14 2014 +0200
@@ -16,13 +16,16 @@
 package jetbrains.buildServer.buildTriggers.vcs.mercurial.command;
 
 import jetbrains.buildServer.ExecResult;
+import jetbrains.buildServer.LineAwareByteArrayOutputStream;
 import jetbrains.buildServer.SimpleCommandLineProcessRunner;
 import jetbrains.buildServer.log.Loggers;
 import jetbrains.buildServer.util.StringUtil;
 import jetbrains.buildServer.vcs.VcsException;
 import org.jetbrains.annotations.NotNull;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.nio.charset.Charset;
 import java.util.Set;
 
 public class CommandUtil {
@@ -43,6 +46,15 @@
                                    @NotNull final Set<String> privateData,
                                    @NotNull CommandSettings settings) {
     final long start = System.currentTimeMillis();
+    ByteArrayOutputStream stdoutBuffer = new ByteArrayOutputStream();
+    ProgressParser.ProgressConsumer progressConsumer = settings.getProgressConsumer();
+    ByteArrayOutputStream stderrBuffer;
+    if (progressConsumer != null) {
+      stderrBuffer =  new LineAwareByteArrayOutputStream(Charset.forName("UTF-8"), new ProgressParser(progressConsumer));
+      ((LineAwareByteArrayOutputStream) stderrBuffer).setCREndsLine(true);
+    } else {
+      stderrBuffer = new ByteArrayOutputStream();
+    }
     ExecResult res = SimpleCommandLineProcessRunner.runCommandSecure(cli, command, null, new SimpleCommandLineProcessRunner.ProcessRunCallbackAdapter() {
       @Override
       public Integer getOutputIdleSecondsTimeout() {
@@ -53,7 +65,7 @@
         long duration = System.currentTimeMillis() - start;
         Loggers.VCS.debug("Command " + command + " took " + duration + "ms");
       }
-    });
+    }, stdoutBuffer, stderrBuffer);
     return new CommandResult(Loggers.VCS, command, res, privateData, settings);
   }
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/ProgressParser.java	Sat May 24 16:45:14 2014 +0200
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2000-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jetbrains.buildServer.buildTriggers.vcs.mercurial.command;
+
+import jetbrains.buildServer.LineAwareByteArrayOutputStream;
+import org.jetbrains.annotations.NotNull;
+
+public class ProgressParser implements LineAwareByteArrayOutputStream.LineListener {
+
+  private final ProgressConsumer myConsumer;
+
+  public ProgressParser(@NotNull ProgressConsumer consumer) {
+    myConsumer = consumer;
+  }
+
+  public interface ProgressConsumer {
+    void consume(float progress, @NotNull String stage);
+  }
+
+  public void newLineDetected(@NotNull String line) {
+    String trimmed = line.trim();
+    if (trimmed.isEmpty())
+      return;
+
+    int spaceIdx = trimmed.indexOf(' ');
+    if (spaceIdx == -1) {
+      myConsumer.consume(-1, trimmed);
+      return;
+    }
+
+    String stage = trimmed.substring(0, spaceIdx);
+    String progress = trimmed.substring(spaceIdx).trim();
+    int ratioIdx = progress.indexOf('/');
+    if (ratioIdx == -1 || ratioIdx == trimmed.length() - 1) {
+      myConsumer.consume(-1, stage);
+      return;
+    }
+
+    try {
+      int nom = Integer.parseInt(progress.substring(0, ratioIdx));
+      int denom = Integer.parseInt(progress.substring(ratioIdx+1));
+      myConsumer.consume(nom * 1.0f / denom, stage);
+    } catch (NumberFormatException e) {
+      myConsumer.consume(-1, stage);
+    }
+  }
+}
--- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/PullCommand.java	Sat May 24 14:29:00 2014 +0200
+++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/PullCommand.java	Sat May 24 16:45:14 2014 +0200
@@ -32,6 +32,7 @@
   private String myPullUrl;
   private int myTimeout;
   private boolean myTraceback;
+  private ProgressParser.ProgressConsumer myProgressConsumer;
 
   public PullCommand(@NotNull CommandSettings commandSettings,
                      @NotNull String hgPath,
@@ -60,6 +61,11 @@
     return this;
   }
 
+  public PullCommand withProgressConsumer(ProgressParser.ProgressConsumer progressConsumer) {
+    myProgressConsumer = progressConsumer;
+    return this;
+  }
+
   public void call() throws VcsException {
     ensureRepositoryIsNotLocked();
     MercurialCommandLine cli = createCommandLine();
@@ -67,14 +73,18 @@
     if (myTraceback)
       cli.addParameter("--traceback");
 
-    cli.addParameters("--config", "extensions.progress=");
-    cli.addParameters("--config", "progress.format=topic number");
-    cli.addParameters("--config", "progress.delay=0");
-    cli.addParameters("--config", "progress.assume-tty=True");
+    CommandSettings settings = myCommandSettings.setTimeout(myTimeout);
+    if (myProgressConsumer != null) {
+      cli.addParameters("--config", "extensions.progress=");
+      cli.addParameters("--config", "progress.format=topic number");
+      cli.addParameters("--config", "progress.delay=0");
+      cli.addParameters("--config", "progress.assume-tty=True");
+      settings.setProgressConsumer(myProgressConsumer);
+    }
 
-    String pullUrl = myAuthSettings != null ? myAuthSettings.getRepositoryUrlWithCredentials(myPullUrl) : myPullUrl;
+    String pullUrl = myAuthSettings.getRepositoryUrlWithCredentials(myPullUrl);
     cli.addParameter(pullUrl);
-    runCommand(cli, myCommandSettings.setTimeout(myTimeout));
+    runCommand(cli, settings);
   }
 
   private void ensureRepositoryIsNotLocked() {
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCollectChangesPolicy.java	Sat May 24 14:29:00 2014 +0200
+++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCollectChangesPolicy.java	Sat May 24 16:45:14 2014 +0200
@@ -61,11 +61,12 @@
   @NotNull
   public RepositoryStateData getCurrentState(@NotNull VcsRoot root) throws VcsException {
     final HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root);
-    Map<String, String> revisions = myVcs.syncRepository(hgRoot, new VcsCallable<Map<String, String>>() {
+    VcsCallable<Map<String, String>> cmd = new VcsCallable<Map<String, String>>() {
       public Map<String, String> call() throws VcsException {
         return getHeads(hgRoot);
       }
-    });
+    };
+    Map<String, String> revisions = myVcs.syncRepository(hgRoot, new SyncSettings<Map<String, String>>(cmd));
     String defaultBranchName = hgRoot.getBranchName();
     if (revisions.get(defaultBranchName) == null) {
       throw new VcsException("Cannot find revision of the default branch '" +
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCommitsInfoBuilderSupport.java	Sat May 24 14:29:00 2014 +0200
+++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCommitsInfoBuilderSupport.java	Sat May 24 16:45:14 2014 +0200
@@ -59,11 +59,12 @@
 
     final HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root);
     final ServerHgRepo repo = mySupport.createRepo(hgRoot);
-    final MultiMapToList<String, String> heads = mySupport.syncRepository(hgRoot, new VcsCallable<MultiMapToList<String, String>>() {
+    VcsCallable<MultiMapToList<String, String>> cmd = new VcsCallable<MultiMapToList<String, String>>() {
       public MultiMapToList<String, String> call() throws VcsException {
         return commitToBranchs(mySupport.getCollectChangesPolicy().getHeads(hgRoot));
       }
-    });
+    };
+    final MultiMapToList<String, String> heads = mySupport.syncRepository(hgRoot, new SyncSettings<MultiMapToList<String, String>>(cmd));
     repo.logSubstates().call(new CommitsAndMountPointsCommand.Callback() {
       private final MercurialCommitsInfoBuilderStates subs = new MercurialCommitsInfoBuilderStates();
 
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialFetchService.java	Sat May 24 14:29:00 2014 +0200
+++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialFetchService.java	Sat May 24 16:45:14 2014 +0200
@@ -16,6 +16,7 @@
 
 package jetbrains.buildServer.buildTriggers.vcs.mercurial;
 
+import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.ProgressParser;
 import jetbrains.buildServer.vcs.CheckoutRules;
 import jetbrains.buildServer.vcs.FetchService;
 import jetbrains.buildServer.vcs.VcsException;
@@ -37,6 +38,19 @@
   public void fetchRepository(@NotNull VcsRoot root,
                               @NotNull CheckoutRules rules,
                               @NotNull FetchRepositoryCallback callback) throws VcsException {
+    SyncSettings<Void> settings = new SyncSettings<Void>(VcsCallable.NO_OP);
+    settings.setProgressConsumer(new FetchProgressConsumer(callback));
     myVcs.syncRepository(myHgVcsRootFactory.createHgRoot(root));
   }
+
+  private class FetchProgressConsumer implements ProgressParser.ProgressConsumer {
+    private final FetchRepositoryCallback myCallback;
+    private FetchProgressConsumer(@NotNull FetchRepositoryCallback callback) {
+      myCallback = callback;
+    }
+
+    public void consume(float progress, @NotNull String stage) {
+      myCallback.update(progress, stage);
+    }
+  }
 }
--- a/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupport.java	Sat May 24 14:29:00 2014 +0200
+++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialVcsSupport.java	Sat May 24 16:45:14 2014 +0200
@@ -542,17 +542,17 @@
   }
 
   public void syncRepository(@NotNull final HgVcsRoot root) throws VcsException {
-    syncRepository(root, VcsCallable.NO_OP);
+    syncRepository(root, new SyncSettings<Void>(VcsCallable.NO_OP));
   }
 
-  public <T> T syncRepository(@NotNull HgVcsRoot root, @NotNull VcsCallable<T> cmd) throws VcsException {
+  public <T> T syncRepository(@NotNull HgVcsRoot root, @NotNull SyncSettings<T> settings) throws VcsException {
     boolean customWorkingDir = root.getCustomWorkingDir() != null;
     File workingDir = getWorkingDir(root);
     int attemptsLeft = 3;
     VcsException lastError = null;
     while (attemptsLeft-- > 0) {
       try {
-        return syncRepositoryOnce(root, cmd, workingDir);
+        return syncRepositoryOnce(root, settings, workingDir);
       } catch (UnrelatedRepositoryException e) {
         if (customWorkingDir)
           throw new VcsException(e.getMessage() + ". VCS root uses a custom clone dir, manual recovery is required.", e);
@@ -573,7 +573,7 @@
   }
 
 
-  private <T> T syncRepositoryOnce(@NotNull HgVcsRoot root, @NotNull VcsCallable<T> cmd, @NotNull File workingDir) throws VcsException {
+  private <T> T syncRepositoryOnce(@NotNull HgVcsRoot root, @NotNull SyncSettings<T> settings, @NotNull File workingDir) throws VcsException {
     lockWorkDir(workingDir);
     HgRepo repo = createRepo(root);
     try {
@@ -581,15 +581,17 @@
           resetBookmarks(repo);
           repo.pull().fromRepository(root.getRepository())
                   .withTimeout(myConfig.getPullTimeout())
+                  .withProgressConsumer(settings.getProgressConsumer())
                   .call();
       } else {
         repo.doClone().fromRepository(root.getRepository())
                 .setUpdateWorkingDir(false)
                 .useUncompressedTransfer(root.isUncompressedTransfer())
+                .withProgressConsumer(settings.getProgressConsumer())
                 .call();
         repo.setDefaultPath(root.getRepository());
       }
-      return cmd.call();
+      return settings.getCmd().call();
     } finally {
       unlockWorkDir(workingDir);
     }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/SyncSettings.java	Sat May 24 16:45:14 2014 +0200
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2000-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package jetbrains.buildServer.buildTriggers.vcs.mercurial;
+
+import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.ProgressParser;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+public class SyncSettings<T> {
+  private VcsCallable<T> myCmd;
+  private ProgressParser.ProgressConsumer myProgressConsumer;
+
+  public SyncSettings(@NotNull VcsCallable<T> cmd) {
+    myCmd = cmd;
+  }
+
+  @NotNull
+  public VcsCallable<T> getCmd() {
+    return myCmd;
+  }
+
+  @Nullable
+  public ProgressParser.ProgressConsumer getProgressConsumer() {
+    return myProgressConsumer;
+  }
+
+  @NotNull
+  public SyncSettings<T> setProgressConsumer(ProgressParser.ProgressConsumer progressConsumer) {
+    myProgressConsumer = progressConsumer;
+    return this;
+  }
+}