# HG changeset patch # User eugene.petrenko@jetbrains.com # Date 1389638531 -3600 # Node ID 0f3c5ac38fc35f7d340249f44a1821abb05c40ae # Parent c37903906fad0600d8858ab3b33e93b83652e079 commits and mount points command completed diff -r c37903906fad -r 0f3c5ac38fc3 mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommitsAndMountPointsCommand.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommitsAndMountPointsCommand.java Mon Jan 13 19:42:11 2014 +0100 @@ -0,0 +1,130 @@ +package jetbrains.buildServer.buildTriggers.vcs.mercurial.command; + +import com.intellij.openapi.diagnostic.Logger; +import jetbrains.buildServer.buildTriggers.vcs.mercurial.HgFileUtil; +import jetbrains.buildServer.util.FileUtil; +import jetbrains.buildServer.vcs.VcsException; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.io.IOException; + +import static jetbrains.buildServer.buildTriggers.vcs.mercurial.command.CommitsAndMountPointsParser.parseCommits; +import static jetbrains.buildServer.buildTriggers.vcs.mercurial.command.CommitsAndMountPointsParser.parseFileLog; + +/** + * Created 03.01.14 14:53 + * + * @author Eugene Petrenko (eugene.petrenko@jetbrains.com) + */ +public class CommitsAndMountPointsCommand extends VcsRootCommand { + private static final Logger LOG = Logger.getInstance(CommitsAndMountPointsCommand.class.getName()); + + public CommitsAndMountPointsCommand(@NotNull final CommandSettings commandSettings, + @NotNull final String hgPath, + @NotNull final File workingDir, + @NotNull final AuthSettings authSettings) { + super(commandSettings.addHgEnv("HGENCODING", "UTF-8"), hgPath, workingDir, authSettings); + } + + @NotNull + private File createTmpDir() throws VcsException { + try { + return HgFileUtil.createTempDir(); + } catch (IOException e) { + throw new VcsException("Unable to create temporary directory", e); + } + } + + @NotNull + private File extractCommandPy(@NotNull final File root) throws VcsException { + try { + final File py = new File(root, "load-substates-command.py"); + + FileUtil.copyResource(getClass(), "/python/load-substates-command.py", py); + + if (py.length() < 100) throw new IOException("Failed to unpack command resource"); + return py; + } catch (IOException e) { + throw new VcsException("Failed to extract .py file: " + e.getMessage(), e); + } + } + + public void call(@NotNull final Callback consumer) throws VcsException { + final File root = createTmpDir(); + + try { + final File py = extractCommandPy(root); + + callImpl(root, py, consumer); + } finally { + FileUtil.delete(root); + } + } + + public interface Callback { + void processHGSubFile(@NotNull final String fileId, @NotNull final String file); + void processHGSubStateFile(@NotNull final String fileId, @NotNull final String file); + void onCommit( + @NotNull String commitNum, + @NotNull String commitId, + @NotNull String[] parents, + @NotNull String branch, + @NotNull String[] tags, + @NotNull String author, + @NotNull String message, + final long timestamp, + @Nullable String hgsubNodeId, + @Nullable String hgsubstateNodeId); + } + + private void callImpl(@NotNull final File root, + @NotNull final File commandPy, + @NotNull final Callback consumer) throws VcsException { + final MercurialCommandLine cli = createCommandLine(); + cli.addParameter("--debug"); + cli.addParameter("--config"); + cli.addParameter("extensions.logextcj=" + commandPy); + cli.addParameter("load-substates"); + cli.addParameter(new File(root, "result").getPath()); + + final CommandResult res = runCommand(cli); + final String output = res.getStdout(); + + if (!output.contains("##Completed##")) throw new VcsException("Command failed: " + output); + + try { + parseFileLog(new File(root, "result.hgsub"), new CommitsAndMountPointsParser.ContentsConsumer() { + public void onCommit(@NotNull final String fileNodeId, @NotNull final String content) { + consumer.processHGSubFile(fileNodeId, content); + } + }); + parseFileLog(new File(root, "result.hgsubstate"), new CommitsAndMountPointsParser.ContentsConsumer() { + public void onCommit(@NotNull final String fileNodeId, @NotNull final String content) { + consumer.processHGSubStateFile(fileNodeId, content); + } + }); + + parseCommits(new File(root, "result.commits"), new CommitsAndMountPointsParser.CommitsConsumer() { + public void onCommit(@NotNull String commitNum, + @NotNull String commitId, + @NotNull String[] parents, + @NotNull String branch, + @NotNull String[] tags, + @NotNull String author, + @NotNull String message, + long timestamp, + @Nullable String hgsubNodeId, + @Nullable String hgsubstateNodeId) { + consumer.onCommit(commitNum, commitId, parents, branch, tags, author, message, timestamp, hgsubNodeId, hgsubstateNodeId); + } + }); + } catch (IOException e) { + throw new VcsException("Failed to parse response files for 'load-substates' command. " + e.getMessage(), e); + } + } + + + +} diff -r c37903906fad -r 0f3c5ac38fc3 mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommitsAndMountPointsParser.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommitsAndMountPointsParser.java Mon Jan 13 19:42:11 2014 +0100 @@ -0,0 +1,151 @@ +package jetbrains.buildServer.buildTriggers.vcs.mercurial.command; + +import org.apache.commons.codec.binary.Base64; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.*; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; + +/** + * Created 09.01.14 17:10 + * + * @author Eugene Petrenko (eugene.petrenko@jetbrains.com) + */ +public class CommitsAndMountPointsParser { + + public static void parseFileLog(@NotNull final File dump, + @NotNull final ContentsConsumer consumer) throws IOException { + final InputStream is = new BufferedInputStream(new FileInputStream(dump)); + + final Decoder fileDecoder = new Decoder(5); + final BufferedReader st = new BufferedReader(new InputStreamReader(is, "utf-8")); + String line; + while((line = st.readLine()) != null) { + if (!line.startsWith("$$@@@@ ")) continue; + final String[] items = line.split(" "); + if (items.length != 1 + 2) continue; + + final String commitId = items[1]; + final String content = fileDecoder.decode(items[2]); + + consumer.onCommit(commitId, content == null ? "" : content); + } + } + + public interface ContentsConsumer { + void onCommit( + @NotNull String fileNodeId, + @NotNull String content); + } + + public interface CommitsConsumer { + void onCommit( + @NotNull String commitNum, + @NotNull String commitId, + @NotNull String[] parents, + @NotNull String branch, + @NotNull String[] tags, + @NotNull String author, + @NotNull String message, + final long timestamp, + @Nullable String hgsubNodeId, + @Nullable String hgsubstateNodeId); + } + + + public static void parseCommits(@NotNull final File dump, @NotNull final CommitsConsumer consumer) throws IOException { + final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'Z'HH:mm:ss'T'Z", Locale.ENGLISH); + + final InputStream is = new BufferedInputStream(new FileInputStream(dump)); + + final Decoder branchDecoder = new Decoder(250); + final Decoder tagsDecoder = new Decoder(250); + final Decoder authorDecoder = new Decoder(200); + final Decoder messageDecoder = new Decoder(210); + + final BufferedReader st = new BufferedReader(new InputStreamReader(is, "utf-8")); + String line; + while((line = st.readLine()) != null) { + if (!line.startsWith("$$@@@@ ")) continue; + final Iterator items = Arrays.asList(line.split(" ")).iterator(); + items.next(); //$$@@@@ + + try { + final String commitNum = items.next(); + final String commitId = items.next(); + final String[] parents = new String[Integer.parseInt(items.next())]; + for (int i = 0; i < parents.length; i++) { + parents[i] = items.next(); + } + final String branch = branchDecoder.decode(items.next()); + final String[] tags = new String[Integer.parseInt(items.next())]; + for (int i = 0; i < tags.length; i++) { + tags[i] = tagsDecoder.decode(items.next()); + } + + final String author = authorDecoder.decode(items.next()); + final String message = messageDecoder.decode(items.next()); + final long time = parseTime(dateFormat, items.next()); + final String hgsub = textOrNull(items.next()); + final String hgsubstate = textOrNull(items.next()); + + consumer.onCommit( + commitNum, + commitId, + parents, + branch, + tags, + author, + message, + time, + hgsub, + hgsubstate); + } catch (NoSuchElementException e) { + //NOP + } + } + } + + private static long parseTime(@NotNull final SimpleDateFormat dateFormat, + @NotNull final String time) { + try { + return dateFormat.parse(time).getTime(); + } catch (ParseException e) { + throw new RuntimeException("Failed to parse datetime: " + time + ". " + e, e); + } + } + + @Nullable + private static String textOrNull(@NotNull final String text) { + if (text.equals("=====")) return null; + return text; + } + + private static class Decoder { + private final Map myCache; + + public Decoder(final int SZ) { + myCache = new LinkedHashMap(SZ, 0.9f) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > SZ; + } + }; + } + + public String decode(@NotNull final String base64) throws UnsupportedEncodingException { + if (textOrNull(base64) == null) return null; + + final String result = myCache.get(base64); + if (result != null) return result; + + final String value = new String(Base64.decodeBase64(base64), "utf-8"); + //noinspection RedundantStringConstructorCall + myCache.put(new String(base64), value); + return value; + } + } +} diff -r c37903906fad -r 0f3c5ac38fc3 mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/LoadSubstatesCommand.java --- a/mercurial-common/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/LoadSubstatesCommand.java Mon Jan 13 19:20:39 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,129 +0,0 @@ -package jetbrains.buildServer.buildTriggers.vcs.mercurial.command; - -import com.intellij.openapi.diagnostic.Logger; -import jetbrains.buildServer.buildTriggers.vcs.mercurial.HgFileUtil; -import jetbrains.buildServer.util.FileUtil; -import jetbrains.buildServer.vcs.VcsException; -import org.apache.commons.codec.binary.Base64; -import org.jetbrains.annotations.NotNull; - -import java.io.*; -import java.util.LinkedHashMap; -import java.util.Map; - -/** - * Created 03.01.14 14:53 - * - * @author Eugene Petrenko (eugene.petrenko@jetbrains.com) - */ -public class LoadSubstatesCommand extends VcsRootCommand { - private static final Logger LOG = Logger.getInstance(LoadSubstatesCommand.class.getName()); - - public LoadSubstatesCommand(@NotNull final CommandSettings commandSettings, - @NotNull final String hgPath, - @NotNull final File workingDir, - @NotNull final AuthSettings authSettings) { - super(commandSettings, hgPath, workingDir, authSettings); - } - - @NotNull - private File createTmpDir() throws VcsException { - try { - return HgFileUtil.createTempDir(); - } catch (IOException e) { - throw new VcsException("Unable to create temporary directory", e); - } - } - - @NotNull - private File extractCommandPy(@NotNull final File root) throws VcsException { - try { - final File py = new File(root, "load-substates-command.py"); - - FileUtil.copyResource(getClass(), "/python/load-substates-command.py", py); - - if (py.length() < 100) throw new IOException("Failed to unpack command resource"); - return py; - } catch (IOException e) { - throw new VcsException("Failed to extract .py file: " + e.getMessage(), e); - } - } - - public void call(@NotNull final Object consumer) throws VcsException { - final File root = createTmpDir(); - - try { - final File py = extractCommandPy(root); - - callImpl(root, py, consumer); - } finally { - FileUtil.delete(root); - } - } - - private void callImpl(@NotNull final File root, - @NotNull final File commandPy, - @NotNull final Object consumer) throws VcsException { - final MercurialCommandLine cli = createCommandLine(); - cli.addParameter("--debug"); - cli.addParameter("--config"); - cli.addParameter("extensions.logextcj=" + commandPy); - cli.addParameter("load-substates-command"); - cli.addParameter(new File(root, "output.txt").getPath()); - - final CommandResult res = runCommand(cli); - final String output = res.getStdout(); - - if (!output.contains("##Completed##")) throw new VcsException("Command failed: " + output); - - - } - - public static interface LogConsumer { - void onCommit(@NotNull final String id, @NotNull final String sub, @NotNull final String state); - } - - - public static void parseDump(@NotNull final File dump, @NotNull final LogConsumer consumer) throws IOException { - final InputStream is = new BufferedInputStream(new FileInputStream(dump)); - - final Decoder sub = new Decoder(); - final Decoder substate = new Decoder(); - - final BufferedReader st = new BufferedReader(new InputStreamReader(is, "utf-8")); - String line; - while((line = st.readLine()) != null) { - if (!line.startsWith("$$@@@@ ")) continue; - final String[] items = line.split(" "); - if (items.length != 4) continue; - - final String commitId = items[1]; - final String hgsub = sub.decode(items[2]); - final String hgsubstate = substate.decode(items[3]); - - consumer.onCommit(commitId, hgsub, hgsubstate); - } - } - - - private static class Decoder { - private final int SZ = 20; - private final Map myCache = new LinkedHashMap(SZ, 0.9f) { - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > SZ; - } - }; - - @NotNull - public String decode(@NotNull final String base64) throws UnsupportedEncodingException { - final String result = myCache.get(base64); - if (result != null) return result; - - final String value = new String(Base64.decodeBase64(base64), "utf-8"); - //noinspection RedundantStringConstructorCall - myCache.put(new String(base64), value); - return value; - } - } -} diff -r c37903906fad -r 0f3c5ac38fc3 mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommitsAndMountPointsCommandParserTest.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/CommitsAndMountPointsCommandParserTest.java Mon Jan 13 19:42:11 2014 +0100 @@ -0,0 +1,67 @@ +package jetbrains.buildServer.buildTriggers.vcs.mercurial.command; + +import org.jetbrains.annotations.NotNull; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Created 08.01.14 22:39 + * + * @author Eugene Petrenko (eugene.petrenko@jetbrains.com) + */ +public class CommitsAndMountPointsCommandParserTest { + + @Test + public void test_parse_commits_a() throws IOException { + final List log = new ArrayList(); + CommitsAndMountPointsParser.parseCommits(new File("./mercurial-tests/testData/subst/a.commits"), proxy(CommitsAndMountPointsParser.CommitsConsumer.class, log)); + + for (String s : log) { + System.out.println(s); + } + Assert.assertEquals(log, Arrays.asList( + "onCommit [0, 4cc56f5e7eee469e4b33db02a22baaac46c1ecee, [0000000000000000000000000000000000000000], default, [], dsha, API changes, 1184063759000, null, null]", + "onCommit [5, 90ca28808a205909463386921939f59eb4259fcf, [e7298836dbcbbf3a79b255db718eca04f5b4824f], default, [], dsl, Switch to Orcas, 1184687116000, null, null]", + "onCommit [2490, 6a4484ecc639cc0e3783a86f40a3ad87405970fc, [1d30763d0f9f94ba7368c36a186119c11c453f4c, 549dca11e6a1080a81ce711849e20c23ec1b4e05], default, [], Sergey.Shkredov, Merge with serjic, 1334149579000, 9a12f93d5158a5cd75962d4ebd24fbe7e45ace53, e64b208e4ae1f3b3302731e358fc4c494dbe881d]", + "onCommit [2513, 6cef50ceefad10da64d256a01d2b2ac3153b465a, [afb1278c857e2dbfe1a6cd4e7e646e5fe7c48b51, 3cc6eef56b1ed4a49b39c0fb785334ca618c5e9c], SDKReferences, [], Kirill.Skrygan, Merge two SDKReferences heads, 1334240172000, 9a12f93d5158a5cd75962d4ebd24fbe7e45ace53, 35b0fe2dd3e1e2553a6407cf732e0fc7cd4c2e08]", + "onCommit [2592, 6f096ed0264b2378f98f125c545f01b227cdf23d, [9816adffa38d039da651b902a08d3ad45bc35a12, 9de1e1e88af7b440c6f4c55cb0410afdad2ce89f], default, [], Alexander.Shvedov, Merge with XamlWorkshop, 1334582015000, 9a12f93d5158a5cd75962d4ebd24fbe7e45ace53, 0b2f91b87bb6b3fc848111b9ddc6c250d6b06453]", + "onCommit [2593, b4047e0d64ecb070d2826166d1b4b9d25672e023, [c3205f71291d525b6d9282b8491abe20ec8373d0], serjic, [], Sergey.Shkredov, fix some problems., 1334574266000, 9a12f93d5158a5cd75962d4ebd24fbe7e45ace53, d8c99c4baa952506946dadd627ca918ffce24bc7]", + "onCommit [2600, ddd190f94b157f1ac215df16f530e51a91b4c7c2, [66f3b250e585ec69f9396bf311ce8fe986e4c1e3], xvost, [], xvost, RSRP-294634, 1334569744000, 9a12f93d5158a5cd75962d4ebd24fbe7e45ace53, c6a6c6fb5b932c1c89e0c411c150859ec4906078]", + "onCommit [812, 68f94180a341a6c174d015cd0fa0b10a66e2615b, [cf898c0dac48a20985a5b72dbc801a03300d4a3c], Subplatforms, [MsiReady], baltic, ++, 1328895848000, 9a12f93d5158a5cd75962d4ebd24fbe7e45ace53, 8418205f67a695ae82b4b4d4946cec9082202553]" + )); + } + + @Test + public void test_parse_files_a() throws IOException { + final List log = new ArrayList(); + CommitsAndMountPointsParser.parseFileLog(new File("./mercurial-tests/testData/subst/a.files"), proxy(CommitsAndMountPointsParser.ContentsConsumer.class, log)); + + for (String s : log) { + System.out.println(s); + } + Assert.assertEquals(log, Arrays.asList( + "onCommit [b80de5d138758541c5f05265ad144ab9fa86d1db, ]", + "onCommit [05a11677a1346201e93dee1968178c019b42bbb0, 0000000000000000000000000000000000000000 Platform\n]", + "onCommit [d0ae0cceebc043697902c66e9fec0ecf621d954c, 0c4b35f8765c032a4101fa2024267be31f3c8343 Platform\n]" + )); + } + + @NotNull + private T proxy(@NotNull final Class clazz, @NotNull final List log) { + return clazz.cast(Proxy.newProxyInstance(getClass().getClassLoader(), new Class[]{clazz}, new InvocationHandler() { + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + log.add(method.getName() + " " + Arrays.deepToString(args)); + return null; + } + })); + } +} diff -r c37903906fad -r 0f3c5ac38fc3 mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/LoadSubstatesCommandParserTest.java --- a/mercurial-tests/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/command/LoadSubstatesCommandParserTest.java Mon Jan 13 19:20:39 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,26 +0,0 @@ -package jetbrains.buildServer.buildTriggers.vcs.mercurial.command; - -import org.jetbrains.annotations.NotNull; -import org.testng.annotations.Test; - -import java.io.File; -import java.io.IOException; - -/** - * Created 08.01.14 22:39 - * - * @author Eugene Petrenko (eugene.petrenko@jetbrains.com) - */ -public class LoadSubstatesCommandParserTest { - - @Test - public void should_parse_huge_sample() throws IOException { - LoadSubstatesCommand.parseDump(new File("./mercurial-tests/testData/subst/substates.dump.txt"), new LoadSubstatesCommand.LogConsumer() { - public void onCommit(@NotNull String id, @NotNull String sub, @NotNull String state) { - System.out.println(id); - System.out.println(sub); - System.out.println(state); - } - }); - } -}