# HG changeset patch # User Evgeniy.Koshkin # Date 1375207830 -14400 # Node ID 662a0be55de7717f26d9b02d34133ce05d90f8ad # Parent 5fb218a7e574a79089d18e99e85d5bbf31aa41c0 - source indexing works! - few tests added diff -r 5fb218a7e574 -r 662a0be55de7 agent/src/jetbrains/buildServer/symbols/PdbFilePatcher.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/agent/src/jetbrains/buildServer/symbols/PdbFilePatcher.java Tue Jul 30 22:10:30 2013 +0400 @@ -0,0 +1,32 @@ +package jetbrains.buildServer.symbols; + +import jetbrains.buildServer.util.FileUtil; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; + +/** + * @author Evgeniy.Koshkin + */ +public class PdbFilePatcher { + + private final PdbStrExe myPdbStrExe = new PdbStrExe(); + private final SrcToolExe mySrcToolExe = new SrcToolExe(); + + private final File myHomeDir; + private SrcSrvStreamProvider myIndexInputProvider; + + public PdbFilePatcher(final File homeDir, final SrcSrvStreamProvider indexInputProvider) { + myHomeDir = homeDir; + myIndexInputProvider = indexInputProvider; + } + + public void patch(File symbolsFile) throws IOException { + final Collection sourceFiles = mySrcToolExe.getReferencedSourceFiles(symbolsFile); + final File tmpFile = FileUtil.createTempFile(myHomeDir, "pdb-patch", ".xml", false); + myIndexInputProvider.dumpStreamToFile(tmpFile, sourceFiles); + myPdbStrExe.doCommand(PdbStrExeCommand.WRITE, symbolsFile, tmpFile, PdbStrExe.SRCSRV_STREAM_NAME); + //TODO: check that data was actually written + } +} diff -r 5fb218a7e574 -r 662a0be55de7 agent/src/jetbrains/buildServer/symbols/PdbStrExe.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/agent/src/jetbrains/buildServer/symbols/PdbStrExe.java Tue Jul 30 22:10:30 2013 +0400 @@ -0,0 +1,48 @@ +/* + * Copyright 2000-2013 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jetbrains.buildServer.symbols; + +import com.intellij.execution.configurations.GeneralCommandLine; +import jetbrains.buildServer.ExecResult; +import jetbrains.buildServer.SimpleCommandLineProcessRunner; + +import java.io.File; + +/** + * @author Evgeniy.Koshkin + */ +public class PdbStrExe { + + public static final String SRCSRV_STREAM_NAME = "srcsrv"; + + private static final String STREAM_NAME_SWITCH = "-s"; + private static final String PATH_TO_PDB_FILE_SWITCH = "-p"; + private static final String PATH_TO_INPUT_FILE_SWITCH = "-i"; + + private final File myPath = new File("c:\\Program Files (x86)\\Windows Kits\\8.0\\Debuggers\\x64\\srcsrv\\pdbstr.exe"); + + public ExecResult doCommand(final PdbStrExeCommand cmd, final File pdbFile, final File inputStreamFile, final String streamName){ + final GeneralCommandLine commandLine = new GeneralCommandLine(); + commandLine.setWorkDirectory(myPath.getParent()); + commandLine.setExePath(myPath.getPath()); + commandLine.addParameter(cmd.getCmdSwitch()); + commandLine.addParameter(String.format("%s:%s", PATH_TO_PDB_FILE_SWITCH, pdbFile.getAbsolutePath())); + commandLine.addParameter(String.format("%s:%s", PATH_TO_INPUT_FILE_SWITCH, inputStreamFile.getAbsolutePath())); + commandLine.addParameter(STREAM_NAME_SWITCH + ":" + streamName); + return SimpleCommandLineProcessRunner.runCommand(commandLine, null); + } +} diff -r 5fb218a7e574 -r 662a0be55de7 agent/src/jetbrains/buildServer/symbols/PdbStrExeCommand.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/agent/src/jetbrains/buildServer/symbols/PdbStrExeCommand.java Tue Jul 30 22:10:30 2013 +0400 @@ -0,0 +1,37 @@ +/* + * Copyright 2000-2013 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jetbrains.buildServer.symbols; + +/** + * @author Evgeniy.Koshkin + */ +public enum PdbStrExeCommand { + READ { + @Override + public String getCmdSwitch() { + return "-r"; + } + }, + WRITE { + @Override + public String getCmdSwitch() { + return "-w"; + } + }; + + public abstract String getCmdSwitch(); +} diff -r 5fb218a7e574 -r 662a0be55de7 agent/src/jetbrains/buildServer/symbols/SrcSrvStreamProvider.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/agent/src/jetbrains/buildServer/symbols/SrcSrvStreamProvider.java Tue Jul 30 22:10:30 2013 +0400 @@ -0,0 +1,70 @@ +/* + * Copyright 2000-2013 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jetbrains.buildServer.symbols; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URI; +import java.util.Collection; +import java.util.Date; + +/** + * @author Evgeniy.Koshkin + */ +public class SrcSrvStreamProvider { + + private String myRestApiUrl = "http://localhost:8111/bs/guestAuth/app/rest"; + private long myBuildId; + private File mySourcesRootDirectory; + + public SrcSrvStreamProvider(final long buildId, final File sourcesRootDirectory) { + myBuildId = buildId; + mySourcesRootDirectory = sourcesRootDirectory; + } + + public void dumpStreamToFile(File targetFile, Collection sourceFiles) throws IOException { + final FileWriter fileWriter = new FileWriter(targetFile.getPath(), true); + try { + fileWriter.write("SRCSRV: ini ------------------------------------------------"); + fileWriter.write(String.format("VERSION=%d", 1)); + fileWriter.write(String.format("INDEXVERSION=%d", 1)); + fileWriter.write("VERCTRL=http"); + fileWriter.write(String.format("DATETIME=%s", (new Date()).toString())); + + fileWriter.write("SRCSRV: variables ------------------------------------------"); + fileWriter.write("SRCSRVVERCTRL=http"); + fileWriter.write(String.format("REST_API_URL=%s", myRestApiUrl)); + fileWriter.write(String.format("BUILD_LOCATOR=id:%d", myBuildId)); + fileWriter.write("HTTP_EXTRACT_TARGET=%REST_API_URL%/%BUILD_LOCATOR%/sources/files/%var2%"); + fileWriter.write("SRCSRVTRG=%HTTP_EXTRACT_TARGET%"); + fileWriter.write("SRCSRVCMD="); + + final URI checkoutDirUri = mySourcesRootDirectory.toURI(); + fileWriter.write("SRCSRV: source files ------------------------------------------"); + for(File sourceFile : sourceFiles){ + final File sourceFileAbsolute = sourceFile.getAbsoluteFile(); + fileWriter.write(String.format("%s*%s", sourceFileAbsolute.getPath(), checkoutDirUri.relativize(sourceFileAbsolute.toURI()).getPath())); + } + + fileWriter.write("SRCSRV: end ------------------------------------------------"); + } + finally { + fileWriter.close(); + } + } +} diff -r 5fb218a7e574 -r 662a0be55de7 agent/src/jetbrains/buildServer/symbols/SrcToolExe.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/agent/src/jetbrains/buildServer/symbols/SrcToolExe.java Tue Jul 30 22:10:30 2013 +0400 @@ -0,0 +1,52 @@ +/* + * Copyright 2000-2013 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jetbrains.buildServer.symbols; + +import com.intellij.execution.configurations.GeneralCommandLine; +import jetbrains.buildServer.ExecResult; +import jetbrains.buildServer.SimpleCommandLineProcessRunner; +import jetbrains.buildServer.util.CollectionsUtil; +import jetbrains.buildServer.util.Converter; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.util.Arrays; +import java.util.Collection; + +/** + * @author Evgeniy.Koshkin + */ +public class SrcToolExe { + private static final String DUMP_SOURCES_FROM_PDB_SWITCH = "-r"; + + private final File mySrcToolPath = new File("c:\\Program Files (x86)\\Windows Kits\\8.0\\Debuggers\\x64\\srcsrv\\srctool.exe"); + + public Collection getReferencedSourceFiles(File symbolsFile) { + final GeneralCommandLine commandLine = new GeneralCommandLine(); + commandLine.setExePath(mySrcToolPath.getPath()); + commandLine.addParameter(symbolsFile.getAbsolutePath()); + commandLine.addParameter(DUMP_SOURCES_FROM_PDB_SWITCH); + final ExecResult execResult = SimpleCommandLineProcessRunner.runCommand(commandLine, null); + return CollectionsUtil.convertAndFilterNulls(Arrays.asList(execResult.getOutLines()), new Converter() { + public File createFrom(@NotNull String source) { + final File file = new File(source); + if (file.isFile()) return file; + return null; //last string is not a source file path + } + }); + } +} diff -r 5fb218a7e574 -r 662a0be55de7 agent/src/jetbrains/buildServer/symbols/SymbolsIndexer.java --- a/agent/src/jetbrains/buildServer/symbols/SymbolsIndexer.java Thu Jul 25 19:11:53 2013 +0400 +++ b/agent/src/jetbrains/buildServer/symbols/SymbolsIndexer.java Tue Jul 30 22:10:30 2013 +0400 @@ -26,8 +26,6 @@ private static final Logger LOG = Logger.getLogger(SymbolsIndexer.class); public static final String PDB_FILE_EXTENSION = "pdb"; - public static final String EXE_FILE_EXTENSION = "exe"; - public static final String DLL_FILE_EXTENSION = "dll"; public static final String SYMBOLS_EXE = "JetBrains.CommandLine.Symbols.exe"; public static final String DUMP_SYMBOL_SIGN_CMD = "dumpSymbolSign"; @@ -35,33 +33,31 @@ @NotNull private final ArtifactsWatcher myArtifactsWatcher; @NotNull private final File myNativeToolPath; @Nullable private AgentRunningBuild myBuild; - @Nullable private Collection myBinariesToProcess; @Nullable private Collection mySymbolsToProcess; public SymbolsIndexer(@NotNull final PluginDescriptor pluginDescriptor, @NotNull final EventDispatcher agentDispatcher, @NotNull final ArtifactsWatcher artifactsWatcher) { + myArtifactsWatcher = artifactsWatcher; myNativeToolPath = new File(new File(pluginDescriptor.getPluginRoot(), "bin"), SYMBOLS_EXE); - myArtifactsWatcher = artifactsWatcher; + agentDispatcher.addListener(new AgentLifeCycleAdapter() { @Override public void buildStarted(@NotNull final AgentRunningBuild runningBuild) { myBuild = runningBuild; - myBinariesToProcess = new HashSet(); mySymbolsToProcess = new HashSet(); } @Override public void afterAtrifactsPublished(@NotNull AgentRunningBuild build, @NotNull BuildFinishedStatus buildStatus) { super.afterAtrifactsPublished(build, buildStatus); - if (myBuild == null || mySymbolsToProcess == null || myBinariesToProcess == null) return; + if (myBuild == null || mySymbolsToProcess == null) return; if(myBuild.getBuildFeaturesOfType(SymbolsConstants.BUILD_FEATURE_TYPE).isEmpty()) return; if (mySymbolsToProcess.isEmpty()) { myBuild.getBuildLogger().warning("Symbols weren't found in artifacts to be published."); LOG.debug("Symbols weren't found in artifacts to be published."); } else { - final File targetDir = myBuild.getBuildTempDirectory(); try { - final File symbolSignaturesFile = dumpSymbolSignatures(mySymbolsToProcess, targetDir, myBuild.getBuildLogger()); + final File symbolSignaturesFile = dumpSymbolSignatures(mySymbolsToProcess, myBuild.getBuildTempDirectory(), myBuild.getBuildLogger()); if(symbolSignaturesFile.exists()){ myArtifactsWatcher.addNewArtifactsPath(symbolSignaturesFile + "=>" + ".teamcity/symbols"); } @@ -72,7 +68,6 @@ } } mySymbolsToProcess = null; - myBinariesToProcess = null; myBuild = null; } }); @@ -81,17 +76,26 @@ @Override public void afterCollectingFiles(@NotNull List artifacts) { super.afterCollectingFiles(artifacts); - if(myBuild == null || mySymbolsToProcess == null || myBinariesToProcess == null) return; + if(myBuild == null || mySymbolsToProcess == null) return; if(myBuild.getBuildFeaturesOfType(SymbolsConstants.BUILD_FEATURE_TYPE).isEmpty()){ myBuild.getBuildLogger().warning(SymbolsConstants.BUILD_FEATURE_TYPE + " build feature disabled. No indexing performed."); LOG.debug(SymbolsConstants.BUILD_FEATURE_TYPE + " build feature disabled. No indexing performed."); return; } - myBuild.getBuildLogger().message(SymbolsConstants.BUILD_FEATURE_TYPE + " build feature enabled. Searching for suitable files."); LOG.debug(SymbolsConstants.BUILD_FEATURE_TYPE + " build feature enabled. Searching for suitable files."); - mySymbolsToProcess.addAll(getArtifactPathsByFileExtension(artifacts, PDB_FILE_EXTENSION)); - myBinariesToProcess.addAll(getArtifactPathsByFileExtension(artifacts, EXE_FILE_EXTENSION)); - myBinariesToProcess.addAll(getArtifactPathsByFileExtension(artifacts, DLL_FILE_EXTENSION)); + Collection pdbFiles = getArtifactPathsByFileExtension(artifacts, PDB_FILE_EXTENSION); + final PdbFilePatcher pdbFilePatcher = new PdbFilePatcher(myBuild.getBuildTempDirectory(), new SrcSrvStreamProvider(myBuild.getBuildId(), myBuild.getCheckoutDirectory())); + for(File pdbFile : pdbFiles){ + try { + myBuild.getBuildLogger().message("Indexing sources appeared in file " + pdbFile.getAbsolutePath()); + pdbFilePatcher.patch(pdbFile); + mySymbolsToProcess.add(pdbFile); + } catch (Throwable e) { + LOG.error("Error occurred while patching symbols file " + pdbFile, e); + myBuild.getBuildLogger().error("Error occurred while patching symbols file " + pdbFile); + myBuild.getBuildLogger().exception(e); + } + } } private Collection getArtifactPathsByFileExtension(List artifactsCollections, String fileExtension){ @@ -108,11 +112,11 @@ private File dumpSymbolSignatures(Collection files, File targetDir, BuildProgressLogger buildLogger) throws IOException { final File tempFile = FileUtil.createTempFile(targetDir, "symbol-signatures-", ".xml", false); - runTool(DUMP_SYMBOL_SIGN_CMD, files, tempFile, buildLogger); + runNativeTool(DUMP_SYMBOL_SIGN_CMD, files, tempFile, buildLogger); return tempFile; } - private void runTool(String cmd, Collection files, File output, BuildProgressLogger buildLogger){ + private void runNativeTool(String cmd, Collection files, File output, BuildProgressLogger buildLogger){ final GeneralCommandLine commandLine = new GeneralCommandLine(); commandLine.setExePath(myNativeToolPath.getPath()); commandLine.addParameter(cmd); diff -r 5fb218a7e574 -r 662a0be55de7 tests/src/PdbFilePatcherTest.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/src/PdbFilePatcherTest.java Tue Jul 30 22:10:30 2013 +0400 @@ -0,0 +1,33 @@ +import com.intellij.openapi.util.io.FileUtil; +import jetbrains.buildServer.BaseTestCase; +import jetbrains.buildServer.symbols.PdbFilePatcher; +import jetbrains.buildServer.symbols.SrcSrvStreamProvider; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; + +/** + * @author Evgeniy.Koshkin + */ +public class PdbFilePatcherTest extends BaseTestCase { + + private PdbFilePatcher myPatcher; + private File myTestHomeDir; + + @Override + @BeforeMethod + public void setUp() throws Exception { + super.setUp(); + myTestHomeDir = createTempDir(); + myPatcher = new PdbFilePatcher(myTestHomeDir, new SrcSrvStreamProvider(1111, myTestHomeDir)); + } + + @Test + public void testFoo() throws IOException { + File tempFile = new File(myTestHomeDir, "tmp.pdb"); + FileUtil.copy(new File("c:\\temp\\JetBrains.CommandLine.Symbols.pdb"), tempFile); + myPatcher.patch(tempFile); + } +} diff -r 5fb218a7e574 -r 662a0be55de7 tests/src/PdbStrExeTest.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/src/PdbStrExeTest.java Tue Jul 30 22:10:30 2013 +0400 @@ -0,0 +1,76 @@ +/* + * Copyright 2000-2013 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.intellij.openapi.util.io.FileUtil; +import jetbrains.buildServer.BaseTestCase; +import jetbrains.buildServer.ExecResult; +import jetbrains.buildServer.symbols.PdbStrExe; +import jetbrains.buildServer.symbols.PdbStrExeCommand; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; + +/** + * @author Evgeniy.Koshkin + */ +public class PdbStrExeTest extends BaseTestCase { + + private PdbStrExe myTool; + private File myNotIndexedPdbFile; + private File myIndexedPdbFile; + + @BeforeMethod + public void setUp() throws Exception { + myTool = new PdbStrExe(); + File homeDir = createTempDir(); + + File file = new File(homeDir, "notIndexed.pdb"); + FileUtil.copy(new File("c:\\temp\\JetBrains.CommandLine.Symbols.pdb"), file); + myNotIndexedPdbFile = file; + assertFalse(myNotIndexedPdbFile.length() == 0); + + file = new File(homeDir, "indexed.pdb"); + FileUtil.copy(new File("c:\\temp\\JetBrains.CommandLine.Symbols.Indexed.pdb"), file); + myIndexedPdbFile = file; + assertFalse(myIndexedPdbFile.length() == 0); + } + + @Test + public void testRead() throws Exception { + final File tempFile = createTempFile(); + assertTrue(tempFile.length() == 0); + ExecResult execResult = myTool.doCommand(PdbStrExeCommand.READ, myIndexedPdbFile, tempFile, PdbStrExe.SRCSRV_STREAM_NAME); + assertEquals(0, execResult.getExitCode()); + assertFalse(tempFile.length() == 0); + } + + @Test + public void testWrite() throws IOException { + final File tempFile = createTempFile(); + assertTrue(tempFile.length() == 0); + myTool.doCommand(PdbStrExeCommand.READ, myNotIndexedPdbFile, tempFile, PdbStrExe.SRCSRV_STREAM_NAME); + assertTrue(tempFile.length() == 0); + + File inputStreamFile = new File("c:\\temp\\pdb-patch.txt"); + assertFalse(inputStreamFile.length() == 0); + myTool.doCommand(PdbStrExeCommand.WRITE, myNotIndexedPdbFile, inputStreamFile, PdbStrExe.SRCSRV_STREAM_NAME); + + myTool.doCommand(PdbStrExeCommand.READ, myNotIndexedPdbFile, tempFile, PdbStrExe.SRCSRV_STREAM_NAME); + assertFalse(tempFile.length() == 0); + } +} diff -r 5fb218a7e574 -r 662a0be55de7 tests/src/SimpleTest.java --- a/tests/src/SimpleTest.java Thu Jul 25 19:11:53 2013 +0400 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,13 +0,0 @@ -import jetbrains.buildServer.BaseTestCase; -import org.testng.annotations.Test; - -/** - * @author Eugene Petrenko (eugene.petrenko@gmail.com) - * Date: 16.11.11 17:14 - */ -public class SimpleTest extends BaseTestCase { - @Test - public void Test() { - - } -} diff -r 5fb218a7e574 -r 662a0be55de7 tests/src/testng-fast.xml --- a/tests/src/testng-fast.xml Thu Jul 25 19:11:53 2013 +0400 +++ b/tests/src/testng-fast.xml Tue Jul 30 22:10:30 2013 +0400 @@ -1,8 +1,9 @@ - - + + - + +