# HG changeset patch
# User Evgeniy.Koshkin@unit-519.Labs.IntelliJ.Net
# Date 1463415742 -10800
# Node ID 52fc27842bc162ba31e9dd398eeca7f21a2a3f12
# Parent d9cb6eec6f741f6420d7572483ba074e1334b911
fixing TW-45209, TW-45095; report resulting artifact path for each pdb file instead of file name from build agent
- will fix performance when locating pdb file on server
- will fix file name collisions when publishing several pdb's with the same name in scope of single build
diff -r d9cb6eec6f74 -r 52fc27842bc1 .idea/libraries/jdom.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/.idea/libraries/jdom.xml Mon May 16 19:22:22 2016 +0300
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff -r d9cb6eec6f74 -r 52fc27842bc1 .idea/misc.xml
--- a/.idea/misc.xml Mon May 16 15:03:56 2016 +0300
+++ b/.idea/misc.xml Mon May 16 19:22:22 2016 +0300
@@ -27,7 +27,7 @@
http://www.w3.org/1999/xhtml
-
+
\ No newline at end of file
diff -r d9cb6eec6f74 -r 52fc27842bc1 agent/src/jetbrains/buildServer/symbols/SymbolsIndexer.java
--- a/agent/src/jetbrains/buildServer/symbols/SymbolsIndexer.java Mon May 16 15:03:56 2016 +0300
+++ b/agent/src/jetbrains/buildServer/symbols/SymbolsIndexer.java Mon May 16 19:22:22 2016 +0300
@@ -9,16 +9,16 @@
import jetbrains.buildServer.util.EventDispatcher;
import jetbrains.buildServer.util.FileUtil;
import org.apache.log4j.Logger;
+import org.jdom.JDOMException;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
/**
* @author Evgeniy.Koshkin
@@ -33,7 +33,7 @@
@NotNull private final ArtifactsWatcher myArtifactsWatcher;
@NotNull private final JetSymbolsExe myJetSymbolsExe;
- @NotNull private final Collection mySymbolsToProcess = new CopyOnWriteArrayList();
+ @NotNull private final Map myFileToArtifactMapToProcess = new ConcurrentHashMap();
@Nullable private BuildProgressLogger myProgressLogger;
@Nullable private File myBuildTempDirectory;
@@ -73,26 +73,31 @@
public void afterAtrifactsPublished(@NotNull AgentRunningBuild build, @NotNull BuildFinishedStatus buildStatus) {
super.afterAtrifactsPublished(build, buildStatus);
if(!isIndexingApplicable()) return;
- if (mySymbolsToProcess.isEmpty()) {
+ if (myFileToArtifactMapToProcess.isEmpty()) {
myProgressLogger.warning("Symbols weren't found in artifacts to be published.");
LOG.debug("Symbols weren't found in artifacts to be published for build with id " + build.getBuildId());
} else {
myProgressLogger.message("Collecting symbol files signatures.");
LOG.debug("Collecting symbol files signatures.");
try {
- final File symbolSignaturesFile = FileUtil.createTempFile(myBuildTempDirectory, "symbol-signatures-", ".xml", false);
- myJetSymbolsExe.dumpGuidsToFile(mySymbolsToProcess, symbolSignaturesFile, myProgressLogger);
- if(symbolSignaturesFile.exists()){
+ final Set signatureLocalFilesData = getSignatures(myFileToArtifactMapToProcess.keySet());
+ if(!signatureLocalFilesData.isEmpty()){
+ final Set indexData = new HashSet();
+ for(PdbSignatureIndexEntry signatureIndexEntry : signatureLocalFilesData){
+ indexData.add(new PdbSignatureIndexEntry(signatureIndexEntry.getGuid(), myFileToArtifactMapToProcess.get(new File(signatureIndexEntry.getArtifactPath()))));
+ }
+ final File indexDataFile = FileUtil.createTempFile(myBuildTempDirectory, SymbolsConstants.SYMBOL_SIGNATURES_FILE_NAME_PREFIX, ".xml", false);
+ PdbSignatureIndexUtil.write(new FileOutputStream(indexDataFile), indexData);
myProgressLogger.message("Publishing collected symbol files signatures.");
- myArtifactsWatcher.addNewArtifactsPath(symbolSignaturesFile + "=>" + ".teamcity/symbols");
+ myArtifactsWatcher.addNewArtifactsPath(indexDataFile + "=>" + ".teamcity/symbols");
}
- } catch (IOException e) {
+ } catch (Exception e) {
LOG.error("Error while dumping symbols/binaries signatures for build with id " + build.getBuildId(), e);
myProgressLogger.error("Error while dumping symbols/binaries signatures.");
myProgressLogger.exception(e);
}
}
- mySymbolsToProcess.clear();
+ myFileToArtifactMapToProcess.clear();
}
});
}
@@ -106,19 +111,19 @@
}
LOG.debug("Searching for symbol files in publishing artifacts.");
- final Collection pdbFiles = getArtifactPathsByFileExtension(artifacts, PDB_FILE_EXTENSION);
+ final Map pdbFiles = getArtifactPathsByFileExtension(artifacts, PDB_FILE_EXTENSION);
if(pdbFiles.isEmpty()) return;
final PdbFilePatcher pdbFilePatcher = new PdbFilePatcher(myBuildTempDirectory, mySrcSrvHomeDir, new SrcSrvStreamBuilder(myFileUrlProvider, myProgressLogger));
- for(File pdbFile : pdbFiles){
- if(mySymbolsToProcess.contains(pdbFile)){
+ for(File pdbFile : pdbFiles.keySet()){
+ if(myFileToArtifactMapToProcess.containsKey(pdbFile)){
LOG.debug(String.format("File %s already processed. Skipped.", pdbFile.getAbsolutePath()));
continue;
}
try {
myProgressLogger.message("Indexing sources appeared in file " + pdbFile.getAbsolutePath());
pdbFilePatcher.patch(pdbFile, myProgressLogger);
- mySymbolsToProcess.add(pdbFile);
+ myFileToArtifactMapToProcess.put(pdbFile, pdbFiles.get(pdbFile));
} catch (Throwable e) {
LOG.error("Error occurred while patching symbols file " + pdbFile, e);
myProgressLogger.error("Error occurred while patching symbols file " + pdbFile);
@@ -127,6 +132,18 @@
}
}
+ private Set getSignatures(Collection files) throws IOException, JDOMException {
+ final File guidDumpFile = FileUtil.createTempFile(myBuildTempDirectory, "symbol-signatures-local-", ".xml", false);
+ myJetSymbolsExe.dumpGuidsToFile(files, guidDumpFile, myProgressLogger);
+ if(guidDumpFile.exists()){
+ myArtifactsWatcher.addNewArtifactsPath(guidDumpFile + "=>" + ".teamcity/symbols");
+ }
+ if(guidDumpFile.isFile())
+ return PdbSignatureIndexUtil.read(new FileInputStream(guidDumpFile));
+ else
+ return Collections.emptySet();
+ }
+
@Nullable
private File getSrcSrvHomeDir(@NotNull AgentRunningBuild build) {
final Map agentConfigParameters = build.getAgentConfiguration().getConfigurationParameters();
@@ -143,13 +160,14 @@
return null;
}
- private Collection getArtifactPathsByFileExtension(List artifactsCollections, String fileExtension){
- final Collection result = new HashSet();
+ private Map getArtifactPathsByFileExtension(List artifactsCollections, String fileExtension){
+ final Map result = new HashMap();
for(ArtifactsCollection artifactsCollection : artifactsCollections){
if(artifactsCollection.isEmpty()) continue;
- for (File artifact : artifactsCollection.getFilePathMap().keySet()){
+ final Map filePathMap = artifactsCollection.getFilePathMap();
+ for (final File artifact : filePathMap.keySet()){
if(FileUtil.getExtension(artifact.getPath()).equalsIgnoreCase(fileExtension))
- result.add(artifact);
+ result.put(artifact, filePathMap.get(artifact));
}
}
return result;
diff -r d9cb6eec6f74 -r 52fc27842bc1 common/common.iml
--- a/common/common.iml Mon May 16 15:03:56 2016 +0300
+++ b/common/common.iml Mon May 16 19:22:22 2016 +0300
@@ -8,8 +8,7 @@
-
+
-
-
+
\ No newline at end of file
diff -r d9cb6eec6f74 -r 52fc27842bc1 common/src/jetbrains/buildServer/symbols/PdbSignatureIndexEntry.java
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/common/src/jetbrains/buildServer/symbols/PdbSignatureIndexEntry.java Mon May 16 19:22:22 2016 +0300
@@ -0,0 +1,26 @@
+package jetbrains.buildServer.symbols;
+
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Created by Evgeniy.Koshkin.
+ */
+public class PdbSignatureIndexEntry {
+ private final String myGuid;
+ private final String myArtifactPath;
+
+ public PdbSignatureIndexEntry(@NotNull String guid, @NotNull String artifactPath) {
+ myGuid = guid;
+ myArtifactPath = artifactPath;
+ }
+
+ @NotNull
+ public String getGuid() {
+ return myGuid;
+ }
+
+ @NotNull
+ public String getArtifactPath() {
+ return myArtifactPath;
+ }
+}
diff -r d9cb6eec6f74 -r 52fc27842bc1 common/src/jetbrains/buildServer/symbols/PdbSignatureIndexUtil.java
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/common/src/jetbrains/buildServer/symbols/PdbSignatureIndexUtil.java Mon May 16 19:22:22 2016 +0300
@@ -0,0 +1,50 @@
+package jetbrains.buildServer.symbols;
+
+import jetbrains.buildServer.util.XmlUtil;
+import org.jdom.Document;
+import org.jdom.Element;
+import org.jdom.JDOMException;
+import org.jdom.input.SAXBuilder;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.*;
+
+/**
+ * Created by Evgeniy.Koshkin.
+ */
+class PdbSignatureIndexUtil {
+ private static final String SIGN = "sign";
+ private static final String FILE = "file";
+ private static final String FILE_SIGNS = "file-signs";
+ private static final String FILE_SIGN_ENTRY = "file-sign-entry";
+
+ @NotNull
+ static Set read(@NotNull final InputStream inputStream) throws JDOMException, IOException {
+ final SAXBuilder builder = new SAXBuilder();
+ final Document document = builder.build(inputStream);
+ final Set result = new HashSet();
+ for (Object signElementObject : document.getRootElement().getChildren()){
+ final Element signElement = (Element) signElementObject;
+ result.add(new PdbSignatureIndexEntry(extractGuid(signElement.getAttributeValue(SIGN)), signElement.getAttributeValue(FILE)));
+ }
+ return result;
+ }
+
+ static void write(@NotNull final OutputStream outputStream, @NotNull final Set indexData) throws IOException {
+ final Element root = new Element(FILE_SIGNS);
+ for (final PdbSignatureIndexEntry indexEntry : indexData){
+ final Element entry = new Element(FILE_SIGN_ENTRY);
+ entry.setAttribute(SIGN, indexEntry.getGuid());
+ entry.setAttribute(FILE, indexEntry.getArtifactPath());
+ root.addContent(root);
+ }
+ XmlUtil.saveDocument(new Document(root), outputStream);
+ }
+
+ private static String extractGuid(String sign) {
+ return sign.substring(0, sign.length() - 1).toLowerCase(); //last symbol is PEDebugType
+ }
+}
diff -r d9cb6eec6f74 -r 52fc27842bc1 common/src/jetbrains/buildServer/symbols/SymbolsConstants.java
--- a/common/src/jetbrains/buildServer/symbols/SymbolsConstants.java Mon May 16 15:03:56 2016 +0300
+++ b/common/src/jetbrains/buildServer/symbols/SymbolsConstants.java Mon May 16 19:22:22 2016 +0300
@@ -11,4 +11,6 @@
public static final String APP_SYMBOLS = "/app/symbols/";
public static final String APP_SOURCES = "/app/sources/";
+
+ public static final String SYMBOL_SIGNATURES_FILE_NAME_PREFIX = "symbol-signatures-artifacts-";
}
diff -r d9cb6eec6f74 -r 52fc27842bc1 server/src/jetbrains/buildServer/symbols/BuildSymbolsIndexProvider.java
--- a/server/src/jetbrains/buildServer/symbols/BuildSymbolsIndexProvider.java Mon May 16 15:03:56 2016 +0300
+++ b/server/src/jetbrains/buildServer/symbols/BuildSymbolsIndexProvider.java Mon May 16 19:22:22 2016 +0300
@@ -9,17 +9,14 @@
import jetbrains.buildServer.util.CollectionsUtil;
import jetbrains.buildServer.util.filters.Filter;
import org.apache.log4j.Logger;
-import org.jdom.Document;
-import org.jdom.Element;
import org.jdom.JDOMException;
-import org.jdom.input.SAXBuilder;
import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
import java.io.IOException;
-import java.io.InputStream;
import java.util.Collections;
import java.util.HashMap;
-import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -44,7 +41,7 @@
final BuildArtifact symbols = sBuild.getArtifacts(BuildArtifactsViewMode.VIEW_HIDDEN_ONLY).getArtifact(".teamcity/symbols");
final BuildArtifact symbolSignaturesSource = symbols == null ? null : CollectionsUtil.findFirst(symbols.getChildren(), new Filter() {
public boolean accept(@NotNull BuildArtifact data) {
- return data.getName().startsWith("symbol-signatures");
+ return data.getName().startsWith(SymbolsConstants.SYMBOL_SIGNATURES_FILE_NAME_PREFIX);
}
});
if(symbolSignaturesSource == null) {
@@ -52,54 +49,44 @@
return;
}
- Map indexData = Collections.emptyMap();
+ Set indexEntries = Collections.emptySet();
try {
- indexData = readIndex(symbolSignaturesSource.getInputStream());
+ indexEntries = PdbSignatureIndexUtil.read(symbolSignaturesSource.getInputStream());
} catch (IOException e) {
LOG.debug("Failed to read symbols index data from artifact " + symbolSignaturesSource.getRelativePath(), e);
} catch (JDOMException e) {
LOG.debug("Failed to read symbols index data from artifact " + symbolSignaturesSource.getRelativePath(), e);
}
- LOG.debug(String.format("Build with id %d provides %d symbol file signatures.", buildId, indexData.size()));
+ LOG.debug(String.format("Build with id %d provides %d symbol file signatures.", buildId, indexEntries.size()));
- for (String signature : indexData.keySet()) {
- final String fileName = indexData.get(signature);
- final String artifactPath = locateArtifact(sBuild, fileName);
+ for (final PdbSignatureIndexEntry indexEntry : indexEntries) {
+ final String signature = indexEntry.getGuid();
+ final String artifactPathOrName = indexEntry.getArtifactPath();
+ final String artifactPath = locateArtifact(sBuild, artifactPathOrName);
if (artifactPath == null) {
- LOG.debug(String.format("Failed to find artifact by name %s and build id %d.", fileName, buildId));
+ LOG.debug(String.format("Failed to find artifact by name %s and build id %d.", artifactPathOrName, buildId));
continue;
}
final HashMap data = new HashMap();
data.put(ARTIFACT_PATH_KEY, artifactPath);
- data.put(FILE_NAME_KEY, fileName);
+ data.put(FILE_NAME_KEY, artifactPathOrName);
metadataStorageWriter.addParameters(signature, data);
- LOG.debug("Stored symbol file signature " + signature + " for file name " + fileName + " build id " + buildId);
+ LOG.debug("Stored symbol file signature " + signature + " for file name " + artifactPathOrName + " build id " + buildId);
}
}
- @NotNull
- private Map readIndex(InputStream inputStream) throws JDOMException, IOException {
- SAXBuilder builder = new SAXBuilder();
- Document document = builder.build(inputStream);
- Map result = new HashMap();
- for (Object signElementObject : document.getRootElement().getChildren()){
- final Element signElement = (Element) signElementObject;
- result.put(extractGuid(signElement.getAttributeValue("sign")), signElement.getAttributeValue("file"));
- }
- return result;
- }
+ @Nullable
+ private String locateArtifact(@NotNull SBuild build, final @NotNull String artifactPathOrName) {
+ final BuildArtifacts artifacts = build.getArtifacts(BuildArtifactsViewMode.VIEW_DEFAULT_WITH_ARCHIVES_CONTENT);
+ final BuildArtifact artifact = artifacts.getArtifact(artifactPathOrName);
+ if(artifact != null) return artifactPathOrName;
- private String extractGuid(String sign) {
- return sign.substring(0, sign.length() - 1).toLowerCase(); //last symbol is PEDebugType
- }
-
- private String locateArtifact(SBuild build, final String fileName) {
final AtomicReference locatedArtifactPath = new AtomicReference(null);
- build.getArtifacts(BuildArtifactsViewMode.VIEW_DEFAULT_WITH_ARCHIVES_CONTENT).iterateArtifacts(new BuildArtifacts.BuildArtifactsProcessor() {
+ artifacts.iterateArtifacts(new BuildArtifacts.BuildArtifactsProcessor() {
@NotNull
public Continuation processBuildArtifact(@NotNull BuildArtifact artifact) {
- if(artifact.getName().equals(fileName)){
+ if(artifact.getName().equals(artifactPathOrName)){
locatedArtifactPath.set(artifact.getRelativePath());
return Continuation.BREAK;
}