Mercurial > hg > tc-symbol-server

changeset 101:52fc27842bc1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fixing TW-45209, TW-45095; report resulting artifact path for each pdb file instead of file name from build agent - will fix performance when locating pdb file on server - will fix file name collisions when publishing several pdb's with the same name in scope of single build
author Evgeniy.Koshkin@unit-519.Labs.IntelliJ.Net
date Mon, 16 May 2016 19:22:22 +0300
parents d9cb6eec6f74
children 0e927b14ccf1
files .idea/libraries/jdom.xml .idea/misc.xml agent/src/jetbrains/buildServer/symbols/SymbolsIndexer.java common/common.iml common/src/jetbrains/buildServer/symbols/PdbSignatureIndexEntry.java common/src/jetbrains/buildServer/symbols/PdbSignatureIndexUtil.java common/src/jetbrains/buildServer/symbols/SymbolsConstants.java server/src/jetbrains/buildServer/symbols/BuildSymbolsIndexProvider.java
diffstat 8 files changed, 149 insertions(+), 58 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/.idea/libraries/jdom.xml	Mon May 16 19:22:22 2016 +0300
@@ -0,0 +1,9 @@
+<component name="libraryTable">
+  <library name="jdom">
+    <CLASSES>
+      <root url="jar://$teamcitydistribution$/webapps/ROOT/WEB-INF/lib/jdom.jar!/" />
+    </CLASSES>
+    <JAVADOC />
+    <SOURCES />
+  </library>
+</component>
\ No newline at end of file
--- a/.idea/misc.xml	Mon May 16 15:03:56 2016 +0300
+++ b/.idea/misc.xml	Mon May 16 19:22:22 2016 +0300
@@ -27,7 +27,7 @@
   <component name="ProjectResources">
     <default-html-doctype>http://www.w3.org/1999/xhtml</default-html-doctype>
   </component>
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_5" default="false" assert-keyword="true" jdk-15="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_6" default="false" assert-keyword="true" jdk-15="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
     <output url="file://$PROJECT_DIR$/out" />
   </component>
 </project>
\ No newline at end of file
--- a/agent/src/jetbrains/buildServer/symbols/SymbolsIndexer.java	Mon May 16 15:03:56 2016 +0300
+++ b/agent/src/jetbrains/buildServer/symbols/SymbolsIndexer.java	Mon May 16 19:22:22 2016 +0300
@@ -9,16 +9,16 @@
 import jetbrains.buildServer.util.EventDispatcher;
 import jetbrains.buildServer.util.FileUtil;
 import org.apache.log4j.Logger;
+import org.jdom.JDOMException;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * @author Evgeniy.Koshkin
@@ -33,7 +33,7 @@
 
   @NotNull private final ArtifactsWatcher myArtifactsWatcher;
   @NotNull private final JetSymbolsExe myJetSymbolsExe;
-  @NotNull private final Collection<File> mySymbolsToProcess = new CopyOnWriteArrayList<File>();
+  @NotNull private final Map<File, String> myFileToArtifactMapToProcess = new ConcurrentHashMap<File, String>();
 
   @Nullable private BuildProgressLogger myProgressLogger;
   @Nullable private File myBuildTempDirectory;
@@ -73,26 +73,31 @@
       public void afterAtrifactsPublished(@NotNull AgentRunningBuild build, @NotNull BuildFinishedStatus buildStatus) {
         super.afterAtrifactsPublished(build, buildStatus);
         if(!isIndexingApplicable()) return;
-        if (mySymbolsToProcess.isEmpty()) {
+        if (myFileToArtifactMapToProcess.isEmpty()) {
           myProgressLogger.warning("Symbols weren't found in artifacts to be published.");
           LOG.debug("Symbols weren't found in artifacts to be published for build with id " + build.getBuildId());
         } else {
           myProgressLogger.message("Collecting symbol files signatures.");
           LOG.debug("Collecting symbol files signatures.");
           try {
-            final File symbolSignaturesFile = FileUtil.createTempFile(myBuildTempDirectory, "symbol-signatures-", ".xml", false);
-            myJetSymbolsExe.dumpGuidsToFile(mySymbolsToProcess, symbolSignaturesFile, myProgressLogger);
-            if(symbolSignaturesFile.exists()){
+            final Set<PdbSignatureIndexEntry> signatureLocalFilesData = getSignatures(myFileToArtifactMapToProcess.keySet());
+            if(!signatureLocalFilesData.isEmpty()){
+              final Set<PdbSignatureIndexEntry> indexData = new HashSet<PdbSignatureIndexEntry>();
+              for(PdbSignatureIndexEntry signatureIndexEntry : signatureLocalFilesData){
+                indexData.add(new PdbSignatureIndexEntry(signatureIndexEntry.getGuid(), myFileToArtifactMapToProcess.get(new File(signatureIndexEntry.getArtifactPath()))));
+              }
+              final File indexDataFile = FileUtil.createTempFile(myBuildTempDirectory, SymbolsConstants.SYMBOL_SIGNATURES_FILE_NAME_PREFIX, ".xml", false);
+              PdbSignatureIndexUtil.write(new FileOutputStream(indexDataFile), indexData);
               myProgressLogger.message("Publishing collected symbol files signatures.");
-              myArtifactsWatcher.addNewArtifactsPath(symbolSignaturesFile + "=>" + ".teamcity/symbols");
+              myArtifactsWatcher.addNewArtifactsPath(indexDataFile + "=>" + ".teamcity/symbols");
             }
-          } catch (IOException e) {
+          } catch (Exception e) {
             LOG.error("Error while dumping symbols/binaries signatures for build with id " + build.getBuildId(), e);
             myProgressLogger.error("Error while dumping symbols/binaries signatures.");
             myProgressLogger.exception(e);
           }
         }
-        mySymbolsToProcess.clear();
+        myFileToArtifactMapToProcess.clear();
       }
     });
   }
@@ -106,19 +111,19 @@
     }
 
     LOG.debug("Searching for symbol files in publishing artifacts.");
-    final Collection<File> pdbFiles = getArtifactPathsByFileExtension(artifacts, PDB_FILE_EXTENSION);
+    final Map<File, String> pdbFiles = getArtifactPathsByFileExtension(artifacts, PDB_FILE_EXTENSION);
     if(pdbFiles.isEmpty()) return;
 
     final PdbFilePatcher pdbFilePatcher = new PdbFilePatcher(myBuildTempDirectory, mySrcSrvHomeDir, new SrcSrvStreamBuilder(myFileUrlProvider, myProgressLogger));
-    for(File pdbFile : pdbFiles){
-      if(mySymbolsToProcess.contains(pdbFile)){
+    for(File pdbFile : pdbFiles.keySet()){
+      if(myFileToArtifactMapToProcess.containsKey(pdbFile)){
         LOG.debug(String.format("File %s already processed. Skipped.", pdbFile.getAbsolutePath()));
         continue;
       }
       try {
         myProgressLogger.message("Indexing sources appeared in file " + pdbFile.getAbsolutePath());
         pdbFilePatcher.patch(pdbFile, myProgressLogger);
-        mySymbolsToProcess.add(pdbFile);
+        myFileToArtifactMapToProcess.put(pdbFile, pdbFiles.get(pdbFile));
       } catch (Throwable e) {
         LOG.error("Error occurred while patching symbols file " + pdbFile, e);
         myProgressLogger.error("Error occurred while patching symbols file " + pdbFile);
@@ -127,6 +132,18 @@
     }
   }
 
+  private Set<PdbSignatureIndexEntry> getSignatures(Collection<File> files) throws IOException, JDOMException {
+    final File guidDumpFile = FileUtil.createTempFile(myBuildTempDirectory, "symbol-signatures-local-", ".xml", false);
+    myJetSymbolsExe.dumpGuidsToFile(files, guidDumpFile, myProgressLogger);
+    if(guidDumpFile.exists()){
+      myArtifactsWatcher.addNewArtifactsPath(guidDumpFile + "=>" + ".teamcity/symbols");
+    }
+    if(guidDumpFile.isFile())
+      return PdbSignatureIndexUtil.read(new FileInputStream(guidDumpFile));
+    else
+      return Collections.emptySet();
+  }
+
   @Nullable
   private File getSrcSrvHomeDir(@NotNull AgentRunningBuild build) {
     final Map<String,String> agentConfigParameters = build.getAgentConfiguration().getConfigurationParameters();
@@ -143,13 +160,14 @@
     return null;
   }
 
-  private Collection<File> getArtifactPathsByFileExtension(List<ArtifactsCollection> artifactsCollections, String fileExtension){
-    final Collection<File> result = new HashSet<File>();
+  private Map<File, String> getArtifactPathsByFileExtension(List<ArtifactsCollection> artifactsCollections, String fileExtension){
+    final Map<File, String> result = new HashMap<File, String>();
     for(ArtifactsCollection artifactsCollection : artifactsCollections){
       if(artifactsCollection.isEmpty()) continue;
-      for (File artifact : artifactsCollection.getFilePathMap().keySet()){
+      final Map<File, String> filePathMap = artifactsCollection.getFilePathMap();
+      for (final File artifact : filePathMap.keySet()){
         if(FileUtil.getExtension(artifact.getPath()).equalsIgnoreCase(fileExtension))
-          result.add(artifact);
+          result.put(artifact, filePathMap.get(artifact));
       }
     }
     return result;
--- a/common/common.iml	Mon May 16 15:03:56 2016 +0300
+++ b/common/common.iml	Mon May 16 19:22:22 2016 +0300
@@ -8,8 +8,7 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="library" name="idea annotations.jar" level="project" />
-    <orderEntry type="library" name="Idea-OpenApi" level="project" />
+    <orderEntry type="library" name="jdom" level="project" />
     <orderEntry type="library" name="Common-Api" level="project" />
   </component>
-</module>
-
+</module>
\ No newline at end of file
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/src/jetbrains/buildServer/symbols/PdbSignatureIndexEntry.java	Mon May 16 19:22:22 2016 +0300
@@ -0,0 +1,26 @@
+package jetbrains.buildServer.symbols;
+
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Created by Evgeniy.Koshkin.
+ */
+public class PdbSignatureIndexEntry {
+    private final String myGuid;
+    private final String myArtifactPath;
+
+    public PdbSignatureIndexEntry(@NotNull String guid, @NotNull String artifactPath) {
+        myGuid = guid;
+        myArtifactPath = artifactPath;
+    }
+
+    @NotNull
+    public String getGuid() {
+        return myGuid;
+    }
+
+    @NotNull
+    public String getArtifactPath() {
+        return myArtifactPath;
+    }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/common/src/jetbrains/buildServer/symbols/PdbSignatureIndexUtil.java	Mon May 16 19:22:22 2016 +0300
@@ -0,0 +1,50 @@
+package jetbrains.buildServer.symbols;
+
+import jetbrains.buildServer.util.XmlUtil;
+import org.jdom.Document;
+import org.jdom.Element;
+import org.jdom.JDOMException;
+import org.jdom.input.SAXBuilder;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.*;
+
+/**
+ * Created by Evgeniy.Koshkin.
+ */
+class PdbSignatureIndexUtil {
+    private static final String SIGN = "sign";
+    private static final String FILE = "file";
+    private static final String FILE_SIGNS = "file-signs";
+    private static final String FILE_SIGN_ENTRY = "file-sign-entry";
+
+    @NotNull
+    static Set<PdbSignatureIndexEntry> read(@NotNull final InputStream inputStream) throws JDOMException, IOException {
+        final SAXBuilder builder = new SAXBuilder();
+        final Document document = builder.build(inputStream);
+        final Set<PdbSignatureIndexEntry> result = new HashSet<PdbSignatureIndexEntry>();
+        for (Object signElementObject : document.getRootElement().getChildren()){
+            final Element signElement = (Element) signElementObject;
+            result.add(new PdbSignatureIndexEntry(extractGuid(signElement.getAttributeValue(SIGN)), signElement.getAttributeValue(FILE)));
+        }
+        return result;
+    }
+
+    static void write(@NotNull final OutputStream outputStream, @NotNull final Set<PdbSignatureIndexEntry> indexData) throws IOException {
+        final Element root = new Element(FILE_SIGNS);
+        for (final PdbSignatureIndexEntry indexEntry : indexData){
+            final Element entry = new Element(FILE_SIGN_ENTRY);
+            entry.setAttribute(SIGN, indexEntry.getGuid());
+            entry.setAttribute(FILE, indexEntry.getArtifactPath());
+            root.addContent(root);
+        }
+        XmlUtil.saveDocument(new Document(root), outputStream);
+    }
+
+    private static String extractGuid(String sign) {
+        return sign.substring(0, sign.length() - 1).toLowerCase(); //last symbol is PEDebugType
+    }
+}
--- a/common/src/jetbrains/buildServer/symbols/SymbolsConstants.java	Mon May 16 15:03:56 2016 +0300
+++ b/common/src/jetbrains/buildServer/symbols/SymbolsConstants.java	Mon May 16 19:22:22 2016 +0300
@@ -11,4 +11,6 @@
 
   public static final String APP_SYMBOLS = "/app/symbols/";
   public static final String APP_SOURCES = "/app/sources/";
+
+  public static final String SYMBOL_SIGNATURES_FILE_NAME_PREFIX = "symbol-signatures-artifacts-";
 }
--- a/server/src/jetbrains/buildServer/symbols/BuildSymbolsIndexProvider.java	Mon May 16 15:03:56 2016 +0300
+++ b/server/src/jetbrains/buildServer/symbols/BuildSymbolsIndexProvider.java	Mon May 16 19:22:22 2016 +0300
@@ -9,17 +9,14 @@
 import jetbrains.buildServer.util.CollectionsUtil;
 import jetbrains.buildServer.util.filters.Filter;
 import org.apache.log4j.Logger;
-import org.jdom.Document;
-import org.jdom.Element;
 import org.jdom.JDOMException;
-import org.jdom.input.SAXBuilder;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
@@ -44,7 +41,7 @@
     final BuildArtifact symbols = sBuild.getArtifacts(BuildArtifactsViewMode.VIEW_HIDDEN_ONLY).getArtifact(".teamcity/symbols");
     final BuildArtifact symbolSignaturesSource = symbols == null ? null : CollectionsUtil.findFirst(symbols.getChildren(), new Filter<BuildArtifact>() {
       public boolean accept(@NotNull BuildArtifact data) {
-        return data.getName().startsWith("symbol-signatures");
+        return data.getName().startsWith(SymbolsConstants.SYMBOL_SIGNATURES_FILE_NAME_PREFIX);
       }
     });
     if(symbolSignaturesSource == null) {
@@ -52,54 +49,44 @@
       return;
     }
 
-    Map<String, String> indexData = Collections.emptyMap();
+    Set<PdbSignatureIndexEntry> indexEntries = Collections.emptySet();
     try {
-      indexData = readIndex(symbolSignaturesSource.getInputStream());
+      indexEntries = PdbSignatureIndexUtil.read(symbolSignaturesSource.getInputStream());
     } catch (IOException e) {
       LOG.debug("Failed to read symbols index data from artifact " + symbolSignaturesSource.getRelativePath(), e);
     } catch (JDOMException e) {
       LOG.debug("Failed to read symbols index data from artifact " + symbolSignaturesSource.getRelativePath(), e);
     }
 
-    LOG.debug(String.format("Build with id %d provides %d symbol file signatures.", buildId, indexData.size()));
+    LOG.debug(String.format("Build with id %d provides %d symbol file signatures.", buildId, indexEntries.size()));
 
-    for (String signature : indexData.keySet()) {
-      final String fileName = indexData.get(signature);
-      final String artifactPath = locateArtifact(sBuild, fileName);
+    for (final PdbSignatureIndexEntry indexEntry : indexEntries) {
+      final String signature = indexEntry.getGuid();
+      final String artifactPathOrName = indexEntry.getArtifactPath();
+      final String artifactPath = locateArtifact(sBuild, artifactPathOrName);
       if (artifactPath == null) {
-        LOG.debug(String.format("Failed to find artifact by name %s and build id %d.", fileName, buildId));
+        LOG.debug(String.format("Failed to find artifact by name %s and build id %d.", artifactPathOrName, buildId));
         continue;
       }
       final HashMap<String, String> data = new HashMap<String, String>();
       data.put(ARTIFACT_PATH_KEY, artifactPath);
-      data.put(FILE_NAME_KEY, fileName);
+      data.put(FILE_NAME_KEY, artifactPathOrName);
       metadataStorageWriter.addParameters(signature, data);
-      LOG.debug("Stored symbol file signature " + signature + " for file name " + fileName + " build id " + buildId);
+      LOG.debug("Stored symbol file signature " + signature + " for file name " + artifactPathOrName + " build id " + buildId);
     }
   }
 
-  @NotNull
-  private Map<String, String> readIndex(InputStream inputStream) throws JDOMException, IOException {
-    SAXBuilder builder = new SAXBuilder();
-    Document document = builder.build(inputStream);
-    Map<String, String> result = new HashMap<String, String>();
-    for (Object signElementObject : document.getRootElement().getChildren()){
-      final Element signElement = (Element) signElementObject;
-      result.put(extractGuid(signElement.getAttributeValue("sign")), signElement.getAttributeValue("file"));
-    }
-    return result;
-  }
+  @Nullable
+  private String locateArtifact(@NotNull SBuild build, final @NotNull String artifactPathOrName) {
+    final BuildArtifacts artifacts = build.getArtifacts(BuildArtifactsViewMode.VIEW_DEFAULT_WITH_ARCHIVES_CONTENT);
+    final BuildArtifact artifact = artifacts.getArtifact(artifactPathOrName);
+    if(artifact != null) return artifactPathOrName;
 
-  private String extractGuid(String sign) {
-    return sign.substring(0, sign.length() - 1).toLowerCase(); //last symbol is PEDebugType
-  }
-
-  private String locateArtifact(SBuild build, final String fileName) {
     final AtomicReference<String> locatedArtifactPath = new AtomicReference<String>(null);
-    build.getArtifacts(BuildArtifactsViewMode.VIEW_DEFAULT_WITH_ARCHIVES_CONTENT).iterateArtifacts(new BuildArtifacts.BuildArtifactsProcessor() {
+    artifacts.iterateArtifacts(new BuildArtifacts.BuildArtifactsProcessor() {
       @NotNull
       public Continuation processBuildArtifact(@NotNull BuildArtifact artifact) {
-        if(artifact.getName().equals(fileName)){
+        if(artifact.getName().equals(artifactPathOrName)){
           locatedArtifactPath.set(artifact.getRelativePath());
           return Continuation.BREAK;
         }