Mercurial > hg > mercurial
view mercurial-server/src/jetbrains/buildServer/buildTriggers/vcs/mercurial/MercurialCollectChangesPolicy.java @ 1024:c0c4bf1db865
TW-66329 fix
using archive command for creating full patch: previous implementation using clone produces very large overhead for big project repositories in case when settings are stored in the project repo since in case of freezing build settings, core system asks plugin to build full patch and cloning repos into temp dir even from local mirror can take tens of minutes
author | Maxim Zaytsev <Maxim.Zaytsev@jetbrains.com> |
---|---|
date | Mon, 15 Jun 2020 13:07:12 +0300 |
parents | 7bf4d943d5bb |
children | 10dc26b32c35 |
line wrap: on
line source
/* * Copyright 2000-2018 JetBrains s.r.o. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package jetbrains.buildServer.buildTriggers.vcs.mercurial; import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.*; import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.exception.UnknownRevisionException; import jetbrains.buildServer.buildTriggers.vcs.mercurial.command.exception.WrongSubrepoUrlException; import jetbrains.buildServer.log.Loggers; import jetbrains.buildServer.vcs.*; import jetbrains.buildServer.vcs.impl.VcsRootImpl; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.*; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; public class MercurialCollectChangesPolicy implements CollectChangesBetweenRoots, CollectChangesBetweenRepositories { private final static AscendingRevNums ASCENDING_REV_NUMS = new AscendingRevNums(); private final VcsOperationProgressProvider myProgressProvider; private final MercurialVcsSupport myVcs; private final ServerPluginConfig myConfig; private final HgVcsRootFactory myHgVcsRootFactory; private final RepoFactory myRepoFactory; public MercurialCollectChangesPolicy(@NotNull VcsOperationProgressProvider progressProvider, @NotNull MercurialVcsSupport vcs, @NotNull ServerPluginConfig config, @NotNull HgVcsRootFactory hgVcsRootFactory, @NotNull RepoFactory repoFactory) { myProgressProvider = progressProvider; myVcs = vcs; myConfig = config; myHgVcsRootFactory = hgVcsRootFactory; myRepoFactory = repoFactory; } @NotNull public RepositoryStateData getCurrentState(@NotNull VcsRoot root) throws VcsException { return getCurrentState(myHgVcsRootFactory.createHgRoot(root)); } @NotNull public RepositoryStateData getCurrentState(@NotNull final HgVcsRoot hgRoot) throws VcsException { final OperationContext context = new OperationContext(myVcs, myRepoFactory, createMercurialProgess()); VcsCallable<Map<String, String>> cmd = new VcsCallable<Map<String, String>>() { public Map<String, String> call() throws VcsException { return getHeads(hgRoot, context); } }; Map<String, String> revisions = context.syncRepository(hgRoot, cmd); String defaultBranchName = hgRoot.getBranchName(); if (revisions.get(defaultBranchName) == null && !hgRoot.isIgnoreMissingDefaultBranch()) { throw new VcsException("Cannot find revision of the default branch '" + defaultBranchName + "' of vcs root " + myVcs.describeVcsRoot(hgRoot)); } return RepositoryStateData.createVersionState(defaultBranchName, revisions); } @NotNull public Map<String, String> getHeads(@NotNull HgVcsRoot hgRoot) throws VcsException { OperationContext context = new OperationContext(myVcs, myRepoFactory, MercurialProgress.NO_OP); return getHeads(hgRoot, context); } @NotNull public Map<String, String> getHeads(@NotNull HgVcsRoot hgRoot, @NotNull OperationContext context) throws VcsException { boolean includeTags = myConfig.useTagsAsBranches() && hgRoot.useTagsAsBranches(); return context.createRepo(hgRoot).getBranchRevisions(myConfig.bookmarksEnabled(), includeTags); } @NotNull public List<ModificationData> collectChanges(@NotNull VcsRoot fromRoot, @NotNull RepositoryStateData fromState, @NotNull VcsRoot toRoot, @NotNull RepositoryStateData toState, @NotNull CheckoutRules rules) throws VcsException { return collectChanges(toRoot, fromState, toState, rules); } @NotNull public List<ModificationData> collectChanges(@NotNull VcsRoot root, @NotNull RepositoryStateData fromState, @NotNull RepositoryStateData toState, @NotNull CheckoutRules rules) throws VcsException { List<ModificationData> changes = new ArrayList<ModificationData>(); HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root); CollectChangesContext ctx = new CollectChangesContext(myConfig, myVcs, myRepoFactory, createMercurialProgess(), fromState); for (Map.Entry<String, String> entry : toState.getBranchRevisions().entrySet()) { String branch = entry.getKey(); String toRevision = entry.getValue(); String fromRevision = fromState.getBranchRevisions().get(branch); if (fromRevision == null) fromRevision = fromState.getBranchRevisions().get(fromState.getDefaultBranchName()); if (toRevision.equals(fromRevision) || fromRevision == null) continue; Collection<String> fromRevisions = ctx.getFromRevisionsForBranch(hgRoot, fromRevision, toRevision, toState); List<ModificationData> branchChanges = collectChanges(ctx, root, fromRevisions, toRevision, rules); for (ModificationData change : branchChanges) { if (!ctx.isReportedModification(change)) { changes.add(change); ctx.markAsReported(change); } } } changes.addAll(getSubrepoChanges(ctx, root, changes)); return changes; } @NotNull public List<ModificationData> collectChanges(@NotNull VcsRoot fromRoot, @NotNull String fromRootRevision, @NotNull VcsRoot toRoot, @Nullable String toRootRevision, @NotNull CheckoutRules checkoutRules) throws VcsException { HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(toRoot); CollectChangesContext context = new CollectChangesContext(myConfig, myVcs, myRepoFactory, createMercurialProgess(), fromRootRevision); context.syncRepository(hgRoot); String toRevision = toRootRevision; if (toRevision == null) { RepositoryStateData state = myVcs.getCollectChangesPolicy().getCurrentState(toRoot); toRevision = state.getBranchRevisions().get(state.getDefaultBranchName()); } String mergeBase = getMergeBase(context, hgRoot, fromRootRevision, toRevision); if (mergeBase == null) return Collections.emptyList(); return collectChanges(context, toRoot, mergeBase, toRootRevision, checkoutRules); } public List<ModificationData> collectChanges(@NotNull VcsRoot root, @NotNull String fromVersion, @Nullable String currentVersion, @NotNull CheckoutRules checkoutRules) throws VcsException { if (currentVersion == null) return emptyList(); CollectChangesContext ctx = new CollectChangesContext(myConfig, myVcs, myRepoFactory, createMercurialProgess(), fromVersion); return collectChanges(ctx, root, fromVersion, currentVersion, checkoutRules); } private List<ModificationData> collectChanges(@NotNull CollectChangesContext ctx, @NotNull VcsRoot root, @NotNull String fromVersion, @Nullable String currentVersion, @NotNull CheckoutRules checkoutRules) throws VcsException { List<ModificationData> changes = collectChanges(ctx, root, asList(fromVersion), currentVersion, checkoutRules); changes.addAll(getSubrepoChanges(ctx, root, changes)); return changes; } @Nullable private String getMergeBase(@NotNull OperationContext context, @NotNull HgVcsRoot root, @NotNull String revision1, @NotNull String revision2) throws VcsException { String result = context.createRepo(root).mergeBase() .revision1(revision1) .revision2(revision2) .call(); if (result == null) result = getMinusNthCommit(context, root, 10); return result; } @Nullable private String getMinusNthCommit(@NotNull OperationContext context, @NotNull HgVcsRoot root, int n) throws VcsException { LogCommand log = context.createRepo(root).log() .inBranch(root.getBranchName()) .toNamedRevision(root.getBranchName()); if (n > 0) log.setLimit(n); List<ChangeSet> changeSets = log.call(); if (changeSets.isEmpty()) return null; return changeSets.get(0).getId(); } @NotNull private List<ModificationData> collectChanges(@NotNull CollectChangesContext ctx, @NotNull VcsRoot root, @NotNull Collection<String> fromVersion, @Nullable String currentVersion, @NotNull CheckoutRules checkoutRules) throws VcsException { HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root); ctx.syncRepository(hgRoot); List<ModificationData> result = new ArrayList<ModificationData>(); List<ChangeSet> csets = getChangesets(ctx, hgRoot, fromVersion, currentVersion); //When commit has no changes in subrepo configuration we can reuse //subrepo revision table calculated for its parent commit(s). To do //that parents should be processed before children: Collections.sort(csets, ASCENDING_REV_NUMS); for (ChangeSet cset : csets) { result.add(createModificationData(ctx, cset, root, hgRoot, checkoutRules)); } return result; } @NotNull private List<ChangeSet> getChangesets(@NotNull CollectChangesContext ctx, @NotNull final HgVcsRoot root, @NotNull final Collection<String> fromVersions, @Nullable final String toVersion) throws VcsException { if (toVersion == null) return emptyList(); List<String> fromCommits = new ArrayList<String>(); for (String fromVersion : fromVersions) { fromCommits.add(new ChangeSetRevision(fromVersion).getId()); } String toCommit = new ChangeSetRevision(toVersion).getId(); try { ServerHgRepo repo = ctx.createRepo(root); List<ChangeSet> csets = repo.collectChanges(root) .fromRevision(fromCommits) .toRevision(toCommit) .includeFromRevision(ctx.includeFromRevisions()) .call(); if (!ctx.includeFromRevisions()) { Iterator<ChangeSet> iter = csets.iterator(); while (iter.hasNext()) { ChangeSet cset = iter.next(); if (fromVersions.contains(cset.getId())) iter.remove(); } } return csets; } catch (UnknownRevisionException e) { Loggers.VCS.warn("Revision '" + e.getRevision() + "' is unknown, will return no changes"); return emptyList(); } } private ModificationData createModificationData(@NotNull final CollectChangesContext ctx, @NotNull final ChangeSet cset, @NotNull final VcsRoot root, @NotNull final HgVcsRoot hgRoot, @NotNull final CheckoutRules checkoutRules) throws VcsException { final ModificationData result = ModificationDataFactory.createModificationData(ctx, cset, root, checkoutRules); result.setAttributes(getAttributes(ctx, root, hgRoot, result)); return result; } @NotNull private List<ModificationData> getSubrepoChanges(@NotNull CollectChangesContext ctx, @NotNull VcsRoot root, @NotNull List<ModificationData> changes) throws VcsException { if (changes.isEmpty()) return emptyList(); ctx.setIncludeFromRevisions(true); HgVcsRoot hgRoot = myHgVcsRootFactory.createHgRoot(root); if (!detectSubrepoChanges(hgRoot)) return emptyList(); List<HgSubrepoConfigChange> subrepoConfigChanges = getSubrepoConfigChanges(ctx, changes); List<ModificationData> subrepoChanges = new ArrayList<ModificationData>(); for (HgSubrepoConfigChange configChange : subrepoConfigChanges) { SubRepo current = configChange.getCurrent(); assert current != null; String subrepoUrl = current.url(); String curRevision = current.revision(); List<String> prevRevisions = new ArrayList<String>(0); for (SubRepo prevSubrepo : configChange.getPrevious()) { prevRevisions.add(prevSubrepo.revision()); } String path = configChange.getPath(); Map<String, String> subrepoParams = new HashMap<String, String>(hgRoot.getProperties()); subrepoParams.put(Constants.REPOSITORY_PROP, subrepoUrl); subrepoParams.put("teamcity.internal.subrepo", "true"); subrepoParams.put("teamcity.internal.subrepo.path", path); VcsRootImpl subrepo = new VcsRootImpl(root.getId(), subrepoParams); if (ctx.isProcessedSubrepoChanges(subrepo, prevRevisions, curRevision)) continue; try { List<ModificationData> subChanges = collectChanges(ctx, subrepo, prevRevisions, curRevision, CheckoutRules.DEFAULT); for (ModificationData m : subChanges) { if (!ctx.isReportedModification(m)) { subrepoChanges.add(m); ctx.markAsReported(m); } } ctx.markProcessedSubrepoChanges(subrepo, prevRevisions, curRevision); } catch (VcsException e) { Loggers.VCS.warn("Error while collecting subrepo changes, repository: " + hgRoot.getRepository() + ", revision: " + configChange.getMainRepoRevision() + ", subrepo: " + subrepoUrl + " at " + configChange.getPath() + ", subrepo revisions interval: [" + prevRevisions + ", " + curRevision + "] "+ "skip collecting subrepo changes in this interval", e); } } List<ModificationData> subSubrepoChanges = getSubrepoChanges(ctx, root, subrepoChanges); subrepoChanges.addAll(subSubrepoChanges); return subrepoChanges; } private boolean detectSubrepoChanges(@NotNull HgVcsRoot root) { return myConfig.detectSubrepoChanges() && root.detectSubrepoChanges(); } private List<HgSubrepoConfigChange> getSubrepoConfigChanges(@NotNull CollectChangesContext ctx, @NotNull List<ModificationData> mainRootChanges) throws VcsException { List<HgSubrepoConfigChange> subrepoConfigChanges = new ArrayList<HgSubrepoConfigChange>(); for (ModificationData m : mainRootChanges) { subrepoConfigChanges.addAll(getSubrepoConfigChanges(ctx, m)); } return subrepoConfigChanges; } @NotNull private List<HgSubrepoConfigChange> getSubrepoConfigChanges(@NotNull CollectChangesContext ctx, @NotNull ModificationData m) throws VcsException { List<HgSubrepoConfigChange> configChanges = new ArrayList<HgSubrepoConfigChange>(); HgVcsRoot mainRoot = myHgVcsRootFactory.createHgRoot(m.getVcsRoot()); ServerHgRepo repo = ctx.createRepo(mainRoot); for (HgSubrepoConfigChange c : repo.getSubrepoConfigChanges(m)) { if (!(c.subrepoUrlChanged() || c.subrepoAdded() || c.subrepoRemoved())) {//report only changes in revisions, because we collect changes only for such changes //map url and path, relative to the main repository try { SubRepo currentSubrepo = c.getCurrent(); assert currentSubrepo != null; String subrepoUrl = ctx.getStringFromPool(currentSubrepo.resolveUrl(mainRoot.getRepository())); String curRevision = ctx.getStringFromPool(currentSubrepo.revision()); List<SubRepo> prevSubrepos = new ArrayList<SubRepo>(0); String path = ctx.getStringFromPool(mainRoot.expandSubrepoPath(c.getPath())); for (SubRepo prevSubrepo : c.getPrevious()) { prevSubrepos.add(new SubRepo(path, subrepoUrl, ctx.getStringFromPool(prevSubrepo.revision()))); } configChanges.add(new HgSubrepoConfigChange(m.getVersion(), path, prevSubrepos, new SubRepo(path, subrepoUrl, curRevision))); } catch (WrongSubrepoUrlException e) { Loggers.VCS.warn("Error while collecting subrepo config changes, repository: " + mainRoot.getRepository() +", commit: " + m.getVersion() + ", skip subrepo changes in this commit", e); } } } return configChanges; } @NotNull private Map<String, String> getAttributes(@NotNull CollectChangesContext ctx, @NotNull VcsRoot mainRoot, @NotNull HgVcsRoot mainHgRoot, @NotNull ModificationData m) throws VcsException { Map<String, String> attributes = new HashMap<String, String>(); if (detectSubrepoChanges(mainHgRoot)) { try { ServerHgRepo repo = ctx.createRepo(mainHgRoot); SubrepoRevisionAttributesBuilder attrBuilder = new SubrepoRevisionAttributesBuilder(); for (SubRepo s : repo.getSubrepositories(m).values()) { attrBuilder.addSubrepo(new SubrepoConfig(mainRoot) .setSubrepoPath(ctx.getStringFromPool(mainHgRoot.expandSubrepoPath(s.path()))) .setSubrepoRootParamDiff(Constants.REPOSITORY_PROP, ctx.getStringFromPool(s.resolveUrl(mainHgRoot.getRepository()))) .setSubrepoRootParamDiff("teamcity.internal.subrepo", "true") .setSubrepoRootParamDiff("teamcity.internal.subrepo.path", ctx.getStringFromPool(mainHgRoot.expandSubrepoPath(s.path()))) .setSubrepoRevision(ctx.getStringFromPool(s.revision()))); } attributes.putAll(attrBuilder.buildAttributes()); } catch (Exception e) { Loggers.VCS.warn("Error while reporting subrepo config changes", e); if (e instanceof VcsException) throw (VcsException) e; throw new VcsException(e); } } if (attributes.isEmpty()) return emptyMap(); return attributes; } private final static class AscendingRevNums implements Comparator<ChangeSet> { public int compare(ChangeSet o1, ChangeSet o2) { int revnum1 = o1.getRevNumber(); int revnum2 = o2.getRevNumber(); if (revnum1 < revnum2) return -1; if (revnum1 > revnum2) return 1; return 0; } } @NotNull private MercurialProgress createMercurialProgess() { try { final VcsOperationProgress progress = myProgressProvider.getProgress(); return new MercurialVcsOperationProgress(progress); } catch (IllegalStateException e) { return MercurialProgress.NO_OP; } } }