BasePackFetchConnection.java

  1. /*
  2.  * Copyright (C) 2008, 2010 Google Inc.
  3.  * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
  4.  * Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others
  5.  *
  6.  * This program and the accompanying materials are made available under the
  7.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  8.  * https://www.eclipse.org/org/documents/edl-v10.php.
  9.  *
  10.  * SPDX-License-Identifier: BSD-3-Clause
  11.  */

  13. package org.eclipse.jgit.transport;

  15. import java.io.IOException;
  16. import java.io.InputStream;
  17. import java.io.OutputStream;
  18. import java.text.MessageFormat;
  19. import java.util.Arrays;
  20. import java.util.Collection;
  21. import java.util.Collections;
  22. import java.util.Date;
  23. import java.util.HashSet;
  24. import java.util.LinkedHashSet;
  25. import java.util.Set;

  27. import org.eclipse.jgit.errors.PackProtocolException;
  28. import org.eclipse.jgit.errors.RemoteRepositoryException;
  29. import org.eclipse.jgit.errors.TransportException;
  30. import org.eclipse.jgit.internal.JGitText;
  31. import org.eclipse.jgit.internal.storage.file.PackLock;
  32. import org.eclipse.jgit.lib.AnyObjectId;
  33. import org.eclipse.jgit.lib.Config;
  34. import org.eclipse.jgit.lib.MutableObjectId;
  35. import org.eclipse.jgit.lib.NullProgressMonitor;
  36. import org.eclipse.jgit.lib.ObjectId;
  37. import org.eclipse.jgit.lib.ObjectInserter;
  38. import org.eclipse.jgit.lib.ProgressMonitor;
  39. import org.eclipse.jgit.lib.Ref;
  40. import org.eclipse.jgit.revwalk.RevCommit;
  41. import org.eclipse.jgit.revwalk.RevCommitList;
  42. import org.eclipse.jgit.revwalk.RevFlag;
  43. import org.eclipse.jgit.revwalk.RevObject;
  44. import org.eclipse.jgit.revwalk.RevSort;
  45. import org.eclipse.jgit.revwalk.RevWalk;
  46. import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter;
  47. import org.eclipse.jgit.revwalk.filter.RevFilter;
  48. import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck;
  49. import org.eclipse.jgit.transport.PacketLineIn.AckNackResult;
  50. import org.eclipse.jgit.util.StringUtils;
  51. import org.eclipse.jgit.util.TemporaryBuffer;

  53. /**
  54.  * Fetch implementation using the native Git pack transfer service.
  55.  * <p>
  56.  * This is the canonical implementation for transferring objects from the remote
  57.  * repository to the local repository by talking to the 'git-upload-pack'
  58.  * service. Objects are packed on the remote side into a pack file and then sent
  59.  * down the pipe to us.
  60.  * <p>
  61.  * This connection requires only a bi-directional pipe or socket, and thus is
  62.  * easily wrapped up into a local process pipe, anonymous TCP socket, or a
  63.  * command executed through an SSH tunnel.
  64.  * <p>
  65.  * If {@link org.eclipse.jgit.transport.BasePackConnection#statelessRPC} is
  66.  * {@code true}, this connection can be tunneled over a request-response style
  67.  * RPC system like HTTP. The RPC call boundary is determined by this class
  68.  * switching from writing to the OutputStream to reading from the InputStream.
  69.  * <p>
  70.  * Concrete implementations should just call
  71.  * {@link #init(java.io.InputStream, java.io.OutputStream)} and
  72.  * {@link #readAdvertisedRefs()} methods in constructor or before any use. They
  73.  * should also handle resources releasing in {@link #close()} method if needed.
  74.  */
  75. public abstract class BasePackFetchConnection extends BasePackConnection
  76.         implements FetchConnection {
  77.     /**
  78.      * Maximum number of 'have' lines to send before giving up.
  79.      * <p>
  80.      * During {@link #negotiate(ProgressMonitor)} we send at most this many
  81.      * commits to the remote peer as 'have' lines without an ACK response before
  82.      * we give up.
  83.      */
  84.     private static final int MAX_HAVES = 256;

  86.     /**
  87.      * Amount of data the client sends before starting to read.
  88.      * <p>
  89.      * Any output stream given to the client must be able to buffer this many
  90.      * bytes before the client will stop writing and start reading from the
  91.      * input stream. If the output stream blocks before this many bytes are in
  92.      * the send queue, the system will deadlock.
  93.      */
  94.     protected static final int MIN_CLIENT_BUFFER = 2 * 32 * 46 + 8;

  96.     /**
  97.      * Include tags if we are also including the referenced objects.
  98.      * @since 2.0
  99.      */
  100.     public static final String OPTION_INCLUDE_TAG = GitProtocolConstants.OPTION_INCLUDE_TAG;

  102.     /**
  103.      * Multi-ACK support for improved negotiation.
  104.      * @since 2.0
  105.      */
  106.     public static final String OPTION_MULTI_ACK = GitProtocolConstants.OPTION_MULTI_ACK;

  108.     /**
  109.      * Multi-ACK detailed support for improved negotiation.
  110.      * @since 2.0
  111.      */
  112.     public static final String OPTION_MULTI_ACK_DETAILED = GitProtocolConstants.OPTION_MULTI_ACK_DETAILED;

  114.     /**
  115.      * The client supports packs with deltas but not their bases.
  116.      * @since 2.0
  117.      */
  118.     public static final String OPTION_THIN_PACK = GitProtocolConstants.OPTION_THIN_PACK;

  120.     /**
  121.      * The client supports using the side-band for progress messages.
  122.      * @since 2.0
  123.      */
  124.     public static final String OPTION_SIDE_BAND = GitProtocolConstants.OPTION_SIDE_BAND;

  126.     /**
  127.      * The client supports using the 64K side-band for progress messages.
  128.      * @since 2.0
  129.      */
  130.     public static final String OPTION_SIDE_BAND_64K = GitProtocolConstants.OPTION_SIDE_BAND_64K;

  132.     /**
  133.      * The client supports packs with OFS deltas.
  134.      * @since 2.0
  135.      */
  136.     public static final String OPTION_OFS_DELTA = GitProtocolConstants.OPTION_OFS_DELTA;

  138.     /**
  139.      * The client supports shallow fetches.
  140.      * @since 2.0
  141.      */
  142.     public static final String OPTION_SHALLOW = GitProtocolConstants.OPTION_SHALLOW;

  144.     /**
  145.      * The client does not want progress messages and will ignore them.
  146.      * @since 2.0
  147.      */
  148.     public static final String OPTION_NO_PROGRESS = GitProtocolConstants.OPTION_NO_PROGRESS;

  150.     /**
  151.      * The client supports receiving a pack before it has sent "done".
  152.      * @since 2.0
  153.      */
  154.     public static final String OPTION_NO_DONE = GitProtocolConstants.OPTION_NO_DONE;

  156.     /**
  157.      * The client supports fetching objects at the tip of any ref, even if not
  158.      * advertised.
  159.      * @since 3.1
  160.      */
  161.     public static final String OPTION_ALLOW_TIP_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;

  163.     /**
  164.      * The client supports fetching objects that are reachable from a tip of a
  165.      * ref that is allowed to fetch.
  166.      * @since 4.1
  167.      */
  168.     public static final String OPTION_ALLOW_REACHABLE_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;

  170.     /**
  171.      * The client specified a filter expression.
  172.      *
  173.      * @since 5.0
  174.      */
  175.     public static final String OPTION_FILTER = GitProtocolConstants.OPTION_FILTER;

  177.     private final RevWalk walk;

  179.     /** All commits that are immediately reachable by a local ref. */
  180.     private RevCommitList<RevCommit> reachableCommits;

  182.     /** Marks an object as having all its dependencies. */
  183.     final RevFlag REACHABLE;

  185.     /** Marks a commit known to both sides of the connection. */
  186.     final RevFlag COMMON;

  188.     /** Like {@link #COMMON} but means its also in {@link #pckState}. */
  189.     private final RevFlag STATE;

  191.     /** Marks a commit listed in the advertised refs. */
  192.     final RevFlag ADVERTISED;

  194.     private MultiAck multiAck = MultiAck.OFF;

  196.     private boolean thinPack;

  198.     private boolean sideband;

  200.     private boolean includeTags;

  202.     private boolean allowOfsDelta;

  204.     private boolean noDone;

  206.     private boolean noProgress;

  208.     private String lockMessage;

  210.     private PackLock packLock;

  212.     private int maxHaves;

  214.     /**
  215.      * RPC state, if {@link BasePackConnection#statelessRPC} is true or protocol
  216.      * V2 is used.
  217.      */
  218.     private TemporaryBuffer.Heap state;

  220.     private PacketLineOut pckState;

  222.     /**
  223.      * Either FilterSpec.NO_FILTER for a filter that doesn't filter
  224.      * anything, or a filter that indicates what and what not to send to the
  225.      * server.
  226.      */
  227.     private final FilterSpec filterSpec;

  229.     /**
  230.      * Create a new connection to fetch using the native git transport.
  231.      *
  232.      * @param packTransport
  233.      *            the transport.
  234.      */
  235.     public BasePackFetchConnection(PackTransport packTransport) {
  236.         super(packTransport);

  238.         if (local != null) {
  239.             final FetchConfig cfg = getFetchConfig();
  240.             allowOfsDelta = cfg.allowOfsDelta;
  241.             maxHaves = cfg.maxHaves;
  242.         } else {
  243.             allowOfsDelta = true;
  244.             maxHaves = Integer.MAX_VALUE;
  245.         }

  247.         includeTags = transport.getTagOpt() != TagOpt.NO_TAGS;
  248.         thinPack = transport.isFetchThin();
  249.         filterSpec = transport.getFilterSpec();

  251.         if (local != null) {
  252.             walk = new RevWalk(local);
  253.             walk.setRetainBody(false);
  254.             reachableCommits = new RevCommitList<>();
  255.             REACHABLE = walk.newFlag("REACHABLE"); //$NON-NLS-1$
  256.             COMMON = walk.newFlag("COMMON"); //$NON-NLS-1$
  257.             STATE = walk.newFlag("STATE"); //$NON-NLS-1$
  258.             ADVERTISED = walk.newFlag("ADVERTISED"); //$NON-NLS-1$

  260.             walk.carry(COMMON);
  261.             walk.carry(REACHABLE);
  262.             walk.carry(ADVERTISED);
  263.         } else {
  264.             walk = null;
  265.             REACHABLE = null;
  266.             COMMON = null;
  267.             STATE = null;
  268.             ADVERTISED = null;
  269.         }
  270.     }

  272.     static class FetchConfig {
  273.         final boolean allowOfsDelta;

  275.         final int maxHaves;

  277.         FetchConfig(Config c) {
  278.             allowOfsDelta = c.getBoolean("repack", "usedeltabaseoffset", true); //$NON-NLS-1$ //$NON-NLS-2$
  279.             maxHaves = c.getInt("fetch", "maxhaves", Integer.MAX_VALUE); //$NON-NLS-1$ //$NON-NLS-2$
  280.         }

  282.         FetchConfig(boolean allowOfsDelta, int maxHaves) {
  283.             this.allowOfsDelta = allowOfsDelta;
  284.             this.maxHaves = maxHaves;
  285.         }
  286.     }

  288.     /** {@inheritDoc} */
  289.     @Override
  290.     public final void fetch(final ProgressMonitor monitor,
  291.             final Collection<Ref> want, final Set<ObjectId> have)
  292.             throws TransportException {
  293.         fetch(monitor, want, have, null);
  294.     }

  296.     /** {@inheritDoc} */
  297.     @Override
  298.     public final void fetch(final ProgressMonitor monitor,
  299.             final Collection<Ref> want, final Set<ObjectId> have,
  300.             OutputStream outputStream) throws TransportException {
  301.         markStartedOperation();
  302.         doFetch(monitor, want, have, outputStream);
  303.     }

  305.     /** {@inheritDoc} */
  306.     @Override
  307.     public boolean didFetchIncludeTags() {
  308.         return false;
  309.     }

  311.     /** {@inheritDoc} */
  312.     @Override
  313.     public boolean didFetchTestConnectivity() {
  314.         return false;
  315.     }

  317.     /** {@inheritDoc} */
  318.     @Override
  319.     public void setPackLockMessage(String message) {
  320.         lockMessage = message;
  321.     }

  323.     /** {@inheritDoc} */
  324.     @Override
  325.     public Collection<PackLock> getPackLocks() {
  326.         if (packLock != null)
  327.             return Collections.singleton(packLock);
  328.         return Collections.<PackLock> emptyList();
  329.     }

  331.     private void clearState() {
  332.         walk.dispose();
  333.         reachableCommits = null;
  334.         state = null;
  335.         pckState = null;
  336.     }

  338.     /**
  339.      * Execute common ancestor negotiation and fetch the objects.
  340.      *
  341.      * @param monitor
  342.      *            progress monitor to receive status updates. If the monitor is
  343.      *            the {@link org.eclipse.jgit.lib.NullProgressMonitor#INSTANCE}, then the no-progress
  344.      *            option enabled.
  345.      * @param want
  346.      *            the advertised remote references the caller wants to fetch.
  347.      * @param have
  348.      *            additional objects to assume that already exist locally. This
  349.      *            will be added to the set of objects reachable from the
  350.      *            destination repository's references.
  351.      * @param outputStream
  352.      *            ouputStream to write sideband messages to
  353.      * @throws org.eclipse.jgit.errors.TransportException
  354.      *             if any exception occurs.
  355.      * @since 3.0
  356.      */
  357.     protected void doFetch(final ProgressMonitor monitor,
  358.             final Collection<Ref> want, final Set<ObjectId> have,
  359.             OutputStream outputStream) throws TransportException {
  360.         try {
  361.             noProgress = monitor == NullProgressMonitor.INSTANCE;

  363.             markRefsAdvertised();
  364.             markReachable(have, maxTimeWanted(want));

  366.             if (TransferConfig.ProtocolVersion.V2
  367.                     .equals(getProtocolVersion())) {
  368.                 // Protocol V2 always is a "stateless" protocol, even over a
  369.                 // bidirectional pipe: the server serves one "fetch" request and
  370.                 // then forgets anything it has learned, so the next fetch
  371.                 // request has to re-send all wants and previously determined
  372.                 // common objects as "have"s again.
  373.                 state = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
  374.                 pckState = new PacketLineOut(state);
  375.                 try {
  376.                     doFetchV2(monitor, want, outputStream);
  377.                 } finally {
  378.                     clearState();
  379.                 }
  380.                 return;
  381.             }
  382.             // Protocol V0/1
  383.             if (statelessRPC) {
  384.                 state = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
  385.                 pckState = new PacketLineOut(state);
  386.             }
  387.             PacketLineOut output = statelessRPC ? pckState : pckOut;
  388.             if (sendWants(want, output)) {
  389.                 output.end();
  390.                 outNeedsEnd = false;
  391.                 negotiate(monitor);

  393.                 clearState();

  395.                 receivePack(monitor, outputStream);
  396.             }
  397.         } catch (CancelledException ce) {
  398.             close();
  399.             return; // Caller should test (or just know) this themselves.
  400.         } catch (IOException | RuntimeException err) {
  401.             close();
  402.             throw new TransportException(err.getMessage(), err);
  403.         }
  404.     }

  406.     private void doFetchV2(ProgressMonitor monitor, Collection<Ref> want,
  407.             OutputStream outputStream) throws IOException, CancelledException {
  408.         sideband = true;
  409.         negotiateBegin();

  411.         pckState.writeString("command=" + GitProtocolConstants.COMMAND_FETCH); //$NON-NLS-1$
  412.         // Capabilities are sent as command arguments in protocol V2
  413.         String agent = UserAgent.get();
  414.         if (agent != null && isCapableOf(GitProtocolConstants.OPTION_AGENT)) {
  415.             pckState.writeString(
  416.                     GitProtocolConstants.OPTION_AGENT + '=' + agent);
  417.         }
  418.         Set<String> capabilities = new HashSet<>();
  419.         String advertised = getCapability(GitProtocolConstants.COMMAND_FETCH);
  420.         if (!StringUtils.isEmptyOrNull(advertised)) {
  421.             capabilities.addAll(Arrays.asList(advertised.split("\\s+"))); //$NON-NLS-1$
  422.         }
  423.         // Arguments
  424.         pckState.writeDelim();
  425.         for (String capability : getCapabilitiesV2(capabilities)) {
  426.             pckState.writeString(capability);
  427.         }
  428.         if (!sendWants(want, pckState)) {
  429.             // We already have everything we wanted.
  430.             return;
  431.         }
  432.         // If we send something, we always close it properly ourselves.
  433.         outNeedsEnd = false;

  435.         FetchStateV2 fetchState = new FetchStateV2();
  436.         boolean sentDone = false;
  437.         for (;;) {
  438.             // The "state" buffer contains the full fetch request with all
  439.             // common objects found so far.
  440.             state.writeTo(out, monitor);
  441.             sentDone = sendNextHaveBatch(fetchState, pckOut, monitor);
  442.             if (sentDone) {
  443.                 break;
  444.             }
  445.             if (readAcknowledgments(fetchState, pckIn, monitor)) {
  446.                 // We got a "ready": next should be a patch file.
  447.                 break;
  448.             }
  449.             // Note: C git reads and requires here (and after a packfile) a
  450.             // "0002" packet in stateless RPC transports (https). This "response
  451.             // end" packet is even mentioned in the protocol V2 technical
  452.             // documentation. However, it is not actually part of the public
  453.             // protocol; it occurs only in an internal protocol wrapper in the C
  454.             // git implementation.
  455.         }
  456.         clearState();
  457.         String line = pckIn.readString();
  458.         // If we sent a done, we may have an error reply here.
  459.         if (sentDone && line.startsWith("ERR ")) { //$NON-NLS-1$
  460.             throw new RemoteRepositoryException(uri, line.substring(4));
  461.         }
  462.         // "shallow-info", "wanted-refs", and "packfile-uris" would have to be
  463.         // handled here in that order.
  464.         if (!GitProtocolConstants.SECTION_PACKFILE.equals(line)) {
  465.             throw new PackProtocolException(
  466.                     MessageFormat.format(JGitText.get().expectedGot,
  467.                             GitProtocolConstants.SECTION_PACKFILE, line));
  468.         }
  469.         receivePack(monitor, outputStream);
  470.     }

  472.     /**
  473.      * Sends the next batch of "have"s and terminates the {@code output}.
  474.      *
  475.      * @param fetchState
  476.      *            is updated with information about the number of items written,
  477.      *            and whether to expect a packfile next
  478.      * @param output
  479.      *            to write to
  480.      * @param monitor
  481.      *            for progress reporting and cancellation
  482.      * @return {@code true} if a "done" was written and we should thus expect a
  483.      *         packfile next
  484.      * @throws IOException
  485.      *             on errors
  486.      * @throws CancelledException
  487.      *             on cancellation
  488.      */
  489.     private boolean sendNextHaveBatch(FetchStateV2 fetchState,
  490.             PacketLineOut output, ProgressMonitor monitor)
  491.             throws IOException, CancelledException {
  492.         long n = 0;
  493.         while (n < fetchState.havesToSend) {
  494.             final RevCommit c = walk.next();
  495.             if (c == null) {
  496.                 break;
  497.             }
  498.             output.writeString("have " + c.getId().name() + '\n'); //$NON-NLS-1$
  499.             n++;
  500.             if (n % 10 == 0 && monitor.isCancelled()) {
  501.                 throw new CancelledException();
  502.             }
  503.         }
  504.         fetchState.havesTotal += n;
  505.         if (n == 0
  506.                 || (fetchState.hadAcks
  507.                         && fetchState.havesWithoutAck > MAX_HAVES)
  508.                 || fetchState.havesTotal > maxHaves) {
  509.             output.writeString("done\n"); //$NON-NLS-1$
  510.             output.end();
  511.             return true;
  512.         }
  513.         // Increment only after the test above. Of course we have no ACKs yet
  514.         // for the newly added "have"s, so it makes no sense to count them
  515.         // against the MAX_HAVES limit.
  516.         fetchState.havesWithoutAck += n;
  517.         output.end();
  518.         fetchState.incHavesToSend(statelessRPC);
  519.         return false;
  520.     }

  522.     /**
  523.      * Reads and processes acknowledgments, adding ACKed objects as "have"s to
  524.      * the global state {@link TemporaryBuffer}.
  525.      *
  526.      * @param fetchState
  527.      *            to update
  528.      * @param input
  529.      *            to read from
  530.      * @param monitor
  531.      *            for progress reporting and cancellation
  532.      * @return {@code true} if a "ready" was received and a packfile is expected
  533.      *         next
  534.      * @throws IOException
  535.      *             on errors
  536.      * @throws CancelledException
  537.      *             on cancellation
  538.      */
  539.     private boolean readAcknowledgments(FetchStateV2 fetchState,
  540.             PacketLineIn input, ProgressMonitor monitor)
  541.             throws IOException, CancelledException {
  542.         String line = input.readString();
  543.         if (!GitProtocolConstants.SECTION_ACKNOWLEDGMENTS.equals(line)) {
  544.             throw new PackProtocolException(MessageFormat.format(
  545.                     JGitText.get().expectedGot,
  546.                     GitProtocolConstants.SECTION_ACKNOWLEDGMENTS, line));
  547.         }
  548.         MutableObjectId returnedId = new MutableObjectId();
  549.         line = input.readString();
  550.         boolean gotReady = false;
  551.         long n = 0;
  552.         while (!PacketLineIn.isEnd(line) && !PacketLineIn.isDelimiter(line)) {
  553.             AckNackResult ack = PacketLineIn.parseACKv2(line, returnedId);
  554.             // If we got a "ready", we just skip the remaining lines after
  555.             // having checked them for being valid. (Normally, the "ready"
  556.             // should be the last line anyway.)
  557.             if (!gotReady) {
  558.                 if (ack == AckNackResult.ACK_COMMON) {
  559.                     // markCommon appends the object to the "state"
  560.                     markCommon(walk.parseAny(returnedId), ack, true);
  561.                     fetchState.havesWithoutAck = 0;
  562.                     fetchState.hadAcks = true;
  563.                 } else if (ack == AckNackResult.ACK_READY) {
  564.                     gotReady = true;
  565.                 }
  566.             }
  567.             n++;
  568.             if (n % 10 == 0 && monitor.isCancelled()) {
  569.                 throw new CancelledException();
  570.             }
  571.             line = input.readString();
  572.         }
  573.         if (gotReady) {
  574.             if (!PacketLineIn.isDelimiter(line)) {
  575.                 throw new PackProtocolException(MessageFormat
  576.                         .format(JGitText.get().expectedGot, "0001", line)); //$NON-NLS-1$
  577.             }
  578.         } else if (!PacketLineIn.isEnd(line)) {
  579.             throw new PackProtocolException(MessageFormat
  580.                     .format(JGitText.get().expectedGot, "0000", line)); //$NON-NLS-1$
  581.         }
  582.         return gotReady;
  583.     }

  585.     /** {@inheritDoc} */
  586.     @Override
  587.     public void close() {
  588.         if (walk != null)
  589.             walk.close();
  590.         super.close();
  591.     }

  593.     FetchConfig getFetchConfig() {
  594.         return local.getConfig().get(FetchConfig::new);
  595.     }

  597.     private int maxTimeWanted(Collection<Ref> wants) {
  598.         int maxTime = 0;
  599.         for (Ref r : wants) {
  600.             try {
  601.                 final RevObject obj = walk.parseAny(r.getObjectId());
  602.                 if (obj instanceof RevCommit) {
  603.                     final int cTime = ((RevCommit) obj).getCommitTime();
  604.                     if (maxTime < cTime)
  605.                         maxTime = cTime;
  606.                 }
  607.             } catch (IOException error) {
  608.                 // We don't have it, but we want to fetch (thus fixing error).
  609.             }
  610.         }
  611.         return maxTime;
  612.     }

  614.     private void markReachable(Set<ObjectId> have, int maxTime)
  615.             throws IOException {
  616.         for (Ref r : local.getRefDatabase().getRefs()) {
  617.             ObjectId id = r.getPeeledObjectId();
  618.             if (id == null)
  619.                 id = r.getObjectId();
  620.             if (id == null)
  621.                 continue;
  622.             parseReachable(id);
  623.         }

  625.         for (ObjectId id : local.getAdditionalHaves())
  626.             parseReachable(id);

  628.         for (ObjectId id : have)
  629.             parseReachable(id);

  631.         if (maxTime > 0) {
  632.             // Mark reachable commits until we reach maxTime. These may
  633.             // wind up later matching up against things we want and we
  634.             // can avoid asking for something we already happen to have.
  635.             //
  636.             final Date maxWhen = new Date(maxTime * 1000L);
  637.             walk.sort(RevSort.COMMIT_TIME_DESC);
  638.             walk.markStart(reachableCommits);
  639.             walk.setRevFilter(CommitTimeRevFilter.after(maxWhen));
  640.             for (;;) {
  641.                 final RevCommit c = walk.next();
  642.                 if (c == null)
  643.                     break;
  644.                 if (c.has(ADVERTISED) && !c.has(COMMON)) {
  645.                     // This is actually going to be a common commit, but
  646.                     // our peer doesn't know that fact yet.
  647.                     //
  648.                     c.add(COMMON);
  649.                     c.carry(COMMON);
  650.                     reachableCommits.add(c);
  651.                 }
  652.             }
  653.         }
  654.     }

  656.     private void parseReachable(ObjectId id) {
  657.         try {
  658.             RevCommit o = walk.parseCommit(id);
  659.             if (!o.has(REACHABLE)) {
  660.                 o.add(REACHABLE);
  661.                 reachableCommits.add(o);
  662.             }
  663.         } catch (IOException readError) {
  664.             // If we cannot read the value of the ref skip it.
  665.         }
  666.     }

  668.     private boolean sendWants(Collection<Ref> want, PacketLineOut p)
  669.             throws IOException {
  670.         boolean first = true;
  671.         for (Ref r : want) {
  672.             ObjectId objectId = r.getObjectId();
  673.             if (objectId == null) {
  674.                 continue;
  675.             }
  676.             try {
  677.                 if (walk.parseAny(objectId).has(REACHABLE)) {
  678.                     // We already have this object. Asking for it is
  679.                     // not a very good idea.
  680.                     //
  681.                     continue;
  682.                 }
  683.             } catch (IOException err) {
  684.                 // Its OK, we don't have it, but we want to fix that
  685.                 // by fetching the object from the other side.
  686.             }

  688.             final StringBuilder line = new StringBuilder(46);
  689.             line.append("want "); //$NON-NLS-1$
  690.             line.append(objectId.name());
  691.             if (first && TransferConfig.ProtocolVersion.V0
  692.                     .equals(getProtocolVersion())) {
  693.                 line.append(enableCapabilities());
  694.             }
  695.             first = false;
  696.             line.append('\n');
  697.             p.writeString(line.toString());
  698.         }
  699.         if (first) {
  700.             return false;
  701.         }
  702.         if (!filterSpec.isNoOp()) {
  703.             p.writeString(filterSpec.filterLine());
  704.         }
  705.         return true;
  706.     }

  708.     private Set<String> getCapabilitiesV2(Set<String> advertisedCapabilities)
  709.             throws TransportException {
  710.         Set<String> capabilities = new LinkedHashSet<>();
  711.         // Protocol V2 is implicitly capable of all these.
  712.         if (noProgress) {
  713.             capabilities.add(OPTION_NO_PROGRESS);
  714.         }
  715.         if (includeTags) {
  716.             capabilities.add(OPTION_INCLUDE_TAG);
  717.         }
  718.         if (allowOfsDelta) {
  719.             capabilities.add(OPTION_OFS_DELTA);
  720.         }
  721.         if (thinPack) {
  722.             capabilities.add(OPTION_THIN_PACK);
  723.         }
  724.         if (!filterSpec.isNoOp()
  725.                 && !advertisedCapabilities.contains(OPTION_FILTER)) {
  726.             throw new PackProtocolException(uri,
  727.                     JGitText.get().filterRequiresCapability);
  728.         }
  729.         // The FilterSpec will be added later in sendWants().
  730.         return capabilities;
  731.     }

  733.     private String enableCapabilities() throws TransportException {
  734.         final StringBuilder line = new StringBuilder();
  735.         if (noProgress)
  736.             wantCapability(line, OPTION_NO_PROGRESS);
  737.         if (includeTags)
  738.             includeTags = wantCapability(line, OPTION_INCLUDE_TAG);
  739.         if (allowOfsDelta)
  740.             wantCapability(line, OPTION_OFS_DELTA);

  742.         if (wantCapability(line, OPTION_MULTI_ACK_DETAILED)) {
  743.             multiAck = MultiAck.DETAILED;
  744.             if (statelessRPC)
  745.                 noDone = wantCapability(line, OPTION_NO_DONE);
  746.         } else if (wantCapability(line, OPTION_MULTI_ACK))
  747.             multiAck = MultiAck.CONTINUE;
  748.         else
  749.             multiAck = MultiAck.OFF;

  751.         if (thinPack)
  752.             thinPack = wantCapability(line, OPTION_THIN_PACK);
  753.         if (wantCapability(line, OPTION_SIDE_BAND_64K))
  754.             sideband = true;
  755.         else if (wantCapability(line, OPTION_SIDE_BAND))
  756.             sideband = true;

  758.         if (statelessRPC && multiAck != MultiAck.DETAILED) {
  759.             // Our stateless RPC implementation relies upon the detailed
  760.             // ACK status to tell us common objects for reuse in future
  761.             // requests.  If its not enabled, we can't talk to the peer.
  762.             //
  763.             throw new PackProtocolException(uri, MessageFormat.format(
  764.                     JGitText.get().statelessRPCRequiresOptionToBeEnabled,
  765.                     OPTION_MULTI_ACK_DETAILED));
  766.         }

  768.         if (!filterSpec.isNoOp() && !wantCapability(line, OPTION_FILTER)) {
  769.             throw new PackProtocolException(uri,
  770.                     JGitText.get().filterRequiresCapability);
  771.         }

  773.         addUserAgentCapability(line);
  774.         return line.toString();
  775.     }

  777.     private void negotiate(ProgressMonitor monitor) throws IOException,
  778.             CancelledException {
  779.         final MutableObjectId ackId = new MutableObjectId();
  780.         int resultsPending = 0;
  781.         int havesSent = 0;
  782.         int havesSinceLastContinue = 0;
  783.         boolean receivedContinue = false;
  784.         boolean receivedAck = false;
  785.         boolean receivedReady = false;

  787.         if (statelessRPC) {
  788.             state.writeTo(out, null);
  789.         }

  791.         negotiateBegin();
  792.         SEND_HAVES: for (;;) {
  793.             final RevCommit c = walk.next();
  794.             if (c == null) {
  795.                 break SEND_HAVES;
  796.             }

  798.             ObjectId o = c.getId();
  799.             pckOut.writeString("have " + o.name() + "\n"); //$NON-NLS-1$ //$NON-NLS-2$
  800.             havesSent++;
  801.             havesSinceLastContinue++;

  803.             if ((31 & havesSent) != 0) {
  804.                 // We group the have lines into blocks of 32, each marked
  805.                 // with a flush (aka end). This one is within a block so
  806.                 // continue with another have line.
  807.                 //
  808.                 continue;
  809.             }

  811.             if (monitor.isCancelled()) {
  812.                 throw new CancelledException();
  813.             }

  815.             pckOut.end();
  816.             resultsPending++; // Each end will cause a result to come back.

  818.             if (havesSent == 32 && !statelessRPC) {
  819.                 // On the first block we race ahead and try to send
  820.                 // more of the second block while waiting for the
  821.                 // remote to respond to our first block request.
  822.                 // This keeps us one block ahead of the peer.
  823.                 //
  824.                 continue;
  825.             }

  827.             READ_RESULT: for (;;) {
  828.                 final AckNackResult anr = pckIn.readACK(ackId);
  829.                 switch (anr) {
  830.                 case NAK:
  831.                     // More have lines are necessary to compute the
  832.                     // pack on the remote side. Keep doing that.
  833.                     //
  834.                     resultsPending--;
  835.                     break READ_RESULT;

  837.                 case ACK:
  838.                     // The remote side is happy and knows exactly what
  839.                     // to send us. There is no further negotiation and
  840.                     // we can break out immediately.
  841.                     //
  842.                     multiAck = MultiAck.OFF;
  843.                     resultsPending = 0;
  844.                     receivedAck = true;
  845.                     if (statelessRPC) {
  846.                         state.writeTo(out, null);
  847.                     }
  848.                     break SEND_HAVES;

  850.                 case ACK_CONTINUE:
  851.                 case ACK_COMMON:
  852.                 case ACK_READY:
  853.                     // The server knows this commit (ackId). We don't
  854.                     // need to send any further along its ancestry, but
  855.                     // we need to continue to talk about other parts of
  856.                     // our local history.
  857.                     //
  858.                     markCommon(walk.parseAny(ackId), anr, statelessRPC);
  859.                     receivedAck = true;
  860.                     receivedContinue = true;
  861.                     havesSinceLastContinue = 0;
  862.                     if (anr == AckNackResult.ACK_READY) {
  863.                         receivedReady = true;
  864.                     }
  865.                     break;
  866.                 }

  868.                 if (monitor.isCancelled()) {
  869.                     throw new CancelledException();
  870.                 }
  871.             }

  873.             if (noDone && receivedReady) {
  874.                 break SEND_HAVES;
  875.             }
  876.             if (statelessRPC) {
  877.                 state.writeTo(out, null);
  878.             }

  880.             if ((receivedContinue && havesSinceLastContinue > MAX_HAVES)
  881.                     || havesSent >= maxHaves) {
  882.                 // Our history must be really different from the remote's.
  883.                 // We just sent a whole slew of have lines, and it did not
  884.                 // recognize any of them. Avoid sending our entire history
  885.                 // to them by giving up early.
  886.                 //
  887.                 break SEND_HAVES;
  888.             }
  889.         }

  891.         // Tell the remote side we have run out of things to talk about.
  892.         //
  893.         if (monitor.isCancelled()) {
  894.             throw new CancelledException();
  895.         }

  897.         if (!receivedReady || !noDone) {
  898.             // When statelessRPC is true we should always leave SEND_HAVES
  899.             // loop above while in the middle of a request. This allows us
  900.             // to just write done immediately.
  901.             //
  902.             pckOut.writeString("done\n"); //$NON-NLS-1$
  903.             pckOut.flush();
  904.         }

  906.         if (!receivedAck) {
  907.             // Apparently if we have never received an ACK earlier
  908.             // there is one more result expected from the done we
  909.             // just sent to the remote.
  910.             //
  911.             multiAck = MultiAck.OFF;
  912.             resultsPending++;
  913.         }

  915.         READ_RESULT: while (resultsPending > 0 || multiAck != MultiAck.OFF) {
  916.             final AckNackResult anr = pckIn.readACK(ackId);
  917.             resultsPending--;
  918.             switch (anr) {
  919.             case NAK:
  920.                 // A NAK is a response to an end we queued earlier
  921.                 // we eat it and look for another ACK/NAK message.
  922.                 //
  923.                 break;

  925.             case ACK:
  926.                 // A solitary ACK at this point means the remote won't
  927.                 // speak anymore, but is going to send us a pack now.
  928.                 //
  929.                 break READ_RESULT;

  931.             case ACK_CONTINUE:
  932.             case ACK_COMMON:
  933.             case ACK_READY:
  934.                 // We will expect a normal ACK to break out of the loop.
  935.                 //
  936.                 multiAck = MultiAck.CONTINUE;
  937.                 break;
  938.             }

  940.             if (monitor.isCancelled()) {
  941.                 throw new CancelledException();
  942.             }
  943.         }
  944.     }

  946.     private void negotiateBegin() throws IOException {
  947.         walk.resetRetain(REACHABLE, ADVERTISED);
  948.         walk.markStart(reachableCommits);
  949.         walk.sort(RevSort.COMMIT_TIME_DESC);
  950.         walk.setRevFilter(new RevFilter() {
  951.             @Override
  952.             public RevFilter clone() {
  953.                 return this;
  954.             }

  956.             @Override
  957.             public boolean include(RevWalk walker, RevCommit c) {
  958.                 final boolean remoteKnowsIsCommon = c.has(COMMON);
  959.                 if (c.has(ADVERTISED)) {
  960.                     // Remote advertised this, and we have it, hence common.
  961.                     // Whether or not the remote knows that fact is tested
  962.                     // before we added the flag. If the remote doesn't know
  963.                     // we have to still send them this object.
  964.                     //
  965.                     c.add(COMMON);
  966.                 }
  967.                 return !remoteKnowsIsCommon;
  968.             }

  970.             @Override
  971.             public boolean requiresCommitBody() {
  972.                 return false;
  973.             }
  974.         });
  975.     }

  977.     private void markRefsAdvertised() {
  978.         for (Ref r : getRefs()) {
  979.             markAdvertised(r.getObjectId());
  980.             if (r.getPeeledObjectId() != null)
  981.                 markAdvertised(r.getPeeledObjectId());
  982.         }
  983.     }

  985.     private void markAdvertised(AnyObjectId id) {
  986.         try {
  987.             walk.parseAny(id).add(ADVERTISED);
  988.         } catch (IOException readError) {
  989.             // We probably just do not have this object locally.
  990.         }
  991.     }

  993.     private void markCommon(RevObject obj, AckNackResult anr, boolean useState)
  994.             throws IOException {
  995.         if (useState && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) {
  996.             pckState.writeString("have " + obj.name() + '\n'); //$NON-NLS-1$
  997.             obj.add(STATE);
  998.         }
  999.         obj.add(COMMON);
  1000.         if (obj instanceof RevCommit)
  1001.             ((RevCommit) obj).carry(COMMON);
  1002.     }

  1004.     private void receivePack(final ProgressMonitor monitor,
  1005.             OutputStream outputStream) throws IOException {
  1006.         onReceivePack();
  1007.         InputStream input = in;
  1008.         if (sideband)
  1009.             input = new SideBandInputStream(input, monitor, getMessageWriter(),
  1010.                     outputStream);

  1012.         try (ObjectInserter ins = local.newObjectInserter()) {
  1013.             PackParser parser = ins.newPackParser(input);
  1014.             parser.setAllowThin(thinPack);
  1015.             parser.setObjectChecker(transport.getObjectChecker());
  1016.             parser.setLockMessage(lockMessage);
  1017.             packLock = parser.parse(monitor);
  1018.             ins.flush();
  1019.         }
  1020.     }

  1022.     /**
  1023.      * Notification event delivered just before the pack is received from the
  1024.      * network. This event can be used by RPC such as {@link org.eclipse.jgit.transport.TransportHttp} to
  1025.      * disable its request magic and ensure the pack stream is read correctly.
  1026.      *
  1027.      * @since 2.0
  1028.      */
  1029.     protected void onReceivePack() {
  1030.         // By default do nothing for TCP based protocols.
  1031.     }

  1033.     private static class CancelledException extends Exception {
  1034.         private static final long serialVersionUID = 1L;
  1035.     }

  1037.     private static class FetchStateV2 {

  1039.         long havesToSend = 32;

  1041.         long havesTotal;

  1043.         // Set to true if we got at least one ACK in protocol V2.
  1044.         boolean hadAcks;

  1046.         // Counts haves without ACK. Use as cutoff for negotiation only once
  1047.         // hadAcks == true.
  1048.         long havesWithoutAck;

  1050.         void incHavesToSend(boolean statelessRPC) {
  1051.             if (statelessRPC) {
  1052.                 // Increase this quicker since connection setup costs accumulate
  1053.                 if (havesToSend < 16384) {
  1054.                     havesToSend *= 2;
  1055.                 } else {
  1056.                     havesToSend = havesToSend * 11 / 10;
  1057.                 }
  1058.             } else {
  1059.                 havesToSend += 32;
  1060.             }
  1061.         }
  1062.     }
  1063. }
Created with JaCoCo 0.8.6.202009150832