PackParser.java

  1. /*
  2.  * Copyright (C) 2008-2011, Google Inc.
  3.  * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com>
  4.  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
  5.  * and other copyright owners as documented in the project's IP log.
  6.  *
  7.  * This program and the accompanying materials are made available
  8.  * under the terms of the Eclipse Distribution License v1.0 which
  9.  * accompanies this distribution, is reproduced below, and is
  10.  * available at http://www.eclipse.org/org/documents/edl-v10.php
  11.  *
  12.  * All rights reserved.
  13.  *
  14.  * Redistribution and use in source and binary forms, with or
  15.  * without modification, are permitted provided that the following
  16.  * conditions are met:
  17.  *
  18.  * - Redistributions of source code must retain the above copyright
  19.  *   notice, this list of conditions and the following disclaimer.
  20.  *
  21.  * - Redistributions in binary form must reproduce the above
  22.  *   copyright notice, this list of conditions and the following
  23.  *   disclaimer in the documentation and/or other materials provided
  24.  *   with the distribution.
  25.  *
  26.  * - Neither the name of the Eclipse Foundation, Inc. nor the
  27.  *   names of its contributors may be used to endorse or promote
  28.  *   products derived from this software without specific prior
  29.  *   written permission.
  30.  *
  31.  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  32.  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  33.  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  34.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  35.  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  36.  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  37.  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  38.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  39.  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  40.  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  41.  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  42.  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  43.  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  44.  */

  46. package org.eclipse.jgit.transport;

  48. import java.io.EOFException;
  49. import java.io.IOException;
  50. import java.io.InputStream;
  51. import java.security.MessageDigest;
  52. import java.text.MessageFormat;
  53. import java.util.ArrayList;
  54. import java.util.Arrays;
  55. import java.util.Comparator;
  56. import java.util.List;
  57. import java.util.concurrent.TimeUnit;
  58. import java.util.zip.DataFormatException;
  59. import java.util.zip.Inflater;

  61. import org.eclipse.jgit.errors.CorruptObjectException;
  62. import org.eclipse.jgit.errors.MissingObjectException;
  63. import org.eclipse.jgit.errors.TooLargeObjectInPackException;
  64. import org.eclipse.jgit.internal.JGitText;
  65. import org.eclipse.jgit.internal.storage.file.PackLock;
  66. import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
  67. import org.eclipse.jgit.lib.AnyObjectId;
  68. import org.eclipse.jgit.lib.BatchingProgressMonitor;
  69. import org.eclipse.jgit.lib.BlobObjectChecker;
  70. import org.eclipse.jgit.lib.Constants;
  71. import org.eclipse.jgit.lib.InflaterCache;
  72. import org.eclipse.jgit.lib.MutableObjectId;
  73. import org.eclipse.jgit.lib.NullProgressMonitor;
  74. import org.eclipse.jgit.lib.ObjectChecker;
  75. import org.eclipse.jgit.lib.ObjectDatabase;
  76. import org.eclipse.jgit.lib.ObjectId;
  77. import org.eclipse.jgit.lib.ObjectIdOwnerMap;
  78. import org.eclipse.jgit.lib.ObjectIdSubclassMap;
  79. import org.eclipse.jgit.lib.ObjectLoader;
  80. import org.eclipse.jgit.lib.ObjectReader;
  81. import org.eclipse.jgit.lib.ObjectStream;
  82. import org.eclipse.jgit.lib.ProgressMonitor;
  83. import org.eclipse.jgit.util.BlockList;
  84. import org.eclipse.jgit.util.IO;
  85. import org.eclipse.jgit.util.LongMap;
  86. import org.eclipse.jgit.util.NB;
  87. import org.eclipse.jgit.util.sha1.SHA1;

  89. /**
  90.  * Parses a pack stream and imports it for an
  91.  * {@link org.eclipse.jgit.lib.ObjectInserter}.
  92.  * <p>
  93.  * Applications can acquire an instance of a parser from ObjectInserter's
  94.  * {@link org.eclipse.jgit.lib.ObjectInserter#newPackParser(InputStream)}
  95.  * method.
  96.  * <p>
  97.  * Implementations of {@link org.eclipse.jgit.lib.ObjectInserter} should
  98.  * subclass this type and provide their own logic for the various {@code on*()}
  99.  * event methods declared to be abstract.
  100.  */
  101. public abstract class PackParser {
  102.     /** Size of the internal stream buffer. */
  103.     private static final int BUFFER_SIZE = 8192;

  105.     /** Location data is being obtained from. */
  106.     public static enum Source {
  107.         /** Data is read from the incoming stream. */
  108.         INPUT,

  110.         /** Data is read back from the database's buffers. */
  111.         DATABASE;
  112.     }

  114.     /** Object database used for loading existing objects. */
  115.     private final ObjectDatabase objectDatabase;

  117.     private InflaterStream inflater;

  119.     private byte[] tempBuffer;

  121.     private byte[] hdrBuf;

  123.     private final SHA1 objectHasher = SHA1.newInstance();
  124.     private final MutableObjectId tempObjectId;

  126.     private InputStream in;

  128.     byte[] buf;

  130.     /** Position in the input stream of {@code buf[0]}. */
  131.     private long bBase;

  133.     private int bOffset;

  135.     int bAvail;

  137.     private ObjectChecker objCheck;

  139.     private boolean allowThin;

  141.     private boolean checkObjectCollisions;

  143.     private boolean needBaseObjectIds;

  145.     private boolean checkEofAfterPackFooter;

  147.     private boolean expectDataAfterPackFooter;

  149.     private long expectedObjectCount;

  151.     private PackedObjectInfo[] entries;

  153.     /**
  154.      * Every object contained within the incoming pack.
  155.      * <p>
  156.      * This is a subset of {@link #entries}, as thin packs can add additional
  157.      * objects to {@code entries} by copying already existing objects from the
  158.      * repository onto the end of the thin pack to make it self-contained.
  159.      */
  160.     private ObjectIdSubclassMap<ObjectId> newObjectIds;

  162.     private int deltaCount;

  164.     private int entryCount;

  166.     private ObjectIdOwnerMap<DeltaChain> baseById;

  168.     /**
  169.      * Objects referenced by their name from deltas, that aren't in this pack.
  170.      * <p>
  171.      * This is the set of objects that were copied onto the end of this pack to
  172.      * make it complete. These objects were not transmitted by the remote peer,
  173.      * but instead were assumed to already exist in the local repository.
  174.      */
  175.     private ObjectIdSubclassMap<ObjectId> baseObjectIds;

  177.     private LongMap<UnresolvedDelta> baseByPos;

  179.     /** Objects need to be double-checked for collision after indexing. */
  180.     private BlockList<PackedObjectInfo> collisionCheckObjs;

  182.     private MessageDigest packDigest;

  184.     private ObjectReader readCurs;

  186.     /** Message to protect the pack data from garbage collection. */
  187.     private String lockMessage;

  189.     /** Git object size limit */
  190.     private long maxObjectSizeLimit;

  192.     private final ReceivedPackStatistics.Builder stats =
  193.             new ReceivedPackStatistics.Builder();

  195.     /**
  196.      * Initialize a pack parser.
  197.      *
  198.      * @param odb
  199.      *            database the parser will write its objects into.
  200.      * @param src
  201.      *            the stream the parser will read.
  202.      */
  203.     protected PackParser(ObjectDatabase odb, InputStream src) {
  204.         objectDatabase = odb.newCachedDatabase();
  205.         in = src;

  207.         inflater = new InflaterStream();
  208.         readCurs = objectDatabase.newReader();
  209.         buf = new byte[BUFFER_SIZE];
  210.         tempBuffer = new byte[BUFFER_SIZE];
  211.         hdrBuf = new byte[64];
  212.         tempObjectId = new MutableObjectId();
  213.         packDigest = Constants.newMessageDigest();
  214.         checkObjectCollisions = true;
  215.     }

  217.     /**
  218.      * Whether a thin pack (missing base objects) is permitted.
  219.      *
  220.      * @return {@code true} if a thin pack (missing base objects) is permitted.
  221.      */
  222.     public boolean isAllowThin() {
  223.         return allowThin;
  224.     }

  226.     /**
  227.      * Configure this index pack instance to allow a thin pack.
  228.      * <p>
  229.      * Thin packs are sometimes used during network transfers to allow a delta
  230.      * to be sent without a base object. Such packs are not permitted on disk.
  231.      *
  232.      * @param allow
  233.      *            true to enable a thin pack.
  234.      */
  235.     public void setAllowThin(boolean allow) {
  236.         allowThin = allow;
  237.     }

  239.     /**
  240.      * Whether received objects are verified to prevent collisions.
  241.      *
  242.      * @return if true received objects are verified to prevent collisions.
  243.      * @since 4.1
  244.      */
  245.     protected boolean isCheckObjectCollisions() {
  246.         return checkObjectCollisions;
  247.     }

  249.     /**
  250.      * Enable checking for collisions with existing objects.
  251.      * <p>
  252.      * By default PackParser looks for each received object in the repository.
  253.      * If the object already exists, the existing object is compared
  254.      * byte-for-byte with the newly received copy to ensure they are identical.
  255.      * The receive is aborted with an exception if any byte differs. This check
  256.      * is necessary to prevent an evil attacker from supplying a replacement
  257.      * object into this repository in the event that a discovery enabling SHA-1
  258.      * collisions is made.
  259.      * <p>
  260.      * This check may be very costly to perform, and some repositories may have
  261.      * other ways to segregate newly received object data. The check is enabled
  262.      * by default, but can be explicitly disabled if the implementation can
  263.      * provide the same guarantee, or is willing to accept the risks associated
  264.      * with bypassing the check.
  265.      *
  266.      * @param check
  267.      *            true to enable collision checking (strongly encouraged).
  268.      * @since 4.1
  269.      */
  270.     protected void setCheckObjectCollisions(boolean check) {
  271.         checkObjectCollisions = check;
  272.     }

  274.     /**
  275.      * Configure this index pack instance to keep track of new objects.
  276.      * <p>
  277.      * By default an index pack doesn't save the new objects that were created
  278.      * when it was instantiated. Setting this flag to {@code true} allows the
  279.      * caller to use {@link #getNewObjectIds()} to retrieve that list.
  280.      *
  281.      * @param b
  282.      *            {@code true} to enable keeping track of new objects.
  283.      */
  284.     public void setNeedNewObjectIds(boolean b) {
  285.         if (b)
  286.             newObjectIds = new ObjectIdSubclassMap<>();
  287.         else
  288.             newObjectIds = null;
  289.     }

  291.     private boolean needNewObjectIds() {
  292.         return newObjectIds != null;
  293.     }

  295.     /**
  296.      * Configure this index pack instance to keep track of the objects assumed
  297.      * for delta bases.
  298.      * <p>
  299.      * By default an index pack doesn't save the objects that were used as delta
  300.      * bases. Setting this flag to {@code true} will allow the caller to use
  301.      * {@link #getBaseObjectIds()} to retrieve that list.
  302.      *
  303.      * @param b
  304.      *            {@code true} to enable keeping track of delta bases.
  305.      */
  306.     public void setNeedBaseObjectIds(boolean b) {
  307.         this.needBaseObjectIds = b;
  308.     }

  310.     /**
  311.      * Whether the EOF should be read from the input after the footer.
  312.      *
  313.      * @return true if the EOF should be read from the input after the footer.
  314.      */
  315.     public boolean isCheckEofAfterPackFooter() {
  316.         return checkEofAfterPackFooter;
  317.     }

  319.     /**
  320.      * Ensure EOF is read from the input stream after the footer.
  321.      *
  322.      * @param b
  323.      *            true if the EOF should be read; false if it is not checked.
  324.      */
  325.     public void setCheckEofAfterPackFooter(boolean b) {
  326.         checkEofAfterPackFooter = b;
  327.     }

  329.     /**
  330.      * Whether there is data expected after the pack footer.
  331.      *
  332.      * @return true if there is data expected after the pack footer.
  333.      */
  334.     public boolean isExpectDataAfterPackFooter() {
  335.         return expectDataAfterPackFooter;
  336.     }

  338.     /**
  339.      * Set if there is additional data in InputStream after pack.
  340.      *
  341.      * @param e
  342.      *            true if there is additional data in InputStream after pack.
  343.      *            This requires the InputStream to support the mark and reset
  344.      *            functions.
  345.      */
  346.     public void setExpectDataAfterPackFooter(boolean e) {
  347.         expectDataAfterPackFooter = e;
  348.     }

  350.     /**
  351.      * Get the new objects that were sent by the user
  352.      *
  353.      * @return the new objects that were sent by the user
  354.      */
  355.     public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
  356.         if (newObjectIds != null)
  357.             return newObjectIds;
  358.         return new ObjectIdSubclassMap<>();
  359.     }

  361.     /**
  362.      * Get set of objects the incoming pack assumed for delta purposes
  363.      *
  364.      * @return set of objects the incoming pack assumed for delta purposes
  365.      */
  366.     public ObjectIdSubclassMap<ObjectId> getBaseObjectIds() {
  367.         if (baseObjectIds != null)
  368.             return baseObjectIds;
  369.         return new ObjectIdSubclassMap<>();
  370.     }

  372.     /**
  373.      * Configure the checker used to validate received objects.
  374.      * <p>
  375.      * Usually object checking isn't necessary, as Git implementations only
  376.      * create valid objects in pack files. However, additional checking may be
  377.      * useful if processing data from an untrusted source.
  378.      *
  379.      * @param oc
  380.      *            the checker instance; null to disable object checking.
  381.      */
  382.     public void setObjectChecker(ObjectChecker oc) {
  383.         objCheck = oc;
  384.     }

  386.     /**
  387.      * Configure the checker used to validate received objects.
  388.      * <p>
  389.      * Usually object checking isn't necessary, as Git implementations only
  390.      * create valid objects in pack files. However, additional checking may be
  391.      * useful if processing data from an untrusted source.
  392.      * <p>
  393.      * This is shorthand for:
  394.      *
  395.      * <pre>
  396.      * setObjectChecker(on ? new ObjectChecker() : null);
  397.      * </pre>
  398.      *
  399.      * @param on
  400.      *            true to enable the default checker; false to disable it.
  401.      */
  402.     public void setObjectChecking(boolean on) {
  403.         setObjectChecker(on ? new ObjectChecker() : null);
  404.     }

  406.     /**
  407.      * Get the message to record with the pack lock.
  408.      *
  409.      * @return the message to record with the pack lock.
  410.      */
  411.     public String getLockMessage() {
  412.         return lockMessage;
  413.     }

  415.     /**
  416.      * Set the lock message for the incoming pack data.
  417.      *
  418.      * @param msg
  419.      *            if not null, the message to associate with the incoming data
  420.      *            while it is locked to prevent garbage collection.
  421.      */
  422.     public void setLockMessage(String msg) {
  423.         lockMessage = msg;
  424.     }

  426.     /**
  427.      * Set the maximum allowed Git object size.
  428.      * <p>
  429.      * If an object is larger than the given size the pack-parsing will throw an
  430.      * exception aborting the parsing.
  431.      *
  432.      * @param limit
  433.      *            the Git object size limit. If zero then there is not limit.
  434.      */
  435.     public void setMaxObjectSizeLimit(long limit) {
  436.         maxObjectSizeLimit = limit;
  437.     }

  439.     /**
  440.      * Get the number of objects in the stream.
  441.      * <p>
  442.      * The object count is only available after {@link #parse(ProgressMonitor)}
  443.      * has returned. The count may have been increased if the stream was a thin
  444.      * pack, and missing bases objects were appending onto it by the subclass.
  445.      *
  446.      * @return number of objects parsed out of the stream.
  447.      */
  448.     public int getObjectCount() {
  449.         return entryCount;
  450.     }

  452.     /**
  453.      * Get the information about the requested object.
  454.      * <p>
  455.      * The object information is only available after
  456.      * {@link #parse(ProgressMonitor)} has returned.
  457.      *
  458.      * @param nth
  459.      *            index of the object in the stream. Must be between 0 and
  460.      *            {@link #getObjectCount()}-1.
  461.      * @return the object information.
  462.      */
  463.     public PackedObjectInfo getObject(int nth) {
  464.         return entries[nth];
  465.     }

  467.     /**
  468.      * Get all of the objects, sorted by their name.
  469.      * <p>
  470.      * The object information is only available after
  471.      * {@link #parse(ProgressMonitor)} has returned.
  472.      * <p>
  473.      * To maintain lower memory usage and good runtime performance, this method
  474.      * sorts the objects in-place and therefore impacts the ordering presented
  475.      * by {@link #getObject(int)}.
  476.      *
  477.      * @param cmp
  478.      *            comparison function, if null objects are stored by ObjectId.
  479.      * @return sorted list of objects in this pack stream.
  480.      */
  481.     public List<PackedObjectInfo> getSortedObjectList(
  482.             Comparator<PackedObjectInfo> cmp) {
  483.         Arrays.sort(entries, 0, entryCount, cmp);
  484.         List<PackedObjectInfo> list = Arrays.asList(entries);
  485.         if (entryCount < entries.length)
  486.             list = list.subList(0, entryCount);
  487.         return list;
  488.     }

  490.     /**
  491.      * Get the size of the newly created pack.
  492.      * <p>
  493.      * This will also include the pack index size if an index was created. This
  494.      * method should only be called after pack parsing is finished.
  495.      *
  496.      * @return the pack size (including the index size) or -1 if the size cannot
  497.      *         be determined
  498.      * @since 3.3
  499.      */
  500.     public long getPackSize() {
  501.         return -1;
  502.     }

  504.     /**
  505.      * Returns the statistics of the parsed pack.
  506.      * <p>
  507.      * This should only be called after pack parsing is finished.
  508.      *
  509.      * @return {@link org.eclipse.jgit.transport.ReceivedPackStatistics}
  510.      * @since 4.6
  511.      */
  512.     public ReceivedPackStatistics getReceivedPackStatistics() {
  513.         return stats.build();
  514.     }

  516.     /**
  517.      * Parse the pack stream.
  518.      *
  519.      * @param progress
  520.      *            callback to provide progress feedback during parsing. If null,
  521.      *            {@link org.eclipse.jgit.lib.NullProgressMonitor} will be used.
  522.      * @return the pack lock, if one was requested by setting
  523.      *         {@link #setLockMessage(String)}.
  524.      * @throws java.io.IOException
  525.      *             the stream is malformed, or contains corrupt objects.
  526.      * @since 3.0
  527.      */
  528.     public final PackLock parse(ProgressMonitor progress) throws IOException {
  529.         return parse(progress, progress);
  530.     }

  532.     /**
  533.      * Parse the pack stream.
  534.      *
  535.      * @param receiving
  536.      *            receives progress feedback during the initial receiving
  537.      *            objects phase. If null,
  538.      *            {@link org.eclipse.jgit.lib.NullProgressMonitor} will be used.
  539.      * @param resolving
  540.      *            receives progress feedback during the resolving objects phase.
  541.      * @return the pack lock, if one was requested by setting
  542.      *         {@link #setLockMessage(String)}.
  543.      * @throws java.io.IOException
  544.      *             the stream is malformed, or contains corrupt objects.
  545.      * @since 3.0
  546.      */
  547.     public PackLock parse(ProgressMonitor receiving, ProgressMonitor resolving)
  548.             throws IOException {
  549.         if (receiving == null)
  550.             receiving = NullProgressMonitor.INSTANCE;
  551.         if (resolving == null)
  552.             resolving = NullProgressMonitor.INSTANCE;

  554.         if (receiving == resolving)
  555.             receiving.start(2 /* tasks */);
  556.         try {
  557.             readPackHeader();

  559.             entries = new PackedObjectInfo[(int) expectedObjectCount];
  560.             baseById = new ObjectIdOwnerMap<>();
  561.             baseByPos = new LongMap<>();
  562.             collisionCheckObjs = new BlockList<>();

  564.             receiving.beginTask(JGitText.get().receivingObjects,
  565.                     (int) expectedObjectCount);
  566.             try {
  567.                 for (int done = 0; done < expectedObjectCount; done++) {
  568.                     indexOneObject();
  569.                     receiving.update(1);
  570.                     if (receiving.isCancelled())
  571.                         throw new IOException(JGitText.get().downloadCancelled);
  572.                 }
  573.                 readPackFooter();
  574.                 endInput();
  575.             } finally {
  576.                 receiving.endTask();
  577.             }

  579.             if (!collisionCheckObjs.isEmpty()) {
  580.                 checkObjectCollision();
  581.             }

  583.             if (deltaCount > 0) {
  584.                 processDeltas(resolving);
  585.             }

  587.             packDigest = null;
  588.             baseById = null;
  589.             baseByPos = null;
  590.         } finally {
  591.             try {
  592.                 if (readCurs != null)
  593.                     readCurs.close();
  594.             } finally {
  595.                 readCurs = null;
  596.             }

  598.             try {
  599.                 inflater.release();
  600.             } finally {
  601.                 inflater = null;
  602.             }
  603.         }
  604.         return null; // By default there is no locking.
  605.     }

  607.     private void processDeltas(ProgressMonitor resolving) throws IOException {
  608.         if (resolving instanceof BatchingProgressMonitor) {
  609.             ((BatchingProgressMonitor) resolving).setDelayStart(1000,
  610.                     TimeUnit.MILLISECONDS);
  611.         }
  612.         resolving.beginTask(JGitText.get().resolvingDeltas, deltaCount);
  613.         resolveDeltas(resolving);
  614.         if (entryCount < expectedObjectCount) {
  615.             if (!isAllowThin()) {
  616.                 throw new IOException(MessageFormat.format(
  617.                         JGitText.get().packHasUnresolvedDeltas,
  618.                         Long.valueOf(expectedObjectCount - entryCount)));
  619.             }

  621.             resolveDeltasWithExternalBases(resolving);

  623.             if (entryCount < expectedObjectCount) {
  624.                 throw new IOException(MessageFormat.format(
  625.                         JGitText.get().packHasUnresolvedDeltas,
  626.                         Long.valueOf(expectedObjectCount - entryCount)));
  627.             }
  628.         }
  629.         resolving.endTask();
  630.     }

  632.     private void resolveDeltas(ProgressMonitor progress)
  633.             throws IOException {
  634.         final int last = entryCount;
  635.         for (int i = 0; i < last; i++) {
  636.             resolveDeltas(entries[i], progress);
  637.             if (progress.isCancelled())
  638.                 throw new IOException(
  639.                         JGitText.get().downloadCancelledDuringIndexing);
  640.         }
  641.     }

  643.     private void resolveDeltas(final PackedObjectInfo oe,
  644.             ProgressMonitor progress) throws IOException {
  645.         UnresolvedDelta children = firstChildOf(oe);
  646.         if (children == null)
  647.             return;

  649.         DeltaVisit visit = new DeltaVisit();
  650.         visit.nextChild = children;

  652.         ObjectTypeAndSize info = openDatabase(oe, new ObjectTypeAndSize());
  653.         switch (info.type) {
  654.         case Constants.OBJ_COMMIT:
  655.         case Constants.OBJ_TREE:
  656.         case Constants.OBJ_BLOB:
  657.         case Constants.OBJ_TAG:
  658.             visit.data = inflateAndReturn(Source.DATABASE, info.size);
  659.             visit.id = oe;
  660.             break;
  661.         default:
  662.             throw new IOException(MessageFormat.format(
  663.                     JGitText.get().unknownObjectType,
  664.                     Integer.valueOf(info.type)));
  665.         }

  667.         if (!checkCRC(oe.getCRC())) {
  668.             throw new IOException(MessageFormat.format(
  669.                     JGitText.get().corruptionDetectedReReadingAt,
  670.                     Long.valueOf(oe.getOffset())));
  671.         }

  673.         resolveDeltas(visit.next(), info.type, info, progress);
  674.     }

  676.     private void resolveDeltas(DeltaVisit visit, final int type,
  677.             ObjectTypeAndSize info, ProgressMonitor progress)
  678.             throws IOException {
  679.         stats.addDeltaObject(type);
  680.         do {
  681.             progress.update(1);
  682.             info = openDatabase(visit.delta, info);
  683.             switch (info.type) {
  684.             case Constants.OBJ_OFS_DELTA:
  685.             case Constants.OBJ_REF_DELTA:
  686.                 break;

  688.             default:
  689.                 throw new IOException(MessageFormat.format(
  690.                         JGitText.get().unknownObjectType,
  691.                         Integer.valueOf(info.type)));
  692.             }

  694.             byte[] delta = inflateAndReturn(Source.DATABASE, info.size);
  695.             checkIfTooLarge(type, BinaryDelta.getResultSize(delta));

  697.             visit.data = BinaryDelta.apply(visit.parent.data, delta);
  698.             delta = null;

  700.             if (!checkCRC(visit.delta.crc))
  701.                 throw new IOException(MessageFormat.format(
  702.                         JGitText.get().corruptionDetectedReReadingAt,
  703.                         Long.valueOf(visit.delta.position)));

  705.             SHA1 objectDigest = objectHasher.reset();
  706.             objectDigest.update(Constants.encodedTypeString(type));
  707.             objectDigest.update((byte) ' ');
  708.             objectDigest.update(Constants.encodeASCII(visit.data.length));
  709.             objectDigest.update((byte) 0);
  710.             objectDigest.update(visit.data);
  711.             objectDigest.digest(tempObjectId);

  713.             verifySafeObject(tempObjectId, type, visit.data);
  714.             if (isCheckObjectCollisions() && readCurs.has(tempObjectId)) {
  715.                 checkObjectCollision(tempObjectId, type, visit.data);
  716.             }

  718.             PackedObjectInfo oe;
  719.             oe = newInfo(tempObjectId, visit.delta, visit.parent.id);
  720.             oe.setOffset(visit.delta.position);
  721.             oe.setType(type);
  722.             onInflatedObjectData(oe, type, visit.data);
  723.             addObjectAndTrack(oe);
  724.             visit.id = oe;

  726.             visit.nextChild = firstChildOf(oe);
  727.             visit = visit.next();
  728.         } while (visit != null);
  729.     }

  731.     private final void checkIfTooLarge(int typeCode, long size)
  732.             throws IOException {
  733.         if (0 < maxObjectSizeLimit && maxObjectSizeLimit < size) {
  734.             switch (typeCode) {
  735.             case Constants.OBJ_COMMIT:
  736.             case Constants.OBJ_TREE:
  737.             case Constants.OBJ_BLOB:
  738.             case Constants.OBJ_TAG:
  739.                 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);

  741.             case Constants.OBJ_OFS_DELTA:
  742.             case Constants.OBJ_REF_DELTA:
  743.                 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);

  745.             default:
  746.                 throw new IOException(MessageFormat.format(
  747.                         JGitText.get().unknownObjectType,
  748.                         Integer.valueOf(typeCode)));
  749.             }
  750.         }
  751.         if (size > Integer.MAX_VALUE - 8) {
  752.             throw new TooLargeObjectInPackException(size, Integer.MAX_VALUE - 8);
  753.         }
  754.     }

  756.     /**
  757.      * Read the header of the current object.
  758.      * <p>
  759.      * After the header has been parsed, this method automatically invokes
  760.      * {@link #onObjectHeader(Source, byte[], int, int)} to allow the
  761.      * implementation to update its internal checksums for the bytes read.
  762.      * <p>
  763.      * When this method returns the database will be positioned on the first
  764.      * byte of the deflated data stream.
  765.      *
  766.      * @param info
  767.      *            the info object to populate.
  768.      * @return {@code info}, after populating.
  769.      * @throws java.io.IOException
  770.      *             the size cannot be read.
  771.      */
  772.     protected ObjectTypeAndSize readObjectHeader(ObjectTypeAndSize info)
  773.             throws IOException {
  774.         int hdrPtr = 0;
  775.         int c = readFrom(Source.DATABASE);
  776.         hdrBuf[hdrPtr++] = (byte) c;

  778.         info.type = (c >> 4) & 7;
  779.         long sz = c & 15;
  780.         int shift = 4;
  781.         while ((c & 0x80) != 0) {
  782.             c = readFrom(Source.DATABASE);
  783.             hdrBuf[hdrPtr++] = (byte) c;
  784.             sz += ((long) (c & 0x7f)) << shift;
  785.             shift += 7;
  786.         }
  787.         info.size = sz;

  789.         switch (info.type) {
  790.         case Constants.OBJ_COMMIT:
  791.         case Constants.OBJ_TREE:
  792.         case Constants.OBJ_BLOB:
  793.         case Constants.OBJ_TAG:
  794.             onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  795.             break;

  797.         case Constants.OBJ_OFS_DELTA:
  798.             c = readFrom(Source.DATABASE);
  799.             hdrBuf[hdrPtr++] = (byte) c;
  800.             while ((c & 128) != 0) {
  801.                 c = readFrom(Source.DATABASE);
  802.                 hdrBuf[hdrPtr++] = (byte) c;
  803.             }
  804.             onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  805.             break;

  807.         case Constants.OBJ_REF_DELTA:
  808.             System.arraycopy(buf, fill(Source.DATABASE, 20), hdrBuf, hdrPtr, 20);
  809.             hdrPtr += 20;
  810.             use(20);
  811.             onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  812.             break;

  814.         default:
  815.             throw new IOException(MessageFormat.format(
  816.                     JGitText.get().unknownObjectType,
  817.                     Integer.valueOf(info.type)));
  818.         }
  819.         return info;
  820.     }

  822.     private UnresolvedDelta removeBaseById(AnyObjectId id) {
  823.         final DeltaChain d = baseById.get(id);
  824.         return d != null ? d.remove() : null;
  825.     }

  827.     private static UnresolvedDelta reverse(UnresolvedDelta c) {
  828.         UnresolvedDelta tail = null;
  829.         while (c != null) {
  830.             final UnresolvedDelta n = c.next;
  831.             c.next = tail;
  832.             tail = c;
  833.             c = n;
  834.         }
  835.         return tail;
  836.     }

  838.     private UnresolvedDelta firstChildOf(PackedObjectInfo oe) {
  839.         UnresolvedDelta a = reverse(removeBaseById(oe));
  840.         UnresolvedDelta b = reverse(baseByPos.remove(oe.getOffset()));

  842.         if (a == null)
  843.             return b;
  844.         if (b == null)
  845.             return a;

  847.         UnresolvedDelta first = null;
  848.         UnresolvedDelta last = null;
  849.         while (a != null || b != null) {
  850.             UnresolvedDelta curr;
  851.             if (b == null || (a != null && a.position < b.position)) {
  852.                 curr = a;
  853.                 a = a.next;
  854.             } else {
  855.                 curr = b;
  856.                 b = b.next;
  857.             }
  858.             if (last != null)
  859.                 last.next = curr;
  860.             else
  861.                 first = curr;
  862.             last = curr;
  863.             curr.next = null;
  864.         }
  865.         return first;
  866.     }

  868.     private void resolveDeltasWithExternalBases(ProgressMonitor progress)
  869.             throws IOException {
  870.         growEntries(baseById.size());

  872.         if (needBaseObjectIds)
  873.             baseObjectIds = new ObjectIdSubclassMap<>();

  875.         final List<DeltaChain> missing = new ArrayList<>(64);
  876.         for (DeltaChain baseId : baseById) {
  877.             if (baseId.head == null)
  878.                 continue;

  880.             if (needBaseObjectIds)
  881.                 baseObjectIds.add(baseId);

  883.             final ObjectLoader ldr;
  884.             try {
  885.                 ldr = readCurs.open(baseId);
  886.             } catch (MissingObjectException notFound) {
  887.                 missing.add(baseId);
  888.                 continue;
  889.             }

  891.             final DeltaVisit visit = new DeltaVisit();
  892.             visit.data = ldr.getCachedBytes(Integer.MAX_VALUE);
  893.             visit.id = baseId;
  894.             final int typeCode = ldr.getType();
  895.             final PackedObjectInfo oe = newInfo(baseId, null, null);
  896.             oe.setType(typeCode);
  897.             if (onAppendBase(typeCode, visit.data, oe))
  898.                 entries[entryCount++] = oe;
  899.             visit.nextChild = firstChildOf(oe);
  900.             resolveDeltas(visit.next(), typeCode,
  901.                     new ObjectTypeAndSize(), progress);

  903.             if (progress.isCancelled())
  904.                 throw new IOException(
  905.                         JGitText.get().downloadCancelledDuringIndexing);
  906.         }

  908.         for (DeltaChain base : missing) {
  909.             if (base.head != null)
  910.                 throw new MissingObjectException(base, "delta base"); //$NON-NLS-1$
  911.         }

  913.         onEndThinPack();
  914.     }

  916.     private void growEntries(int extraObjects) {
  917.         final PackedObjectInfo[] ne;

  919.         ne = new PackedObjectInfo[(int) expectedObjectCount + extraObjects];
  920.         System.arraycopy(entries, 0, ne, 0, entryCount);
  921.         entries = ne;
  922.     }

  924.     private void readPackHeader() throws IOException {
  925.         if (expectDataAfterPackFooter) {
  926.             if (!in.markSupported())
  927.                 throw new IOException(
  928.                         JGitText.get().inputStreamMustSupportMark);
  929.             in.mark(buf.length);
  930.         }

  932.         final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
  933.         final int p = fill(Source.INPUT, hdrln);
  934.         for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
  935.             if (buf[p + k] != Constants.PACK_SIGNATURE[k])
  936.                 throw new IOException(JGitText.get().notAPACKFile);

  938.         final long vers = NB.decodeUInt32(buf, p + 4);
  939.         if (vers != 2 && vers != 3)
  940.             throw new IOException(MessageFormat.format(
  941.                     JGitText.get().unsupportedPackVersion, Long.valueOf(vers)));
  942.         final long objectCount = NB.decodeUInt32(buf, p + 8);
  943.         use(hdrln);
  944.         setExpectedObjectCount(objectCount);
  945.         onPackHeader(objectCount);
  946.     }

  948.     private void readPackFooter() throws IOException {
  949.         sync();
  950.         final byte[] actHash = packDigest.digest();

  952.         final int c = fill(Source.INPUT, 20);
  953.         final byte[] srcHash = new byte[20];
  954.         System.arraycopy(buf, c, srcHash, 0, 20);
  955.         use(20);

  957.         if (bAvail != 0 && !expectDataAfterPackFooter)
  958.             throw new CorruptObjectException(MessageFormat.format(
  959.                     JGitText.get().expectedEOFReceived,
  960.                     "\\x" + Integer.toHexString(buf[bOffset] & 0xff))); //$NON-NLS-1$
  961.         if (isCheckEofAfterPackFooter()) {
  962.             int eof = in.read();
  963.             if (0 <= eof)
  964.                 throw new CorruptObjectException(MessageFormat.format(
  965.                         JGitText.get().expectedEOFReceived,
  966.                         "\\x" + Integer.toHexString(eof))); //$NON-NLS-1$
  967.         } else if (bAvail > 0 && expectDataAfterPackFooter) {
  968.             in.reset();
  969.             IO.skipFully(in, bOffset);
  970.         }

  972.         if (!Arrays.equals(actHash, srcHash))
  973.             throw new CorruptObjectException(
  974.                     JGitText.get().corruptObjectPackfileChecksumIncorrect);

  976.         onPackFooter(srcHash);
  977.     }

  979.     // Cleanup all resources associated with our input parsing.
  980.     private void endInput() {
  981.         stats.setNumBytesRead(streamPosition());
  982.         in = null;
  983.     }

  985.     // Read one entire object or delta from the input.
  986.     private void indexOneObject() throws IOException {
  987.         final long streamPosition = streamPosition();

  989.         int hdrPtr = 0;
  990.         int c = readFrom(Source.INPUT);
  991.         hdrBuf[hdrPtr++] = (byte) c;

  993.         final int typeCode = (c >> 4) & 7;
  994.         long sz = c & 15;
  995.         int shift = 4;
  996.         while ((c & 0x80) != 0) {
  997.             c = readFrom(Source.INPUT);
  998.             hdrBuf[hdrPtr++] = (byte) c;
  999.             sz += ((long) (c & 0x7f)) << shift;
  1000.             shift += 7;
  1001.         }

  1003.         checkIfTooLarge(typeCode, sz);

  1005.         switch (typeCode) {
  1006.         case Constants.OBJ_COMMIT:
  1007.         case Constants.OBJ_TREE:
  1008.         case Constants.OBJ_BLOB:
  1009.         case Constants.OBJ_TAG:
  1010.             stats.addWholeObject(typeCode);
  1011.             onBeginWholeObject(streamPosition, typeCode, sz);
  1012.             onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  1013.             whole(streamPosition, typeCode, sz);
  1014.             break;

  1016.         case Constants.OBJ_OFS_DELTA: {
  1017.             stats.addOffsetDelta();
  1018.             c = readFrom(Source.INPUT);
  1019.             hdrBuf[hdrPtr++] = (byte) c;
  1020.             long ofs = c & 127;
  1021.             while ((c & 128) != 0) {
  1022.                 ofs += 1;
  1023.                 c = readFrom(Source.INPUT);
  1024.                 hdrBuf[hdrPtr++] = (byte) c;
  1025.                 ofs <<= 7;
  1026.                 ofs += (c & 127);
  1027.             }
  1028.             final long base = streamPosition - ofs;
  1029.             onBeginOfsDelta(streamPosition, base, sz);
  1030.             onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  1031.             inflateAndSkip(Source.INPUT, sz);
  1032.             UnresolvedDelta n = onEndDelta();
  1033.             n.position = streamPosition;
  1034.             n.next = baseByPos.put(base, n);
  1035.             deltaCount++;
  1036.             break;
  1037.         }

  1039.         case Constants.OBJ_REF_DELTA: {
  1040.             stats.addRefDelta();
  1041.             c = fill(Source.INPUT, 20);
  1042.             final ObjectId base = ObjectId.fromRaw(buf, c);
  1043.             System.arraycopy(buf, c, hdrBuf, hdrPtr, 20);
  1044.             hdrPtr += 20;
  1045.             use(20);
  1046.             DeltaChain r = baseById.get(base);
  1047.             if (r == null) {
  1048.                 r = new DeltaChain(base);
  1049.                 baseById.add(r);
  1050.             }
  1051.             onBeginRefDelta(streamPosition, base, sz);
  1052.             onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  1053.             inflateAndSkip(Source.INPUT, sz);
  1054.             UnresolvedDelta n = onEndDelta();
  1055.             n.position = streamPosition;
  1056.             r.add(n);
  1057.             deltaCount++;
  1058.             break;
  1059.         }

  1061.         default:
  1062.             throw new IOException(
  1063.                     MessageFormat.format(JGitText.get().unknownObjectType,
  1064.                             Integer.valueOf(typeCode)));
  1065.         }
  1066.     }

  1068.     private void whole(long pos, int type, long sz)
  1069.             throws IOException {
  1070.         SHA1 objectDigest = objectHasher.reset();
  1071.         objectDigest.update(Constants.encodedTypeString(type));
  1072.         objectDigest.update((byte) ' ');
  1073.         objectDigest.update(Constants.encodeASCII(sz));
  1074.         objectDigest.update((byte) 0);

  1076.         final byte[] data;
  1077.         if (type == Constants.OBJ_BLOB) {
  1078.             byte[] readBuffer = buffer();
  1079.             BlobObjectChecker checker = null;
  1080.             if (objCheck != null) {
  1081.                 checker = objCheck.newBlobObjectChecker();
  1082.             }
  1083.             if (checker == null) {
  1084.                 checker = BlobObjectChecker.NULL_CHECKER;
  1085.             }
  1086.             long cnt = 0;
  1087.             try (InputStream inf = inflate(Source.INPUT, sz)) {
  1088.                 while (cnt < sz) {
  1089.                     int r = inf.read(readBuffer);
  1090.                     if (r <= 0)
  1091.                         break;
  1092.                     objectDigest.update(readBuffer, 0, r);
  1093.                     checker.update(readBuffer, 0, r);
  1094.                     cnt += r;
  1095.                 }
  1096.             }
  1097.             objectDigest.digest(tempObjectId);
  1098.             checker.endBlob(tempObjectId);
  1099.             data = null;
  1100.         } else {
  1101.             data = inflateAndReturn(Source.INPUT, sz);
  1102.             objectDigest.update(data);
  1103.             objectDigest.digest(tempObjectId);
  1104.             verifySafeObject(tempObjectId, type, data);
  1105.         }

  1107.         PackedObjectInfo obj = newInfo(tempObjectId, null, null);
  1108.         obj.setOffset(pos);
  1109.         obj.setType(type);
  1110.         onEndWholeObject(obj);
  1111.         if (data != null)
  1112.             onInflatedObjectData(obj, type, data);
  1113.         addObjectAndTrack(obj);

  1115.         if (isCheckObjectCollisions()) {
  1116.             collisionCheckObjs.add(obj);
  1117.         }
  1118.     }

  1120.     /**
  1121.      * Verify the integrity of the object.
  1122.      *
  1123.      * @param id
  1124.      *            identity of the object to be checked.
  1125.      * @param type
  1126.      *            the type of the object.
  1127.      * @param data
  1128.      *            raw content of the object.
  1129.      * @throws org.eclipse.jgit.errors.CorruptObjectException
  1130.      * @since 4.9
  1131.      */
  1132.     protected void verifySafeObject(final AnyObjectId id, final int type,
  1133.             final byte[] data) throws CorruptObjectException {
  1134.         if (objCheck != null) {
  1135.             try {
  1136.                 objCheck.check(id, type, data);
  1137.             } catch (CorruptObjectException e) {
  1138.                 if (e.getErrorType() != null) {
  1139.                     throw e;
  1140.                 }
  1141.                 throw new CorruptObjectException(
  1142.                         MessageFormat.format(JGitText.get().invalidObject,
  1143.                                 Constants.typeString(type), id.name(),
  1144.                                 e.getMessage()),
  1145.                         e);
  1146.             }
  1147.         }
  1148.     }

  1150.     private void checkObjectCollision() throws IOException {
  1151.         for (PackedObjectInfo obj : collisionCheckObjs) {
  1152.             if (!readCurs.has(obj)) {
  1153.                 continue;
  1154.             }
  1155.             checkObjectCollision(obj);
  1156.         }
  1157.     }

  1159.     private void checkObjectCollision(PackedObjectInfo obj)
  1160.             throws IOException {
  1161.         ObjectTypeAndSize info = openDatabase(obj, new ObjectTypeAndSize());
  1162.         final byte[] readBuffer = buffer();
  1163.         final byte[] curBuffer = new byte[readBuffer.length];
  1164.         long sz = info.size;
  1165.         try (ObjectStream cur = readCurs.open(obj, info.type).openStream()) {
  1166.             if (cur.getSize() != sz) {
  1167.                 throw new IOException(MessageFormat.format(
  1168.                         JGitText.get().collisionOn, obj.name()));
  1169.             }
  1170.             try (InputStream pck = inflate(Source.DATABASE, sz)) {
  1171.                 while (0 < sz) {
  1172.                     int n = (int) Math.min(readBuffer.length, sz);
  1173.                     IO.readFully(cur, curBuffer, 0, n);
  1174.                     IO.readFully(pck, readBuffer, 0, n);
  1175.                     for (int i = 0; i < n; i++) {
  1176.                         if (curBuffer[i] != readBuffer[i]) {
  1177.                             throw new IOException(MessageFormat.format(
  1178.                                     JGitText.get().collisionOn, obj.name()));
  1179.                         }
  1180.                     }
  1181.                     sz -= n;
  1182.                 }
  1183.             }
  1184.         } catch (MissingObjectException notLocal) {
  1185.             // This is OK, we don't have a copy of the object locally
  1186.             // but the API throws when we try to read it as usually it's
  1187.             // an error to read something that doesn't exist.
  1188.         }
  1189.     }

  1191.     private void checkObjectCollision(AnyObjectId obj, int type, byte[] data)
  1192.             throws IOException {
  1193.         try {
  1194.             final ObjectLoader ldr = readCurs.open(obj, type);
  1195.             final byte[] existingData = ldr.getCachedBytes(data.length);
  1196.             if (!Arrays.equals(data, existingData)) {
  1197.                 throw new IOException(MessageFormat.format(
  1198.                         JGitText.get().collisionOn, obj.name()));
  1199.             }
  1200.         } catch (MissingObjectException notLocal) {
  1201.             // This is OK, we don't have a copy of the object locally
  1202.             // but the API throws when we try to read it as usually its
  1203.             // an error to read something that doesn't exist.
  1204.         }
  1205.     }

  1207.     /** @return current position of the input stream being parsed. */
  1208.     private long streamPosition() {
  1209.         return bBase + bOffset;
  1210.     }

  1212.     private ObjectTypeAndSize openDatabase(PackedObjectInfo obj,
  1213.             ObjectTypeAndSize info) throws IOException {
  1214.         bOffset = 0;
  1215.         bAvail = 0;
  1216.         return seekDatabase(obj, info);
  1217.     }

  1219.     private ObjectTypeAndSize openDatabase(UnresolvedDelta delta,
  1220.             ObjectTypeAndSize info) throws IOException {
  1221.         bOffset = 0;
  1222.         bAvail = 0;
  1223.         return seekDatabase(delta, info);
  1224.     }

  1226.     // Consume exactly one byte from the buffer and return it.
  1227.     private int readFrom(Source src) throws IOException {
  1228.         if (bAvail == 0)
  1229.             fill(src, 1);
  1230.         bAvail--;
  1231.         return buf[bOffset++] & 0xff;
  1232.     }

  1234.     // Consume cnt bytes from the buffer.
  1235.     void use(int cnt) {
  1236.         bOffset += cnt;
  1237.         bAvail -= cnt;
  1238.     }

  1240.     // Ensure at least need bytes are available in {@link #buf}.
  1241.     int fill(Source src, int need) throws IOException {
  1242.         while (bAvail < need) {
  1243.             int next = bOffset + bAvail;
  1244.             int free = buf.length - next;
  1245.             if (free + bAvail < need) {
  1246.                 switch (src) {
  1247.                 case INPUT:
  1248.                     sync();
  1249.                     break;
  1250.                 case DATABASE:
  1251.                     if (bAvail > 0)
  1252.                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
  1253.                     bOffset = 0;
  1254.                     break;
  1255.                 }
  1256.                 next = bAvail;
  1257.                 free = buf.length - next;
  1258.             }
  1259.             switch (src) {
  1260.             case INPUT:
  1261.                 next = in.read(buf, next, free);
  1262.                 break;
  1263.             case DATABASE:
  1264.                 next = readDatabase(buf, next, free);
  1265.                 break;
  1266.             }
  1267.             if (next <= 0)
  1268.                 throw new EOFException(
  1269.                         JGitText.get().packfileIsTruncatedNoParam);
  1270.             bAvail += next;
  1271.         }
  1272.         return bOffset;
  1273.     }

  1275.     // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
  1276.     private void sync() throws IOException {
  1277.         packDigest.update(buf, 0, bOffset);
  1278.         onStoreStream(buf, 0, bOffset);
  1279.         if (expectDataAfterPackFooter) {
  1280.             if (bAvail > 0) {
  1281.                 in.reset();
  1282.                 IO.skipFully(in, bOffset);
  1283.                 bAvail = 0;
  1284.             }
  1285.             in.mark(buf.length);
  1286.         } else if (bAvail > 0)
  1287.             System.arraycopy(buf, bOffset, buf, 0, bAvail);
  1288.         bBase += bOffset;
  1289.         bOffset = 0;
  1290.     }

  1292.     /**
  1293.      * Get a temporary byte array for use by the caller.
  1294.      *
  1295.      * @return a temporary byte array for use by the caller.
  1296.      */
  1297.     protected byte[] buffer() {
  1298.         return tempBuffer;
  1299.     }

  1301.     /**
  1302.      * Construct a PackedObjectInfo instance for this parser.
  1303.      *
  1304.      * @param id
  1305.      *            identity of the object to be tracked.
  1306.      * @param delta
  1307.      *            if the object was previously an unresolved delta, this is the
  1308.      *            delta object that was tracking it. Otherwise null.
  1309.      * @param deltaBase
  1310.      *            if the object was previously an unresolved delta, this is the
  1311.      *            ObjectId of the base of the delta. The base may be outside of
  1312.      *            the pack stream if the stream was a thin-pack.
  1313.      * @return info object containing this object's data.
  1314.      */
  1315.     protected PackedObjectInfo newInfo(AnyObjectId id, UnresolvedDelta delta,
  1316.             ObjectId deltaBase) {
  1317.         PackedObjectInfo oe = new PackedObjectInfo(id);
  1318.         if (delta != null)
  1319.             oe.setCRC(delta.crc);
  1320.         return oe;
  1321.     }

  1323.     /**
  1324.      * Set the expected number of objects in the pack stream.
  1325.      * <p>
  1326.      * The object count in the pack header is not always correct for some Dfs
  1327.      * pack files. e.g. INSERT pack always assume 1 object in the header since
  1328.      * the actual object count is unknown when the pack is written.
  1329.      * <p>
  1330.      * If external implementation wants to overwrite the expectedObjectCount,
  1331.      * they should call this method during {@link #onPackHeader(long)}.
  1332.      *
  1333.      * @param expectedObjectCount a long.
  1334.      * @since 4.9
  1335.      */
  1336.     protected void setExpectedObjectCount(long expectedObjectCount) {
  1337.         this.expectedObjectCount = expectedObjectCount;
  1338.     }

  1340.     /**
  1341.      * Store bytes received from the raw stream.
  1342.      * <p>
  1343.      * This method is invoked during {@link #parse(ProgressMonitor)} as data is
  1344.      * consumed from the incoming stream. Implementors may use this event to
  1345.      * archive the raw incoming stream to the destination repository in large
  1346.      * chunks, without paying attention to object boundaries.
  1347.      * <p>
  1348.      * The only component of the pack not supplied to this method is the last 20
  1349.      * bytes of the pack that comprise the trailing SHA-1 checksum. Those are
  1350.      * passed to {@link #onPackFooter(byte[])}.
  1351.      *
  1352.      * @param raw
  1353.      *            buffer to copy data out of.
  1354.      * @param pos
  1355.      *            first offset within the buffer that is valid.
  1356.      * @param len
  1357.      *            number of bytes in the buffer that are valid.
  1358.      * @throws java.io.IOException
  1359.      *             the stream cannot be archived.
  1360.      */
  1361.     protected abstract void onStoreStream(byte[] raw, int pos, int len)
  1362.             throws IOException;

  1364.     /**
  1365.      * Store (and/or checksum) an object header.
  1366.      * <p>
  1367.      * Invoked after any of the {@code onBegin()} events. The entire header is
  1368.      * supplied in a single invocation, before any object data is supplied.
  1369.      *
  1370.      * @param src
  1371.      *            where the data came from
  1372.      * @param raw
  1373.      *            buffer to read data from.
  1374.      * @param pos
  1375.      *            first offset within buffer that is valid.
  1376.      * @param len
  1377.      *            number of bytes in buffer that are valid.
  1378.      * @throws java.io.IOException
  1379.      *             the stream cannot be archived.
  1380.      */
  1381.     protected abstract void onObjectHeader(Source src, byte[] raw, int pos,
  1382.             int len) throws IOException;

  1384.     /**
  1385.      * Store (and/or checksum) a portion of an object's data.
  1386.      * <p>
  1387.      * This method may be invoked multiple times per object, depending on the
  1388.      * size of the object, the size of the parser's internal read buffer, and
  1389.      * the alignment of the object relative to the read buffer.
  1390.      * <p>
  1391.      * Invoked after {@link #onObjectHeader(Source, byte[], int, int)}.
  1392.      *
  1393.      * @param src
  1394.      *            where the data came from
  1395.      * @param raw
  1396.      *            buffer to read data from.
  1397.      * @param pos
  1398.      *            first offset within buffer that is valid.
  1399.      * @param len
  1400.      *            number of bytes in buffer that are valid.
  1401.      * @throws java.io.IOException
  1402.      *             the stream cannot be archived.
  1403.      */
  1404.     protected abstract void onObjectData(Source src, byte[] raw, int pos,
  1405.             int len) throws IOException;

  1407.     /**
  1408.      * Invoked for commits, trees, tags, and small blobs.
  1409.      *
  1410.      * @param obj
  1411.      *            the object info, populated.
  1412.      * @param typeCode
  1413.      *            the type of the object.
  1414.      * @param data
  1415.      *            inflated data for the object.
  1416.      * @throws java.io.IOException
  1417.      *             the object cannot be archived.
  1418.      */
  1419.     protected abstract void onInflatedObjectData(PackedObjectInfo obj,
  1420.             int typeCode, byte[] data) throws IOException;

  1422.     /**
  1423.      * Provide the implementation with the original stream's pack header.
  1424.      *
  1425.      * @param objCnt
  1426.      *            number of objects expected in the stream.
  1427.      * @throws java.io.IOException
  1428.      *             the implementation refuses to work with this many objects.
  1429.      */
  1430.     protected abstract void onPackHeader(long objCnt) throws IOException;

  1432.     /**
  1433.      * Provide the implementation with the original stream's pack footer.
  1434.      *
  1435.      * @param hash
  1436.      *            the trailing 20 bytes of the pack, this is a SHA-1 checksum of
  1437.      *            all of the pack data.
  1438.      * @throws java.io.IOException
  1439.      *             the stream cannot be archived.
  1440.      */
  1441.     protected abstract void onPackFooter(byte[] hash) throws IOException;

  1443.     /**
  1444.      * Provide the implementation with a base that was outside of the pack.
  1445.      * <p>
  1446.      * This event only occurs on a thin pack for base objects that were outside
  1447.      * of the pack and came from the local repository. Usually an implementation
  1448.      * uses this event to compress the base and append it onto the end of the
  1449.      * pack, so the pack stays self-contained.
  1450.      *
  1451.      * @param typeCode
  1452.      *            type of the base object.
  1453.      * @param data
  1454.      *            complete content of the base object.
  1455.      * @param info
  1456.      *            packed object information for this base. Implementors must
  1457.      *            populate the CRC and offset members if returning true.
  1458.      * @return true if the {@code info} should be included in the object list
  1459.      *         returned by {@link #getSortedObjectList(Comparator)}, false if it
  1460.      *         should not be included.
  1461.      * @throws java.io.IOException
  1462.      *             the base could not be included into the pack.
  1463.      */
  1464.     protected abstract boolean onAppendBase(int typeCode, byte[] data,
  1465.             PackedObjectInfo info) throws IOException;

  1467.     /**
  1468.      * Event indicating a thin pack has been completely processed.
  1469.      * <p>
  1470.      * This event is invoked only if a thin pack has delta references to objects
  1471.      * external from the pack. The event is called after all of those deltas
  1472.      * have been resolved.
  1473.      *
  1474.      * @throws java.io.IOException
  1475.      *             the pack cannot be archived.
  1476.      */
  1477.     protected abstract void onEndThinPack() throws IOException;

  1479.     /**
  1480.      * Reposition the database to re-read a previously stored object.
  1481.      * <p>
  1482.      * If the database is computing CRC-32 checksums for object data, it should
  1483.      * reset its internal CRC instance during this method call.
  1484.      *
  1485.      * @param obj
  1486.      *            the object position to begin reading from. This is from
  1487.      *            {@link #newInfo(AnyObjectId, UnresolvedDelta, ObjectId)}.
  1488.      * @param info
  1489.      *            object to populate with type and size.
  1490.      * @return the {@code info} object.
  1491.      * @throws java.io.IOException
  1492.      *             the database cannot reposition to this location.
  1493.      */
  1494.     protected abstract ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
  1495.             ObjectTypeAndSize info) throws IOException;

  1497.     /**
  1498.      * Reposition the database to re-read a previously stored object.
  1499.      * <p>
  1500.      * If the database is computing CRC-32 checksums for object data, it should
  1501.      * reset its internal CRC instance during this method call.
  1502.      *
  1503.      * @param delta
  1504.      *            the object position to begin reading from. This is an instance
  1505.      *            previously returned by {@link #onEndDelta()}.
  1506.      * @param info
  1507.      *            object to populate with type and size.
  1508.      * @return the {@code info} object.
  1509.      * @throws java.io.IOException
  1510.      *             the database cannot reposition to this location.
  1511.      */
  1512.     protected abstract ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
  1513.             ObjectTypeAndSize info) throws IOException;

  1515.     /**
  1516.      * Read from the database's current position into the buffer.
  1517.      *
  1518.      * @param dst
  1519.      *            the buffer to copy read data into.
  1520.      * @param pos
  1521.      *            position within {@code dst} to start copying data into.
  1522.      * @param cnt
  1523.      *            ideal target number of bytes to read. Actual read length may
  1524.      *            be shorter.
  1525.      * @return number of bytes stored.
  1526.      * @throws java.io.IOException
  1527.      *             the database cannot be accessed.
  1528.      */
  1529.     protected abstract int readDatabase(byte[] dst, int pos, int cnt)
  1530.             throws IOException;

  1532.     /**
  1533.      * Check the current CRC matches the expected value.
  1534.      * <p>
  1535.      * This method is invoked when an object is read back in from the database
  1536.      * and its data is used during delta resolution. The CRC is validated after
  1537.      * the object has been fully read, allowing the parser to verify there was
  1538.      * no silent data corruption.
  1539.      * <p>
  1540.      * Implementations are free to ignore this check by always returning true if
  1541.      * they are performing other data integrity validations at a lower level.
  1542.      *
  1543.      * @param oldCRC
  1544.      *            the prior CRC that was recorded during the first scan of the
  1545.      *            object from the pack stream.
  1546.      * @return true if the CRC matches; false if it does not.
  1547.      */
  1548.     protected abstract boolean checkCRC(int oldCRC);

  1550.     /**
  1551.      * Event notifying the start of an object stored whole (not as a delta).
  1552.      *
  1553.      * @param streamPosition
  1554.      *            position of this object in the incoming stream.
  1555.      * @param type
  1556.      *            type of the object; one of
  1557.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_COMMIT},
  1558.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_TREE},
  1559.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_BLOB}, or
  1560.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_TAG}.
  1561.      * @param inflatedSize
  1562.      *            size of the object when fully inflated. The size stored within
  1563.      *            the pack may be larger or smaller, and is not yet known.
  1564.      * @throws java.io.IOException
  1565.      *             the object cannot be recorded.
  1566.      */
  1567.     protected abstract void onBeginWholeObject(long streamPosition, int type,
  1568.             long inflatedSize) throws IOException;

  1570.     /**
  1571.      * Event notifying the current object.
  1572.      *
  1573.      *@param info
  1574.      *            object information.
  1575.      * @throws java.io.IOException
  1576.      *             the object cannot be recorded.
  1577.      */
  1578.     protected abstract void onEndWholeObject(PackedObjectInfo info)
  1579.             throws IOException;

  1581.     /**
  1582.      * Event notifying start of a delta referencing its base by offset.
  1583.      *
  1584.      * @param deltaStreamPosition
  1585.      *            position of this object in the incoming stream.
  1586.      * @param baseStreamPosition
  1587.      *            position of the base object in the incoming stream. The base
  1588.      *            must be before the delta, therefore {@code baseStreamPosition
  1589.      *            &lt; deltaStreamPosition}. This is <b>not</b> the position
  1590.      *            returned by a prior end object event.
  1591.      * @param inflatedSize
  1592.      *            size of the delta when fully inflated. The size stored within
  1593.      *            the pack may be larger or smaller, and is not yet known.
  1594.      * @throws java.io.IOException
  1595.      *             the object cannot be recorded.
  1596.      */
  1597.     protected abstract void onBeginOfsDelta(long deltaStreamPosition,
  1598.             long baseStreamPosition, long inflatedSize) throws IOException;

  1600.     /**
  1601.      * Event notifying start of a delta referencing its base by ObjectId.
  1602.      *
  1603.      * @param deltaStreamPosition
  1604.      *            position of this object in the incoming stream.
  1605.      * @param baseId
  1606.      *            name of the base object. This object may be later in the
  1607.      *            stream, or might not appear at all in the stream (in the case
  1608.      *            of a thin-pack).
  1609.      * @param inflatedSize
  1610.      *            size of the delta when fully inflated. The size stored within
  1611.      *            the pack may be larger or smaller, and is not yet known.
  1612.      * @throws java.io.IOException
  1613.      *             the object cannot be recorded.
  1614.      */
  1615.     protected abstract void onBeginRefDelta(long deltaStreamPosition,
  1616.             AnyObjectId baseId, long inflatedSize) throws IOException;

  1618.     /**
  1619.      * Event notifying the current object.
  1620.      *
  1621.      *@return object information that must be populated with at least the
  1622.      *         offset.
  1623.      * @throws java.io.IOException
  1624.      *             the object cannot be recorded.
  1625.      */
  1626.     protected UnresolvedDelta onEndDelta() throws IOException {
  1627.         return new UnresolvedDelta();
  1628.     }

  1630.     /** Type and size information about an object in the database buffer. */
  1631.     public static class ObjectTypeAndSize {
  1632.         /** The type of the object. */
  1633.         public int type;

  1635.         /** The inflated size of the object. */
  1636.         public long size;
  1637.     }

  1639.     private void inflateAndSkip(Source src, long inflatedSize)
  1640.             throws IOException {
  1641.         try (InputStream inf = inflate(src, inflatedSize)) {
  1642.             IO.skipFully(inf, inflatedSize);
  1643.         }
  1644.     }

  1646.     private byte[] inflateAndReturn(Source src, long inflatedSize)
  1647.             throws IOException {
  1648.         final byte[] dst = new byte[(int) inflatedSize];
  1649.         try (InputStream inf = inflate(src, inflatedSize)) {
  1650.             IO.readFully(inf, dst, 0, dst.length);
  1651.         }
  1652.         return dst;
  1653.     }

  1655.     private InputStream inflate(Source src, long inflatedSize)
  1656.             throws IOException {
  1657.         inflater.open(src, inflatedSize);
  1658.         return inflater;
  1659.     }

  1661.     private static class DeltaChain extends ObjectIdOwnerMap.Entry {
  1662.         UnresolvedDelta head;

  1664.         DeltaChain(AnyObjectId id) {
  1665.             super(id);
  1666.         }

  1668.         UnresolvedDelta remove() {
  1669.             final UnresolvedDelta r = head;
  1670.             if (r != null)
  1671.                 head = null;
  1672.             return r;
  1673.         }

  1675.         void add(UnresolvedDelta d) {
  1676.             d.next = head;
  1677.             head = d;
  1678.         }
  1679.     }

  1681.     /** Information about an unresolved delta in this pack stream. */
  1682.     public static class UnresolvedDelta {
  1683.         long position;

  1685.         int crc;

  1687.         UnresolvedDelta next;

  1689.         /** @return offset within the input stream. */
  1690.         public long getOffset() {
  1691.             return position;
  1692.         }

  1694.         /** @return the CRC-32 checksum of the stored delta data. */
  1695.         public int getCRC() {
  1696.             return crc;
  1697.         }

  1699.         /**
  1700.          * @param crc32
  1701.          *            the CRC-32 checksum of the stored delta data.
  1702.          */
  1703.         public void setCRC(int crc32) {
  1704.             crc = crc32;
  1705.         }
  1706.     }

  1708.     private static class DeltaVisit {
  1709.         final UnresolvedDelta delta;

  1711.         ObjectId id;

  1713.         byte[] data;

  1715.         DeltaVisit parent;

  1717.         UnresolvedDelta nextChild;

  1719.         DeltaVisit() {
  1720.             this.delta = null; // At the root of the stack we have a base.
  1721.         }

  1723.         DeltaVisit(DeltaVisit parent) {
  1724.             this.parent = parent;
  1725.             this.delta = parent.nextChild;
  1726.             parent.nextChild = delta.next;
  1727.         }

  1729.         DeltaVisit next() {
  1730.             // If our parent has no more children, discard it.
  1731.             if (parent != null && parent.nextChild == null) {
  1732.                 parent.data = null;
  1733.                 parent = parent.parent;
  1734.             }

  1736.             if (nextChild != null)
  1737.                 return new DeltaVisit(this);

  1739.             // If we have no child ourselves, our parent must (if it exists),
  1740.             // due to the discard rule above. With no parent, we are done.
  1741.             if (parent != null)
  1742.                 return new DeltaVisit(parent);
  1743.             return null;
  1744.         }
  1745.     }

  1747.     private void addObjectAndTrack(PackedObjectInfo oe) {
  1748.         entries[entryCount++] = oe;
  1749.         if (needNewObjectIds())
  1750.             newObjectIds.add(oe);
  1751.     }

  1753.     private class InflaterStream extends InputStream {
  1754.         private final Inflater inf;

  1756.         private final byte[] skipBuffer;

  1758.         private Source src;

  1760.         private long expectedSize;

  1762.         private long actualSize;

  1764.         private int p;

  1766.         InflaterStream() {
  1767.             inf = InflaterCache.get();
  1768.             skipBuffer = new byte[512];
  1769.         }

  1771.         void release() {
  1772.             inf.reset();
  1773.             InflaterCache.release(inf);
  1774.         }

  1776.         void open(Source source, long inflatedSize) throws IOException {
  1777.             src = source;
  1778.             expectedSize = inflatedSize;
  1779.             actualSize = 0;

  1781.             p = fill(src, 1);
  1782.             inf.setInput(buf, p, bAvail);
  1783.         }

  1785.         @Override
  1786.         public long skip(long toSkip) throws IOException {
  1787.             long n = 0;
  1788.             while (n < toSkip) {
  1789.                 final int cnt = (int) Math.min(skipBuffer.length, toSkip - n);
  1790.                 final int r = read(skipBuffer, 0, cnt);
  1791.                 if (r <= 0)
  1792.                     break;
  1793.                 n += r;
  1794.             }
  1795.             return n;
  1796.         }

  1798.         @Override
  1799.         public int read() throws IOException {
  1800.             int n = read(skipBuffer, 0, 1);
  1801.             return n == 1 ? skipBuffer[0] & 0xff : -1;
  1802.         }

  1804.         @Override
  1805.         public int read(byte[] dst, int pos, int cnt) throws IOException {
  1806.             try {
  1807.                 int n = 0;
  1808.                 while (n < cnt) {
  1809.                     int r = inf.inflate(dst, pos + n, cnt - n);
  1810.                     n += r;
  1811.                     if (inf.finished())
  1812.                         break;
  1813.                     if (inf.needsInput()) {
  1814.                         onObjectData(src, buf, p, bAvail);
  1815.                         use(bAvail);

  1817.                         p = fill(src, 1);
  1818.                         inf.setInput(buf, p, bAvail);
  1819.                     } else if (r == 0) {
  1820.                         throw new CorruptObjectException(MessageFormat.format(
  1821.                                 JGitText.get().packfileCorruptionDetected,
  1822.                                 JGitText.get().unknownZlibError));
  1823.                     }
  1824.                 }
  1825.                 actualSize += n;
  1826.                 return 0 < n ? n : -1;
  1827.             } catch (DataFormatException dfe) {
  1828.                 throw new CorruptObjectException(MessageFormat.format(JGitText
  1829.                         .get().packfileCorruptionDetected, dfe.getMessage()));
  1830.             }
  1831.         }

  1833.         @Override
  1834.         public void close() throws IOException {
  1835.             // We need to read here to enter the loop above and pump the
  1836.             // trailing checksum into the Inflater. It should return -1 as the
  1837.             // caller was supposed to consume all content.
  1838.             //
  1839.             if (read(skipBuffer) != -1 || actualSize != expectedSize) {
  1840.                 throw new CorruptObjectException(MessageFormat.format(JGitText
  1841.                         .get().packfileCorruptionDetected,
  1842.                         JGitText.get().wrongDecompressedLength));
  1843.             }

  1845.             int used = bAvail - inf.getRemaining();
  1846.             if (0 < used) {
  1847.                 onObjectData(src, buf, p, used);
  1848.                 use(used);
  1849.             }

  1851.             inf.reset();
  1852.         }
  1853.     }
  1854. }
Created with JaCoCo 0.8.4.201905082037