PackParser.java

  1. /*
  2.  * Copyright (C) 2008-2011, Google Inc.
  3.  * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com>
  4.  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
  5.  *
  6.  * This program and the accompanying materials are made available under the
  7.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  8.  * https://www.eclipse.org/org/documents/edl-v10.php.
  9.  *
  10.  * SPDX-License-Identifier: BSD-3-Clause
  11.  */

  13. package org.eclipse.jgit.transport;

  15. import java.io.EOFException;
  16. import java.io.IOException;
  17. import java.io.InputStream;
  18. import java.security.MessageDigest;
  19. import java.text.MessageFormat;
  20. import java.util.ArrayList;
  21. import java.util.Arrays;
  22. import java.util.Comparator;
  23. import java.util.List;
  24. import java.util.concurrent.TimeUnit;
  25. import java.util.zip.DataFormatException;
  26. import java.util.zip.Inflater;

  28. import org.eclipse.jgit.errors.CorruptObjectException;
  29. import org.eclipse.jgit.errors.MissingObjectException;
  30. import org.eclipse.jgit.errors.TooLargeObjectInPackException;
  31. import org.eclipse.jgit.internal.JGitText;
  32. import org.eclipse.jgit.internal.storage.file.PackLock;
  33. import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
  34. import org.eclipse.jgit.lib.AnyObjectId;
  35. import org.eclipse.jgit.lib.BatchingProgressMonitor;
  36. import org.eclipse.jgit.lib.BlobObjectChecker;
  37. import org.eclipse.jgit.lib.Constants;
  38. import org.eclipse.jgit.lib.InflaterCache;
  39. import org.eclipse.jgit.lib.MutableObjectId;
  40. import org.eclipse.jgit.lib.NullProgressMonitor;
  41. import org.eclipse.jgit.lib.ObjectChecker;
  42. import org.eclipse.jgit.lib.ObjectDatabase;
  43. import org.eclipse.jgit.lib.ObjectId;
  44. import org.eclipse.jgit.lib.ObjectIdOwnerMap;
  45. import org.eclipse.jgit.lib.ObjectIdSubclassMap;
  46. import org.eclipse.jgit.lib.ObjectLoader;
  47. import org.eclipse.jgit.lib.ObjectReader;
  48. import org.eclipse.jgit.lib.ObjectStream;
  49. import org.eclipse.jgit.lib.ProgressMonitor;
  50. import org.eclipse.jgit.util.BlockList;
  51. import org.eclipse.jgit.util.IO;
  52. import org.eclipse.jgit.util.LongMap;
  53. import org.eclipse.jgit.util.NB;
  54. import org.eclipse.jgit.util.sha1.SHA1;

  56. /**
  57.  * Parses a pack stream and imports it for an
  58.  * {@link org.eclipse.jgit.lib.ObjectInserter}.
  59.  * <p>
  60.  * Applications can acquire an instance of a parser from ObjectInserter's
  61.  * {@link org.eclipse.jgit.lib.ObjectInserter#newPackParser(InputStream)}
  62.  * method.
  63.  * <p>
  64.  * Implementations of {@link org.eclipse.jgit.lib.ObjectInserter} should
  65.  * subclass this type and provide their own logic for the various {@code on*()}
  66.  * event methods declared to be abstract.
  67.  */
  68. public abstract class PackParser {
  69.     /** Size of the internal stream buffer. */
  70.     private static final int BUFFER_SIZE = 8192;

  72.     /** Location data is being obtained from. */
  73.     public enum Source {
  74.         /** Data is read from the incoming stream. */
  75.         INPUT,

  77.         /** Data is read back from the database's buffers. */
  78.         DATABASE;
  79.     }

  81.     /** Object database used for loading existing objects. */
  82.     private final ObjectDatabase objectDatabase;

  84.     private InflaterStream inflater;

  86.     private byte[] tempBuffer;

  88.     private byte[] hdrBuf;

  90.     private final SHA1 objectHasher = SHA1.newInstance();
  91.     private final MutableObjectId tempObjectId;

  93.     private InputStream in;

  95.     byte[] buf;

  97.     /** Position in the input stream of {@code buf[0]}. */
  98.     private long bBase;

  100.     private int bOffset;

  102.     int bAvail;

  104.     private ObjectChecker objCheck;

  106.     private boolean allowThin;

  108.     private boolean checkObjectCollisions;

  110.     private boolean needBaseObjectIds;

  112.     private boolean checkEofAfterPackFooter;

  114.     private boolean expectDataAfterPackFooter;

  116.     private long expectedObjectCount;

  118.     private PackedObjectInfo[] entries;

  120.     /**
  121.      * Every object contained within the incoming pack.
  122.      * <p>
  123.      * This is a subset of {@link #entries}, as thin packs can add additional
  124.      * objects to {@code entries} by copying already existing objects from the
  125.      * repository onto the end of the thin pack to make it self-contained.
  126.      */
  127.     private ObjectIdSubclassMap<ObjectId> newObjectIds;

  129.     private int deltaCount;

  131.     private int entryCount;

  133.     private ObjectIdOwnerMap<DeltaChain> baseById;

  135.     /**
  136.      * Objects referenced by their name from deltas, that aren't in this pack.
  137.      * <p>
  138.      * This is the set of objects that were copied onto the end of this pack to
  139.      * make it complete. These objects were not transmitted by the remote peer,
  140.      * but instead were assumed to already exist in the local repository.
  141.      */
  142.     private ObjectIdSubclassMap<ObjectId> baseObjectIds;

  144.     private LongMap<UnresolvedDelta> baseByPos;

  146.     /** Objects need to be double-checked for collision after indexing. */
  147.     private BlockList<PackedObjectInfo> collisionCheckObjs;

  149.     private MessageDigest packDigest;

  151.     private ObjectReader readCurs;

  153.     /** Message to protect the pack data from garbage collection. */
  154.     private String lockMessage;

  156.     /** Git object size limit */
  157.     private long maxObjectSizeLimit;

  159.     private final ReceivedPackStatistics.Builder stats =
  160.             new ReceivedPackStatistics.Builder();

  162.     /**
  163.      * Initialize a pack parser.
  164.      *
  165.      * @param odb
  166.      *            database the parser will write its objects into.
  167.      * @param src
  168.      *            the stream the parser will read.
  169.      */
  170.     protected PackParser(ObjectDatabase odb, InputStream src) {
  171.         objectDatabase = odb.newCachedDatabase();
  172.         in = src;

  174.         inflater = new InflaterStream();
  175.         readCurs = objectDatabase.newReader();
  176.         buf = new byte[BUFFER_SIZE];
  177.         tempBuffer = new byte[BUFFER_SIZE];
  178.         hdrBuf = new byte[64];
  179.         tempObjectId = new MutableObjectId();
  180.         packDigest = Constants.newMessageDigest();
  181.         checkObjectCollisions = true;
  182.     }

  184.     /**
  185.      * Whether a thin pack (missing base objects) is permitted.
  186.      *
  187.      * @return {@code true} if a thin pack (missing base objects) is permitted.
  188.      */
  189.     public boolean isAllowThin() {
  190.         return allowThin;
  191.     }

  193.     /**
  194.      * Configure this index pack instance to allow a thin pack.
  195.      * <p>
  196.      * Thin packs are sometimes used during network transfers to allow a delta
  197.      * to be sent without a base object. Such packs are not permitted on disk.
  198.      *
  199.      * @param allow
  200.      *            true to enable a thin pack.
  201.      */
  202.     public void setAllowThin(boolean allow) {
  203.         allowThin = allow;
  204.     }

  206.     /**
  207.      * Whether received objects are verified to prevent collisions.
  208.      *
  209.      * @return if true received objects are verified to prevent collisions.
  210.      * @since 4.1
  211.      */
  212.     protected boolean isCheckObjectCollisions() {
  213.         return checkObjectCollisions;
  214.     }

  216.     /**
  217.      * Enable checking for collisions with existing objects.
  218.      * <p>
  219.      * By default PackParser looks for each received object in the repository.
  220.      * If the object already exists, the existing object is compared
  221.      * byte-for-byte with the newly received copy to ensure they are identical.
  222.      * The receive is aborted with an exception if any byte differs. This check
  223.      * is necessary to prevent an evil attacker from supplying a replacement
  224.      * object into this repository in the event that a discovery enabling SHA-1
  225.      * collisions is made.
  226.      * <p>
  227.      * This check may be very costly to perform, and some repositories may have
  228.      * other ways to segregate newly received object data. The check is enabled
  229.      * by default, but can be explicitly disabled if the implementation can
  230.      * provide the same guarantee, or is willing to accept the risks associated
  231.      * with bypassing the check.
  232.      *
  233.      * @param check
  234.      *            true to enable collision checking (strongly encouraged).
  235.      * @since 4.1
  236.      */
  237.     protected void setCheckObjectCollisions(boolean check) {
  238.         checkObjectCollisions = check;
  239.     }

  241.     /**
  242.      * Configure this index pack instance to keep track of new objects.
  243.      * <p>
  244.      * By default an index pack doesn't save the new objects that were created
  245.      * when it was instantiated. Setting this flag to {@code true} allows the
  246.      * caller to use {@link #getNewObjectIds()} to retrieve that list.
  247.      *
  248.      * @param b
  249.      *            {@code true} to enable keeping track of new objects.
  250.      */
  251.     public void setNeedNewObjectIds(boolean b) {
  252.         if (b)
  253.             newObjectIds = new ObjectIdSubclassMap<>();
  254.         else
  255.             newObjectIds = null;
  256.     }

  258.     private boolean needNewObjectIds() {
  259.         return newObjectIds != null;
  260.     }

  262.     /**
  263.      * Configure this index pack instance to keep track of the objects assumed
  264.      * for delta bases.
  265.      * <p>
  266.      * By default an index pack doesn't save the objects that were used as delta
  267.      * bases. Setting this flag to {@code true} will allow the caller to use
  268.      * {@link #getBaseObjectIds()} to retrieve that list.
  269.      *
  270.      * @param b
  271.      *            {@code true} to enable keeping track of delta bases.
  272.      */
  273.     public void setNeedBaseObjectIds(boolean b) {
  274.         this.needBaseObjectIds = b;
  275.     }

  277.     /**
  278.      * Whether the EOF should be read from the input after the footer.
  279.      *
  280.      * @return true if the EOF should be read from the input after the footer.
  281.      */
  282.     public boolean isCheckEofAfterPackFooter() {
  283.         return checkEofAfterPackFooter;
  284.     }

  286.     /**
  287.      * Ensure EOF is read from the input stream after the footer.
  288.      *
  289.      * @param b
  290.      *            true if the EOF should be read; false if it is not checked.
  291.      */
  292.     public void setCheckEofAfterPackFooter(boolean b) {
  293.         checkEofAfterPackFooter = b;
  294.     }

  296.     /**
  297.      * Whether there is data expected after the pack footer.
  298.      *
  299.      * @return true if there is data expected after the pack footer.
  300.      */
  301.     public boolean isExpectDataAfterPackFooter() {
  302.         return expectDataAfterPackFooter;
  303.     }

  305.     /**
  306.      * Set if there is additional data in InputStream after pack.
  307.      *
  308.      * @param e
  309.      *            true if there is additional data in InputStream after pack.
  310.      *            This requires the InputStream to support the mark and reset
  311.      *            functions.
  312.      */
  313.     public void setExpectDataAfterPackFooter(boolean e) {
  314.         expectDataAfterPackFooter = e;
  315.     }

  317.     /**
  318.      * Get the new objects that were sent by the user
  319.      *
  320.      * @return the new objects that were sent by the user
  321.      */
  322.     public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
  323.         if (newObjectIds != null)
  324.             return newObjectIds;
  325.         return new ObjectIdSubclassMap<>();
  326.     }

  328.     /**
  329.      * Get set of objects the incoming pack assumed for delta purposes
  330.      *
  331.      * @return set of objects the incoming pack assumed for delta purposes
  332.      */
  333.     public ObjectIdSubclassMap<ObjectId> getBaseObjectIds() {
  334.         if (baseObjectIds != null)
  335.             return baseObjectIds;
  336.         return new ObjectIdSubclassMap<>();
  337.     }

  339.     /**
  340.      * Configure the checker used to validate received objects.
  341.      * <p>
  342.      * Usually object checking isn't necessary, as Git implementations only
  343.      * create valid objects in pack files. However, additional checking may be
  344.      * useful if processing data from an untrusted source.
  345.      *
  346.      * @param oc
  347.      *            the checker instance; null to disable object checking.
  348.      */
  349.     public void setObjectChecker(ObjectChecker oc) {
  350.         objCheck = oc;
  351.     }

  353.     /**
  354.      * Configure the checker used to validate received objects.
  355.      * <p>
  356.      * Usually object checking isn't necessary, as Git implementations only
  357.      * create valid objects in pack files. However, additional checking may be
  358.      * useful if processing data from an untrusted source.
  359.      * <p>
  360.      * This is shorthand for:
  361.      *
  362.      * <pre>
  363.      * setObjectChecker(on ? new ObjectChecker() : null);
  364.      * </pre>
  365.      *
  366.      * @param on
  367.      *            true to enable the default checker; false to disable it.
  368.      */
  369.     public void setObjectChecking(boolean on) {
  370.         setObjectChecker(on ? new ObjectChecker() : null);
  371.     }

  373.     /**
  374.      * Get the message to record with the pack lock.
  375.      *
  376.      * @return the message to record with the pack lock.
  377.      */
  378.     public String getLockMessage() {
  379.         return lockMessage;
  380.     }

  382.     /**
  383.      * Set the lock message for the incoming pack data.
  384.      *
  385.      * @param msg
  386.      *            if not null, the message to associate with the incoming data
  387.      *            while it is locked to prevent garbage collection.
  388.      */
  389.     public void setLockMessage(String msg) {
  390.         lockMessage = msg;
  391.     }

  393.     /**
  394.      * Set the maximum allowed Git object size.
  395.      * <p>
  396.      * If an object is larger than the given size the pack-parsing will throw an
  397.      * exception aborting the parsing.
  398.      *
  399.      * @param limit
  400.      *            the Git object size limit. If zero then there is not limit.
  401.      */
  402.     public void setMaxObjectSizeLimit(long limit) {
  403.         maxObjectSizeLimit = limit;
  404.     }

  406.     /**
  407.      * Get the number of objects in the stream.
  408.      * <p>
  409.      * The object count is only available after {@link #parse(ProgressMonitor)}
  410.      * has returned. The count may have been increased if the stream was a thin
  411.      * pack, and missing bases objects were appending onto it by the subclass.
  412.      *
  413.      * @return number of objects parsed out of the stream.
  414.      */
  415.     public int getObjectCount() {
  416.         return entryCount;
  417.     }

  419.     /**
  420.      * Get the information about the requested object.
  421.      * <p>
  422.      * The object information is only available after
  423.      * {@link #parse(ProgressMonitor)} has returned.
  424.      *
  425.      * @param nth
  426.      *            index of the object in the stream. Must be between 0 and
  427.      *            {@link #getObjectCount()}-1.
  428.      * @return the object information.
  429.      */
  430.     public PackedObjectInfo getObject(int nth) {
  431.         return entries[nth];
  432.     }

  434.     /**
  435.      * Get all of the objects, sorted by their name.
  436.      * <p>
  437.      * The object information is only available after
  438.      * {@link #parse(ProgressMonitor)} has returned.
  439.      * <p>
  440.      * To maintain lower memory usage and good runtime performance, this method
  441.      * sorts the objects in-place and therefore impacts the ordering presented
  442.      * by {@link #getObject(int)}.
  443.      *
  444.      * @param cmp
  445.      *            comparison function, if null objects are stored by ObjectId.
  446.      * @return sorted list of objects in this pack stream.
  447.      */
  448.     public List<PackedObjectInfo> getSortedObjectList(
  449.             Comparator<PackedObjectInfo> cmp) {
  450.         Arrays.sort(entries, 0, entryCount, cmp);
  451.         List<PackedObjectInfo> list = Arrays.asList(entries);
  452.         if (entryCount < entries.length)
  453.             list = list.subList(0, entryCount);
  454.         return list;
  455.     }

  457.     /**
  458.      * Get the size of the newly created pack.
  459.      * <p>
  460.      * This will also include the pack index size if an index was created. This
  461.      * method should only be called after pack parsing is finished.
  462.      *
  463.      * @return the pack size (including the index size) or -1 if the size cannot
  464.      *         be determined
  465.      * @since 3.3
  466.      */
  467.     public long getPackSize() {
  468.         return -1;
  469.     }

  471.     /**
  472.      * Returns the statistics of the parsed pack.
  473.      * <p>
  474.      * This should only be called after pack parsing is finished.
  475.      *
  476.      * @return {@link org.eclipse.jgit.transport.ReceivedPackStatistics}
  477.      * @since 4.6
  478.      */
  479.     public ReceivedPackStatistics getReceivedPackStatistics() {
  480.         return stats.build();
  481.     }

  483.     /**
  484.      * Parse the pack stream.
  485.      *
  486.      * @param progress
  487.      *            callback to provide progress feedback during parsing. If null,
  488.      *            {@link org.eclipse.jgit.lib.NullProgressMonitor} will be used.
  489.      * @return the pack lock, if one was requested by setting
  490.      *         {@link #setLockMessage(String)}.
  491.      * @throws java.io.IOException
  492.      *             the stream is malformed, or contains corrupt objects.
  493.      * @since 3.0
  494.      */
  495.     public final PackLock parse(ProgressMonitor progress) throws IOException {
  496.         return parse(progress, progress);
  497.     }

  499.     /**
  500.      * Parse the pack stream.
  501.      *
  502.      * @param receiving
  503.      *            receives progress feedback during the initial receiving
  504.      *            objects phase. If null,
  505.      *            {@link org.eclipse.jgit.lib.NullProgressMonitor} will be used.
  506.      * @param resolving
  507.      *            receives progress feedback during the resolving objects phase.
  508.      * @return the pack lock, if one was requested by setting
  509.      *         {@link #setLockMessage(String)}.
  510.      * @throws java.io.IOException
  511.      *             the stream is malformed, or contains corrupt objects.
  512.      * @since 3.0
  513.      */
  514.     public PackLock parse(ProgressMonitor receiving, ProgressMonitor resolving)
  515.             throws IOException {
  516.         if (receiving == null)
  517.             receiving = NullProgressMonitor.INSTANCE;
  518.         if (resolving == null)
  519.             resolving = NullProgressMonitor.INSTANCE;

  521.         if (receiving == resolving)
  522.             receiving.start(2 /* tasks */);
  523.         try {
  524.             readPackHeader();

  526.             entries = new PackedObjectInfo[(int) expectedObjectCount];
  527.             baseById = new ObjectIdOwnerMap<>();
  528.             baseByPos = new LongMap<>();
  529.             collisionCheckObjs = new BlockList<>();

  531.             receiving.beginTask(JGitText.get().receivingObjects,
  532.                     (int) expectedObjectCount);
  533.             try {
  534.                 for (int done = 0; done < expectedObjectCount; done++) {
  535.                     indexOneObject();
  536.                     receiving.update(1);
  537.                     if (receiving.isCancelled())
  538.                         throw new IOException(JGitText.get().downloadCancelled);
  539.                 }
  540.                 readPackFooter();
  541.                 endInput();
  542.             } finally {
  543.                 receiving.endTask();
  544.             }

  546.             if (!collisionCheckObjs.isEmpty()) {
  547.                 checkObjectCollision();
  548.             }

  550.             if (deltaCount > 0) {
  551.                 processDeltas(resolving);
  552.             }

  554.             packDigest = null;
  555.             baseById = null;
  556.             baseByPos = null;
  557.         } finally {
  558.             try {
  559.                 if (readCurs != null)
  560.                     readCurs.close();
  561.             } finally {
  562.                 readCurs = null;
  563.             }

  565.             try {
  566.                 inflater.release();
  567.             } finally {
  568.                 inflater = null;
  569.             }
  570.         }
  571.         return null; // By default there is no locking.
  572.     }

  574.     private void processDeltas(ProgressMonitor resolving) throws IOException {
  575.         if (resolving instanceof BatchingProgressMonitor) {
  576.             ((BatchingProgressMonitor) resolving).setDelayStart(1000,
  577.                     TimeUnit.MILLISECONDS);
  578.         }
  579.         resolving.beginTask(JGitText.get().resolvingDeltas, deltaCount);
  580.         resolveDeltas(resolving);
  581.         if (entryCount < expectedObjectCount) {
  582.             if (!isAllowThin()) {
  583.                 throw new IOException(MessageFormat.format(
  584.                         JGitText.get().packHasUnresolvedDeltas,
  585.                         Long.valueOf(expectedObjectCount - entryCount)));
  586.             }

  588.             resolveDeltasWithExternalBases(resolving);

  590.             if (entryCount < expectedObjectCount) {
  591.                 throw new IOException(MessageFormat.format(
  592.                         JGitText.get().packHasUnresolvedDeltas,
  593.                         Long.valueOf(expectedObjectCount - entryCount)));
  594.             }
  595.         }
  596.         resolving.endTask();
  597.     }

  599.     private void resolveDeltas(ProgressMonitor progress)
  600.             throws IOException {
  601.         final int last = entryCount;
  602.         for (int i = 0; i < last; i++) {
  603.             resolveDeltas(entries[i], progress);
  604.             if (progress.isCancelled())
  605.                 throw new IOException(
  606.                         JGitText.get().downloadCancelledDuringIndexing);
  607.         }
  608.     }

  610.     private void resolveDeltas(final PackedObjectInfo oe,
  611.             ProgressMonitor progress) throws IOException {
  612.         UnresolvedDelta children = firstChildOf(oe);
  613.         if (children == null)
  614.             return;

  616.         DeltaVisit visit = new DeltaVisit();
  617.         visit.nextChild = children;

  619.         ObjectTypeAndSize info = openDatabase(oe, new ObjectTypeAndSize());
  620.         switch (info.type) {
  621.         case Constants.OBJ_COMMIT:
  622.         case Constants.OBJ_TREE:
  623.         case Constants.OBJ_BLOB:
  624.         case Constants.OBJ_TAG:
  625.             visit.data = inflateAndReturn(Source.DATABASE, info.size);
  626.             visit.id = oe;
  627.             break;
  628.         default:
  629.             throw new IOException(MessageFormat.format(
  630.                     JGitText.get().unknownObjectType,
  631.                     Integer.valueOf(info.type)));
  632.         }

  634.         if (!checkCRC(oe.getCRC())) {
  635.             throw new IOException(MessageFormat.format(
  636.                     JGitText.get().corruptionDetectedReReadingAt,
  637.                     Long.valueOf(oe.getOffset())));
  638.         }

  640.         resolveDeltas(visit.next(), info.type, info, progress);
  641.     }

  643.     private void resolveDeltas(DeltaVisit visit, final int type,
  644.             ObjectTypeAndSize info, ProgressMonitor progress)
  645.             throws IOException {
  646.         stats.addDeltaObject(type);
  647.         do {
  648.             progress.update(1);
  649.             info = openDatabase(visit.delta, info);
  650.             switch (info.type) {
  651.             case Constants.OBJ_OFS_DELTA:
  652.             case Constants.OBJ_REF_DELTA:
  653.                 break;

  655.             default:
  656.                 throw new IOException(MessageFormat.format(
  657.                         JGitText.get().unknownObjectType,
  658.                         Integer.valueOf(info.type)));
  659.             }

  661.             byte[] delta = inflateAndReturn(Source.DATABASE, info.size);
  662.             checkIfTooLarge(type, BinaryDelta.getResultSize(delta));

  664.             visit.data = BinaryDelta.apply(visit.parent.data, delta);
  665.             delta = null;

  667.             if (!checkCRC(visit.delta.crc))
  668.                 throw new IOException(MessageFormat.format(
  669.                         JGitText.get().corruptionDetectedReReadingAt,
  670.                         Long.valueOf(visit.delta.position)));

  672.             SHA1 objectDigest = objectHasher.reset();
  673.             objectDigest.update(Constants.encodedTypeString(type));
  674.             objectDigest.update((byte) ' ');
  675.             objectDigest.update(Constants.encodeASCII(visit.data.length));
  676.             objectDigest.update((byte) 0);
  677.             objectDigest.update(visit.data);
  678.             objectDigest.digest(tempObjectId);

  680.             verifySafeObject(tempObjectId, type, visit.data);
  681.             if (isCheckObjectCollisions() && readCurs.has(tempObjectId)) {
  682.                 checkObjectCollision(tempObjectId, type, visit.data,
  683.                         visit.delta.sizeBeforeInflating);
  684.             }

  686.             PackedObjectInfo oe;
  687.             oe = newInfo(tempObjectId, visit.delta, visit.parent.id);
  688.             oe.setOffset(visit.delta.position);
  689.             oe.setType(type);
  690.             onInflatedObjectData(oe, type, visit.data);
  691.             addObjectAndTrack(oe);
  692.             visit.id = oe;

  694.             visit.nextChild = firstChildOf(oe);
  695.             visit = visit.next();
  696.         } while (visit != null);
  697.     }

  699.     private final void checkIfTooLarge(int typeCode, long size)
  700.             throws IOException {
  701.         if (0 < maxObjectSizeLimit && maxObjectSizeLimit < size) {
  702.             switch (typeCode) {
  703.             case Constants.OBJ_COMMIT:
  704.             case Constants.OBJ_TREE:
  705.             case Constants.OBJ_BLOB:
  706.             case Constants.OBJ_TAG:
  707.                 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);

  709.             case Constants.OBJ_OFS_DELTA:
  710.             case Constants.OBJ_REF_DELTA:
  711.                 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);

  713.             default:
  714.                 throw new IOException(MessageFormat.format(
  715.                         JGitText.get().unknownObjectType,
  716.                         Integer.valueOf(typeCode)));
  717.             }
  718.         }
  719.         if (size > Integer.MAX_VALUE - 8) {
  720.             throw new TooLargeObjectInPackException(size, Integer.MAX_VALUE - 8);
  721.         }
  722.     }

  724.     /**
  725.      * Read the header of the current object.
  726.      * <p>
  727.      * After the header has been parsed, this method automatically invokes
  728.      * {@link #onObjectHeader(Source, byte[], int, int)} to allow the
  729.      * implementation to update its internal checksums for the bytes read.
  730.      * <p>
  731.      * When this method returns the database will be positioned on the first
  732.      * byte of the deflated data stream.
  733.      *
  734.      * @param info
  735.      *            the info object to populate.
  736.      * @return {@code info}, after populating.
  737.      * @throws java.io.IOException
  738.      *             the size cannot be read.
  739.      */
  740.     protected ObjectTypeAndSize readObjectHeader(ObjectTypeAndSize info)
  741.             throws IOException {
  742.         int hdrPtr = 0;
  743.         int c = readFrom(Source.DATABASE);
  744.         hdrBuf[hdrPtr++] = (byte) c;

  746.         info.type = (c >> 4) & 7;
  747.         long sz = c & 15;
  748.         int shift = 4;
  749.         while ((c & 0x80) != 0) {
  750.             c = readFrom(Source.DATABASE);
  751.             hdrBuf[hdrPtr++] = (byte) c;
  752.             sz += ((long) (c & 0x7f)) << shift;
  753.             shift += 7;
  754.         }
  755.         info.size = sz;

  757.         switch (info.type) {
  758.         case Constants.OBJ_COMMIT:
  759.         case Constants.OBJ_TREE:
  760.         case Constants.OBJ_BLOB:
  761.         case Constants.OBJ_TAG:
  762.             onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  763.             break;

  765.         case Constants.OBJ_OFS_DELTA:
  766.             c = readFrom(Source.DATABASE);
  767.             hdrBuf[hdrPtr++] = (byte) c;
  768.             while ((c & 128) != 0) {
  769.                 c = readFrom(Source.DATABASE);
  770.                 hdrBuf[hdrPtr++] = (byte) c;
  771.             }
  772.             onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  773.             break;

  775.         case Constants.OBJ_REF_DELTA:
  776.             System.arraycopy(buf, fill(Source.DATABASE, 20), hdrBuf, hdrPtr, 20);
  777.             hdrPtr += 20;
  778.             use(20);
  779.             onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  780.             break;

  782.         default:
  783.             throw new IOException(MessageFormat.format(
  784.                     JGitText.get().unknownObjectType,
  785.                     Integer.valueOf(info.type)));
  786.         }
  787.         return info;
  788.     }

  790.     private UnresolvedDelta removeBaseById(AnyObjectId id) {
  791.         final DeltaChain d = baseById.get(id);
  792.         return d != null ? d.remove() : null;
  793.     }

  795.     private static UnresolvedDelta reverse(UnresolvedDelta c) {
  796.         UnresolvedDelta tail = null;
  797.         while (c != null) {
  798.             final UnresolvedDelta n = c.next;
  799.             c.next = tail;
  800.             tail = c;
  801.             c = n;
  802.         }
  803.         return tail;
  804.     }

  806.     private UnresolvedDelta firstChildOf(PackedObjectInfo oe) {
  807.         UnresolvedDelta a = reverse(removeBaseById(oe));
  808.         UnresolvedDelta b = reverse(baseByPos.remove(oe.getOffset()));

  810.         if (a == null)
  811.             return b;
  812.         if (b == null)
  813.             return a;

  815.         UnresolvedDelta first = null;
  816.         UnresolvedDelta last = null;
  817.         while (a != null || b != null) {
  818.             UnresolvedDelta curr;
  819.             if (b == null || (a != null && a.position < b.position)) {
  820.                 curr = a;
  821.                 a = a.next;
  822.             } else {
  823.                 curr = b;
  824.                 b = b.next;
  825.             }
  826.             if (last != null)
  827.                 last.next = curr;
  828.             else
  829.                 first = curr;
  830.             last = curr;
  831.             curr.next = null;
  832.         }
  833.         return first;
  834.     }

  836.     private void resolveDeltasWithExternalBases(ProgressMonitor progress)
  837.             throws IOException {
  838.         growEntries(baseById.size());

  840.         if (needBaseObjectIds)
  841.             baseObjectIds = new ObjectIdSubclassMap<>();

  843.         final List<DeltaChain> missing = new ArrayList<>(64);
  844.         for (DeltaChain baseId : baseById) {
  845.             if (baseId.head == null)
  846.                 continue;

  848.             if (needBaseObjectIds)
  849.                 baseObjectIds.add(baseId);

  851.             final ObjectLoader ldr;
  852.             try {
  853.                 ldr = readCurs.open(baseId);
  854.             } catch (MissingObjectException notFound) {
  855.                 missing.add(baseId);
  856.                 continue;
  857.             }

  859.             final DeltaVisit visit = new DeltaVisit();
  860.             visit.data = ldr.getCachedBytes(Integer.MAX_VALUE);
  861.             visit.id = baseId;
  862.             final int typeCode = ldr.getType();
  863.             final PackedObjectInfo oe = newInfo(baseId, null, null);
  864.             oe.setType(typeCode);
  865.             if (onAppendBase(typeCode, visit.data, oe))
  866.                 entries[entryCount++] = oe;
  867.             visit.nextChild = firstChildOf(oe);
  868.             resolveDeltas(visit.next(), typeCode,
  869.                     new ObjectTypeAndSize(), progress);

  871.             if (progress.isCancelled())
  872.                 throw new IOException(
  873.                         JGitText.get().downloadCancelledDuringIndexing);
  874.         }

  876.         for (DeltaChain base : missing) {
  877.             if (base.head != null)
  878.                 throw new MissingObjectException(base, "delta base"); //$NON-NLS-1$
  879.         }

  881.         onEndThinPack();
  882.     }

  884.     private void growEntries(int extraObjects) {
  885.         final PackedObjectInfo[] ne;

  887.         ne = new PackedObjectInfo[(int) expectedObjectCount + extraObjects];
  888.         System.arraycopy(entries, 0, ne, 0, entryCount);
  889.         entries = ne;
  890.     }

  892.     private void readPackHeader() throws IOException {
  893.         if (expectDataAfterPackFooter) {
  894.             if (!in.markSupported())
  895.                 throw new IOException(
  896.                         JGitText.get().inputStreamMustSupportMark);
  897.             in.mark(buf.length);
  898.         }

  900.         final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
  901.         final int p = fill(Source.INPUT, hdrln);
  902.         for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
  903.             if (buf[p + k] != Constants.PACK_SIGNATURE[k])
  904.                 throw new IOException(JGitText.get().notAPACKFile);

  906.         final long vers = NB.decodeUInt32(buf, p + 4);
  907.         if (vers != 2 && vers != 3)
  908.             throw new IOException(MessageFormat.format(
  909.                     JGitText.get().unsupportedPackVersion, Long.valueOf(vers)));
  910.         final long objectCount = NB.decodeUInt32(buf, p + 8);
  911.         use(hdrln);
  912.         setExpectedObjectCount(objectCount);
  913.         onPackHeader(objectCount);
  914.     }

  916.     private void readPackFooter() throws IOException {
  917.         sync();
  918.         final byte[] actHash = packDigest.digest();

  920.         final int c = fill(Source.INPUT, 20);
  921.         final byte[] srcHash = new byte[20];
  922.         System.arraycopy(buf, c, srcHash, 0, 20);
  923.         use(20);

  925.         if (bAvail != 0 && !expectDataAfterPackFooter)
  926.             throw new CorruptObjectException(MessageFormat.format(
  927.                     JGitText.get().expectedEOFReceived,
  928.                     "\\x" + Integer.toHexString(buf[bOffset] & 0xff))); //$NON-NLS-1$
  929.         if (isCheckEofAfterPackFooter()) {
  930.             int eof = in.read();
  931.             if (0 <= eof)
  932.                 throw new CorruptObjectException(MessageFormat.format(
  933.                         JGitText.get().expectedEOFReceived,
  934.                         "\\x" + Integer.toHexString(eof))); //$NON-NLS-1$
  935.         } else if (bAvail > 0 && expectDataAfterPackFooter) {
  936.             in.reset();
  937.             IO.skipFully(in, bOffset);
  938.         }

  940.         if (!Arrays.equals(actHash, srcHash))
  941.             throw new CorruptObjectException(
  942.                     JGitText.get().corruptObjectPackfileChecksumIncorrect);

  944.         onPackFooter(srcHash);
  945.     }

  947.     // Cleanup all resources associated with our input parsing.
  948.     private void endInput() {
  949.         stats.setNumBytesRead(streamPosition());
  950.         in = null;
  951.     }

  953.     // Read one entire object or delta from the input.
  954.     private void indexOneObject() throws IOException {
  955.         final long streamPosition = streamPosition();

  957.         int hdrPtr = 0;
  958.         int c = readFrom(Source.INPUT);
  959.         hdrBuf[hdrPtr++] = (byte) c;

  961.         final int typeCode = (c >> 4) & 7;
  962.         long sz = c & 15;
  963.         int shift = 4;
  964.         while ((c & 0x80) != 0) {
  965.             c = readFrom(Source.INPUT);
  966.             hdrBuf[hdrPtr++] = (byte) c;
  967.             sz += ((long) (c & 0x7f)) << shift;
  968.             shift += 7;
  969.         }

  971.         checkIfTooLarge(typeCode, sz);

  973.         switch (typeCode) {
  974.         case Constants.OBJ_COMMIT:
  975.         case Constants.OBJ_TREE:
  976.         case Constants.OBJ_BLOB:
  977.         case Constants.OBJ_TAG:
  978.             stats.addWholeObject(typeCode);
  979.             onBeginWholeObject(streamPosition, typeCode, sz);
  980.             onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  981.             whole(streamPosition, typeCode, sz);
  982.             break;

  984.         case Constants.OBJ_OFS_DELTA: {
  985.             stats.addOffsetDelta();
  986.             c = readFrom(Source.INPUT);
  987.             hdrBuf[hdrPtr++] = (byte) c;
  988.             long ofs = c & 127;
  989.             while ((c & 128) != 0) {
  990.                 ofs += 1;
  991.                 c = readFrom(Source.INPUT);
  992.                 hdrBuf[hdrPtr++] = (byte) c;
  993.                 ofs <<= 7;
  994.                 ofs += (c & 127);
  995.             }
  996.             final long base = streamPosition - ofs;
  997.             onBeginOfsDelta(streamPosition, base, sz);
  998.             onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  999.             inflateAndSkip(Source.INPUT, sz);
  1000.             UnresolvedDelta n = onEndDelta();
  1001.             n.position = streamPosition;
  1002.             n.next = baseByPos.put(base, n);
  1003.             n.sizeBeforeInflating = streamPosition() - streamPosition;
  1004.             deltaCount++;
  1005.             break;
  1006.         }

  1008.         case Constants.OBJ_REF_DELTA: {
  1009.             stats.addRefDelta();
  1010.             c = fill(Source.INPUT, 20);
  1011.             final ObjectId base = ObjectId.fromRaw(buf, c);
  1012.             System.arraycopy(buf, c, hdrBuf, hdrPtr, 20);
  1013.             hdrPtr += 20;
  1014.             use(20);
  1015.             DeltaChain r = baseById.get(base);
  1016.             if (r == null) {
  1017.                 r = new DeltaChain(base);
  1018.                 baseById.add(r);
  1019.             }
  1020.             onBeginRefDelta(streamPosition, base, sz);
  1021.             onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  1022.             inflateAndSkip(Source.INPUT, sz);
  1023.             UnresolvedDelta n = onEndDelta();
  1024.             n.position = streamPosition;
  1025.             n.sizeBeforeInflating = streamPosition() - streamPosition;
  1026.             r.add(n);
  1027.             deltaCount++;
  1028.             break;
  1029.         }

  1031.         default:
  1032.             throw new IOException(
  1033.                     MessageFormat.format(JGitText.get().unknownObjectType,
  1034.                             Integer.valueOf(typeCode)));
  1035.         }
  1036.     }

  1038.     private void whole(long pos, int type, long sz)
  1039.             throws IOException {
  1040.         SHA1 objectDigest = objectHasher.reset();
  1041.         objectDigest.update(Constants.encodedTypeString(type));
  1042.         objectDigest.update((byte) ' ');
  1043.         objectDigest.update(Constants.encodeASCII(sz));
  1044.         objectDigest.update((byte) 0);

  1046.         final byte[] data;
  1047.         if (type == Constants.OBJ_BLOB) {
  1048.             byte[] readBuffer = buffer();
  1049.             BlobObjectChecker checker = null;
  1050.             if (objCheck != null) {
  1051.                 checker = objCheck.newBlobObjectChecker();
  1052.             }
  1053.             if (checker == null) {
  1054.                 checker = BlobObjectChecker.NULL_CHECKER;
  1055.             }
  1056.             long cnt = 0;
  1057.             try (InputStream inf = inflate(Source.INPUT, sz)) {
  1058.                 while (cnt < sz) {
  1059.                     int r = inf.read(readBuffer);
  1060.                     if (r <= 0)
  1061.                         break;
  1062.                     objectDigest.update(readBuffer, 0, r);
  1063.                     checker.update(readBuffer, 0, r);
  1064.                     cnt += r;
  1065.                 }
  1066.             }
  1067.             objectDigest.digest(tempObjectId);
  1068.             checker.endBlob(tempObjectId);
  1069.             data = null;
  1070.         } else {
  1071.             data = inflateAndReturn(Source.INPUT, sz);
  1072.             objectDigest.update(data);
  1073.             objectDigest.digest(tempObjectId);
  1074.             verifySafeObject(tempObjectId, type, data);
  1075.         }

  1077.         long sizeBeforeInflating = streamPosition() - pos;
  1078.         PackedObjectInfo obj = newInfo(tempObjectId, null, null);
  1079.         obj.setOffset(pos);
  1080.         obj.setType(type);
  1081.         obj.setSize(sizeBeforeInflating);
  1082.         onEndWholeObject(obj);
  1083.         if (data != null)
  1084.             onInflatedObjectData(obj, type, data);
  1085.         addObjectAndTrack(obj);

  1087.         if (isCheckObjectCollisions()) {
  1088.             collisionCheckObjs.add(obj);
  1089.         }
  1090.     }

  1092.     /**
  1093.      * Verify the integrity of the object.
  1094.      *
  1095.      * @param id
  1096.      *            identity of the object to be checked.
  1097.      * @param type
  1098.      *            the type of the object.
  1099.      * @param data
  1100.      *            raw content of the object.
  1101.      * @throws org.eclipse.jgit.errors.CorruptObjectException
  1102.      * @since 4.9
  1103.      */
  1104.     protected void verifySafeObject(final AnyObjectId id, final int type,
  1105.             final byte[] data) throws CorruptObjectException {
  1106.         if (objCheck != null) {
  1107.             try {
  1108.                 objCheck.check(id, type, data);
  1109.             } catch (CorruptObjectException e) {
  1110.                 if (e.getErrorType() != null) {
  1111.                     throw e;
  1112.                 }
  1113.                 throw new CorruptObjectException(
  1114.                         MessageFormat.format(JGitText.get().invalidObject,
  1115.                                 Constants.typeString(type), id.name(),
  1116.                                 e.getMessage()),
  1117.                         e);
  1118.             }
  1119.         }
  1120.     }

  1122.     private void checkObjectCollision() throws IOException {
  1123.         for (PackedObjectInfo obj : collisionCheckObjs) {
  1124.             if (!readCurs.has(obj)) {
  1125.                 continue;
  1126.             }
  1127.             checkObjectCollision(obj);
  1128.         }
  1129.     }

  1131.     private void checkObjectCollision(PackedObjectInfo obj)
  1132.             throws IOException {
  1133.         ObjectTypeAndSize info = openDatabase(obj, new ObjectTypeAndSize());
  1134.         final byte[] readBuffer = buffer();
  1135.         final byte[] curBuffer = new byte[readBuffer.length];
  1136.         long sz = info.size;
  1137.         try (ObjectStream cur = readCurs.open(obj, info.type).openStream()) {
  1138.             if (cur.getSize() != sz) {
  1139.                 throw new IOException(MessageFormat.format(
  1140.                         JGitText.get().collisionOn, obj.name()));
  1141.             }
  1142.             try (InputStream pck = inflate(Source.DATABASE, sz)) {
  1143.                 while (0 < sz) {
  1144.                     int n = (int) Math.min(readBuffer.length, sz);
  1145.                     IO.readFully(cur, curBuffer, 0, n);
  1146.                     IO.readFully(pck, readBuffer, 0, n);
  1147.                     for (int i = 0; i < n; i++) {
  1148.                         if (curBuffer[i] != readBuffer[i]) {
  1149.                             throw new IOException(MessageFormat.format(
  1150.                                     JGitText.get().collisionOn, obj.name()));
  1151.                         }
  1152.                     }
  1153.                     sz -= n;
  1154.                 }
  1155.             }
  1156.             stats.incrementObjectsDuplicated();
  1157.             stats.incrementNumBytesDuplicated(obj.getSize());
  1158.         } catch (MissingObjectException notLocal) {
  1159.             // This is OK, we don't have a copy of the object locally
  1160.             // but the API throws when we try to read it as usually it's
  1161.             // an error to read something that doesn't exist.
  1162.         }
  1163.     }

  1165.     private void checkObjectCollision(AnyObjectId obj, int type, byte[] data,
  1166.             long sizeBeforeInflating) throws IOException {
  1167.         try {
  1168.             final ObjectLoader ldr = readCurs.open(obj, type);
  1169.             final byte[] existingData = ldr.getCachedBytes(data.length);
  1170.             if (!Arrays.equals(data, existingData)) {
  1171.                 throw new IOException(MessageFormat
  1172.                         .format(JGitText.get().collisionOn, obj.name()));
  1173.             }
  1174.             stats.incrementObjectsDuplicated();
  1175.             stats.incrementNumBytesDuplicated(sizeBeforeInflating);
  1176.         } catch (MissingObjectException notLocal) {
  1177.             // This is OK, we don't have a copy of the object locally
  1178.             // but the API throws when we try to read it as usually its
  1179.             // an error to read something that doesn't exist.
  1180.         }
  1181.     }

  1183.     /** @return current position of the input stream being parsed. */
  1184.     private long streamPosition() {
  1185.         return bBase + bOffset;
  1186.     }

  1188.     private ObjectTypeAndSize openDatabase(PackedObjectInfo obj,
  1189.             ObjectTypeAndSize info) throws IOException {
  1190.         bOffset = 0;
  1191.         bAvail = 0;
  1192.         return seekDatabase(obj, info);
  1193.     }

  1195.     private ObjectTypeAndSize openDatabase(UnresolvedDelta delta,
  1196.             ObjectTypeAndSize info) throws IOException {
  1197.         bOffset = 0;
  1198.         bAvail = 0;
  1199.         return seekDatabase(delta, info);
  1200.     }

  1202.     // Consume exactly one byte from the buffer and return it.
  1203.     private int readFrom(Source src) throws IOException {
  1204.         if (bAvail == 0)
  1205.             fill(src, 1);
  1206.         bAvail--;
  1207.         return buf[bOffset++] & 0xff;
  1208.     }

  1210.     // Consume cnt bytes from the buffer.
  1211.     void use(int cnt) {
  1212.         bOffset += cnt;
  1213.         bAvail -= cnt;
  1214.     }

  1216.     // Ensure at least need bytes are available in {@link #buf}.
  1217.     int fill(Source src, int need) throws IOException {
  1218.         while (bAvail < need) {
  1219.             int next = bOffset + bAvail;
  1220.             int free = buf.length - next;
  1221.             if (free + bAvail < need) {
  1222.                 switch (src) {
  1223.                 case INPUT:
  1224.                     sync();
  1225.                     break;
  1226.                 case DATABASE:
  1227.                     if (bAvail > 0)
  1228.                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
  1229.                     bOffset = 0;
  1230.                     break;
  1231.                 }
  1232.                 next = bAvail;
  1233.                 free = buf.length - next;
  1234.             }
  1235.             switch (src) {
  1236.             case INPUT:
  1237.                 next = in.read(buf, next, free);
  1238.                 break;
  1239.             case DATABASE:
  1240.                 next = readDatabase(buf, next, free);
  1241.                 break;
  1242.             }
  1243.             if (next <= 0)
  1244.                 throw new EOFException(
  1245.                         JGitText.get().packfileIsTruncatedNoParam);
  1246.             bAvail += next;
  1247.         }
  1248.         return bOffset;
  1249.     }

  1251.     // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
  1252.     private void sync() throws IOException {
  1253.         packDigest.update(buf, 0, bOffset);
  1254.         onStoreStream(buf, 0, bOffset);
  1255.         if (expectDataAfterPackFooter) {
  1256.             if (bAvail > 0) {
  1257.                 in.reset();
  1258.                 IO.skipFully(in, bOffset);
  1259.                 bAvail = 0;
  1260.             }
  1261.             in.mark(buf.length);
  1262.         } else if (bAvail > 0)
  1263.             System.arraycopy(buf, bOffset, buf, 0, bAvail);
  1264.         bBase += bOffset;
  1265.         bOffset = 0;
  1266.     }

  1268.     /**
  1269.      * Get a temporary byte array for use by the caller.
  1270.      *
  1271.      * @return a temporary byte array for use by the caller.
  1272.      */
  1273.     protected byte[] buffer() {
  1274.         return tempBuffer;
  1275.     }

  1277.     /**
  1278.      * Construct a PackedObjectInfo instance for this parser.
  1279.      *
  1280.      * @param id
  1281.      *            identity of the object to be tracked.
  1282.      * @param delta
  1283.      *            if the object was previously an unresolved delta, this is the
  1284.      *            delta object that was tracking it. Otherwise null.
  1285.      * @param deltaBase
  1286.      *            if the object was previously an unresolved delta, this is the
  1287.      *            ObjectId of the base of the delta. The base may be outside of
  1288.      *            the pack stream if the stream was a thin-pack.
  1289.      * @return info object containing this object's data.
  1290.      */
  1291.     protected PackedObjectInfo newInfo(AnyObjectId id, UnresolvedDelta delta,
  1292.             ObjectId deltaBase) {
  1293.         PackedObjectInfo oe = new PackedObjectInfo(id);
  1294.         if (delta != null)
  1295.             oe.setCRC(delta.crc);
  1296.         return oe;
  1297.     }

  1299.     /**
  1300.      * Set the expected number of objects in the pack stream.
  1301.      * <p>
  1302.      * The object count in the pack header is not always correct for some Dfs
  1303.      * pack files. e.g. INSERT pack always assume 1 object in the header since
  1304.      * the actual object count is unknown when the pack is written.
  1305.      * <p>
  1306.      * If external implementation wants to overwrite the expectedObjectCount,
  1307.      * they should call this method during {@link #onPackHeader(long)}.
  1308.      *
  1309.      * @param expectedObjectCount a long.
  1310.      * @since 4.9
  1311.      */
  1312.     protected void setExpectedObjectCount(long expectedObjectCount) {
  1313.         this.expectedObjectCount = expectedObjectCount;
  1314.     }

  1316.     /**
  1317.      * Store bytes received from the raw stream.
  1318.      * <p>
  1319.      * This method is invoked during {@link #parse(ProgressMonitor)} as data is
  1320.      * consumed from the incoming stream. Implementors may use this event to
  1321.      * archive the raw incoming stream to the destination repository in large
  1322.      * chunks, without paying attention to object boundaries.
  1323.      * <p>
  1324.      * The only component of the pack not supplied to this method is the last 20
  1325.      * bytes of the pack that comprise the trailing SHA-1 checksum. Those are
  1326.      * passed to {@link #onPackFooter(byte[])}.
  1327.      *
  1328.      * @param raw
  1329.      *            buffer to copy data out of.
  1330.      * @param pos
  1331.      *            first offset within the buffer that is valid.
  1332.      * @param len
  1333.      *            number of bytes in the buffer that are valid.
  1334.      * @throws java.io.IOException
  1335.      *             the stream cannot be archived.
  1336.      */
  1337.     protected abstract void onStoreStream(byte[] raw, int pos, int len)
  1338.             throws IOException;

  1340.     /**
  1341.      * Store (and/or checksum) an object header.
  1342.      * <p>
  1343.      * Invoked after any of the {@code onBegin()} events. The entire header is
  1344.      * supplied in a single invocation, before any object data is supplied.
  1345.      *
  1346.      * @param src
  1347.      *            where the data came from
  1348.      * @param raw
  1349.      *            buffer to read data from.
  1350.      * @param pos
  1351.      *            first offset within buffer that is valid.
  1352.      * @param len
  1353.      *            number of bytes in buffer that are valid.
  1354.      * @throws java.io.IOException
  1355.      *             the stream cannot be archived.
  1356.      */
  1357.     protected abstract void onObjectHeader(Source src, byte[] raw, int pos,
  1358.             int len) throws IOException;

  1360.     /**
  1361.      * Store (and/or checksum) a portion of an object's data.
  1362.      * <p>
  1363.      * This method may be invoked multiple times per object, depending on the
  1364.      * size of the object, the size of the parser's internal read buffer, and
  1365.      * the alignment of the object relative to the read buffer.
  1366.      * <p>
  1367.      * Invoked after {@link #onObjectHeader(Source, byte[], int, int)}.
  1368.      *
  1369.      * @param src
  1370.      *            where the data came from
  1371.      * @param raw
  1372.      *            buffer to read data from.
  1373.      * @param pos
  1374.      *            first offset within buffer that is valid.
  1375.      * @param len
  1376.      *            number of bytes in buffer that are valid.
  1377.      * @throws java.io.IOException
  1378.      *             the stream cannot be archived.
  1379.      */
  1380.     protected abstract void onObjectData(Source src, byte[] raw, int pos,
  1381.             int len) throws IOException;

  1383.     /**
  1384.      * Invoked for commits, trees, tags, and small blobs.
  1385.      *
  1386.      * @param obj
  1387.      *            the object info, populated.
  1388.      * @param typeCode
  1389.      *            the type of the object.
  1390.      * @param data
  1391.      *            inflated data for the object.
  1392.      * @throws java.io.IOException
  1393.      *             the object cannot be archived.
  1394.      */
  1395.     protected abstract void onInflatedObjectData(PackedObjectInfo obj,
  1396.             int typeCode, byte[] data) throws IOException;

  1398.     /**
  1399.      * Provide the implementation with the original stream's pack header.
  1400.      *
  1401.      * @param objCnt
  1402.      *            number of objects expected in the stream.
  1403.      * @throws java.io.IOException
  1404.      *             the implementation refuses to work with this many objects.
  1405.      */
  1406.     protected abstract void onPackHeader(long objCnt) throws IOException;

  1408.     /**
  1409.      * Provide the implementation with the original stream's pack footer.
  1410.      *
  1411.      * @param hash
  1412.      *            the trailing 20 bytes of the pack, this is a SHA-1 checksum of
  1413.      *            all of the pack data.
  1414.      * @throws java.io.IOException
  1415.      *             the stream cannot be archived.
  1416.      */
  1417.     protected abstract void onPackFooter(byte[] hash) throws IOException;

  1419.     /**
  1420.      * Provide the implementation with a base that was outside of the pack.
  1421.      * <p>
  1422.      * This event only occurs on a thin pack for base objects that were outside
  1423.      * of the pack and came from the local repository. Usually an implementation
  1424.      * uses this event to compress the base and append it onto the end of the
  1425.      * pack, so the pack stays self-contained.
  1426.      *
  1427.      * @param typeCode
  1428.      *            type of the base object.
  1429.      * @param data
  1430.      *            complete content of the base object.
  1431.      * @param info
  1432.      *            packed object information for this base. Implementors must
  1433.      *            populate the CRC and offset members if returning true.
  1434.      * @return true if the {@code info} should be included in the object list
  1435.      *         returned by {@link #getSortedObjectList(Comparator)}, false if it
  1436.      *         should not be included.
  1437.      * @throws java.io.IOException
  1438.      *             the base could not be included into the pack.
  1439.      */
  1440.     protected abstract boolean onAppendBase(int typeCode, byte[] data,
  1441.             PackedObjectInfo info) throws IOException;

  1443.     /**
  1444.      * Event indicating a thin pack has been completely processed.
  1445.      * <p>
  1446.      * This event is invoked only if a thin pack has delta references to objects
  1447.      * external from the pack. The event is called after all of those deltas
  1448.      * have been resolved.
  1449.      *
  1450.      * @throws java.io.IOException
  1451.      *             the pack cannot be archived.
  1452.      */
  1453.     protected abstract void onEndThinPack() throws IOException;

  1455.     /**
  1456.      * Reposition the database to re-read a previously stored object.
  1457.      * <p>
  1458.      * If the database is computing CRC-32 checksums for object data, it should
  1459.      * reset its internal CRC instance during this method call.
  1460.      *
  1461.      * @param obj
  1462.      *            the object position to begin reading from. This is from
  1463.      *            {@link #newInfo(AnyObjectId, UnresolvedDelta, ObjectId)}.
  1464.      * @param info
  1465.      *            object to populate with type and size.
  1466.      * @return the {@code info} object.
  1467.      * @throws java.io.IOException
  1468.      *             the database cannot reposition to this location.
  1469.      */
  1470.     protected abstract ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
  1471.             ObjectTypeAndSize info) throws IOException;

  1473.     /**
  1474.      * Reposition the database to re-read a previously stored object.
  1475.      * <p>
  1476.      * If the database is computing CRC-32 checksums for object data, it should
  1477.      * reset its internal CRC instance during this method call.
  1478.      *
  1479.      * @param delta
  1480.      *            the object position to begin reading from. This is an instance
  1481.      *            previously returned by {@link #onEndDelta()}.
  1482.      * @param info
  1483.      *            object to populate with type and size.
  1484.      * @return the {@code info} object.
  1485.      * @throws java.io.IOException
  1486.      *             the database cannot reposition to this location.
  1487.      */
  1488.     protected abstract ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
  1489.             ObjectTypeAndSize info) throws IOException;

  1491.     /**
  1492.      * Read from the database's current position into the buffer.
  1493.      *
  1494.      * @param dst
  1495.      *            the buffer to copy read data into.
  1496.      * @param pos
  1497.      *            position within {@code dst} to start copying data into.
  1498.      * @param cnt
  1499.      *            ideal target number of bytes to read. Actual read length may
  1500.      *            be shorter.
  1501.      * @return number of bytes stored.
  1502.      * @throws java.io.IOException
  1503.      *             the database cannot be accessed.
  1504.      */
  1505.     protected abstract int readDatabase(byte[] dst, int pos, int cnt)
  1506.             throws IOException;

  1508.     /**
  1509.      * Check the current CRC matches the expected value.
  1510.      * <p>
  1511.      * This method is invoked when an object is read back in from the database
  1512.      * and its data is used during delta resolution. The CRC is validated after
  1513.      * the object has been fully read, allowing the parser to verify there was
  1514.      * no silent data corruption.
  1515.      * <p>
  1516.      * Implementations are free to ignore this check by always returning true if
  1517.      * they are performing other data integrity validations at a lower level.
  1518.      *
  1519.      * @param oldCRC
  1520.      *            the prior CRC that was recorded during the first scan of the
  1521.      *            object from the pack stream.
  1522.      * @return true if the CRC matches; false if it does not.
  1523.      */
  1524.     protected abstract boolean checkCRC(int oldCRC);

  1526.     /**
  1527.      * Event notifying the start of an object stored whole (not as a delta).
  1528.      *
  1529.      * @param streamPosition
  1530.      *            position of this object in the incoming stream.
  1531.      * @param type
  1532.      *            type of the object; one of
  1533.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_COMMIT},
  1534.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_TREE},
  1535.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_BLOB}, or
  1536.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_TAG}.
  1537.      * @param inflatedSize
  1538.      *            size of the object when fully inflated. The size stored within
  1539.      *            the pack may be larger or smaller, and is not yet known.
  1540.      * @throws java.io.IOException
  1541.      *             the object cannot be recorded.
  1542.      */
  1543.     protected abstract void onBeginWholeObject(long streamPosition, int type,
  1544.             long inflatedSize) throws IOException;

  1546.     /**
  1547.      * Event notifying the current object.
  1548.      *
  1549.      *@param info
  1550.      *            object information.
  1551.      * @throws java.io.IOException
  1552.      *             the object cannot be recorded.
  1553.      */
  1554.     protected abstract void onEndWholeObject(PackedObjectInfo info)
  1555.             throws IOException;

  1557.     /**
  1558.      * Event notifying start of a delta referencing its base by offset.
  1559.      *
  1560.      * @param deltaStreamPosition
  1561.      *            position of this object in the incoming stream.
  1562.      * @param baseStreamPosition
  1563.      *            position of the base object in the incoming stream. The base
  1564.      *            must be before the delta, therefore {@code baseStreamPosition
  1565.      *            &lt; deltaStreamPosition}. This is <b>not</b> the position
  1566.      *            returned by a prior end object event.
  1567.      * @param inflatedSize
  1568.      *            size of the delta when fully inflated. The size stored within
  1569.      *            the pack may be larger or smaller, and is not yet known.
  1570.      * @throws java.io.IOException
  1571.      *             the object cannot be recorded.
  1572.      */
  1573.     protected abstract void onBeginOfsDelta(long deltaStreamPosition,
  1574.             long baseStreamPosition, long inflatedSize) throws IOException;

  1576.     /**
  1577.      * Event notifying start of a delta referencing its base by ObjectId.
  1578.      *
  1579.      * @param deltaStreamPosition
  1580.      *            position of this object in the incoming stream.
  1581.      * @param baseId
  1582.      *            name of the base object. This object may be later in the
  1583.      *            stream, or might not appear at all in the stream (in the case
  1584.      *            of a thin-pack).
  1585.      * @param inflatedSize
  1586.      *            size of the delta when fully inflated. The size stored within
  1587.      *            the pack may be larger or smaller, and is not yet known.
  1588.      * @throws java.io.IOException
  1589.      *             the object cannot be recorded.
  1590.      */
  1591.     protected abstract void onBeginRefDelta(long deltaStreamPosition,
  1592.             AnyObjectId baseId, long inflatedSize) throws IOException;

  1594.     /**
  1595.      * Event notifying the current object.
  1596.      *
  1597.      *@return object information that must be populated with at least the
  1598.      *         offset.
  1599.      * @throws java.io.IOException
  1600.      *             the object cannot be recorded.
  1601.      */
  1602.     protected UnresolvedDelta onEndDelta() throws IOException {
  1603.         return new UnresolvedDelta();
  1604.     }

  1606.     /** Type and size information about an object in the database buffer. */
  1607.     public static class ObjectTypeAndSize {
  1608.         /** The type of the object. */
  1609.         public int type;

  1611.         /** The inflated size of the object. */
  1612.         public long size;
  1613.     }

  1615.     private void inflateAndSkip(Source src, long inflatedSize)
  1616.             throws IOException {
  1617.         try (InputStream inf = inflate(src, inflatedSize)) {
  1618.             IO.skipFully(inf, inflatedSize);
  1619.         }
  1620.     }

  1622.     private byte[] inflateAndReturn(Source src, long inflatedSize)
  1623.             throws IOException {
  1624.         final byte[] dst = new byte[(int) inflatedSize];
  1625.         try (InputStream inf = inflate(src, inflatedSize)) {
  1626.             IO.readFully(inf, dst, 0, dst.length);
  1627.         }
  1628.         return dst;
  1629.     }

  1631.     private InputStream inflate(Source src, long inflatedSize)
  1632.             throws IOException {
  1633.         inflater.open(src, inflatedSize);
  1634.         return inflater;
  1635.     }

  1637.     private static class DeltaChain extends ObjectIdOwnerMap.Entry {
  1638.         UnresolvedDelta head;

  1640.         DeltaChain(AnyObjectId id) {
  1641.             super(id);
  1642.         }

  1644.         UnresolvedDelta remove() {
  1645.             final UnresolvedDelta r = head;
  1646.             if (r != null)
  1647.                 head = null;
  1648.             return r;
  1649.         }

  1651.         void add(UnresolvedDelta d) {
  1652.             d.next = head;
  1653.             head = d;
  1654.         }
  1655.     }

  1657.     /** Information about an unresolved delta in this pack stream. */
  1658.     public static class UnresolvedDelta {
  1659.         long position;

  1661.         int crc;

  1663.         UnresolvedDelta next;

  1665.         long sizeBeforeInflating;

  1667.         /** @return offset within the input stream. */
  1668.         public long getOffset() {
  1669.             return position;
  1670.         }

  1672.         /** @return the CRC-32 checksum of the stored delta data. */
  1673.         public int getCRC() {
  1674.             return crc;
  1675.         }

  1677.         /**
  1678.          * @param crc32
  1679.          *            the CRC-32 checksum of the stored delta data.
  1680.          */
  1681.         public void setCRC(int crc32) {
  1682.             crc = crc32;
  1683.         }
  1684.     }

  1686.     private static class DeltaVisit {
  1687.         final UnresolvedDelta delta;

  1689.         ObjectId id;

  1691.         byte[] data;

  1693.         DeltaVisit parent;

  1695.         UnresolvedDelta nextChild;

  1697.         DeltaVisit() {
  1698.             this.delta = null; // At the root of the stack we have a base.
  1699.         }

  1701.         DeltaVisit(DeltaVisit parent) {
  1702.             this.parent = parent;
  1703.             this.delta = parent.nextChild;
  1704.             parent.nextChild = delta.next;
  1705.         }

  1707.         DeltaVisit next() {
  1708.             // If our parent has no more children, discard it.
  1709.             if (parent != null && parent.nextChild == null) {
  1710.                 parent.data = null;
  1711.                 parent = parent.parent;
  1712.             }

  1714.             if (nextChild != null)
  1715.                 return new DeltaVisit(this);

  1717.             // If we have no child ourselves, our parent must (if it exists),
  1718.             // due to the discard rule above. With no parent, we are done.
  1719.             if (parent != null)
  1720.                 return new DeltaVisit(parent);
  1721.             return null;
  1722.         }
  1723.     }

  1725.     private void addObjectAndTrack(PackedObjectInfo oe) {
  1726.         entries[entryCount++] = oe;
  1727.         if (needNewObjectIds())
  1728.             newObjectIds.add(oe);
  1729.     }

  1731.     private class InflaterStream extends InputStream {
  1732.         private final Inflater inf;

  1734.         private final byte[] skipBuffer;

  1736.         private Source src;

  1738.         private long expectedSize;

  1740.         private long actualSize;

  1742.         private int p;

  1744.         InflaterStream() {
  1745.             inf = InflaterCache.get();
  1746.             skipBuffer = new byte[512];
  1747.         }

  1749.         void release() {
  1750.             inf.reset();
  1751.             InflaterCache.release(inf);
  1752.         }

  1754.         void open(Source source, long inflatedSize) throws IOException {
  1755.             src = source;
  1756.             expectedSize = inflatedSize;
  1757.             actualSize = 0;

  1759.             p = fill(src, 1);
  1760.             inf.setInput(buf, p, bAvail);
  1761.         }

  1763.         @Override
  1764.         public long skip(long toSkip) throws IOException {
  1765.             long n = 0;
  1766.             while (n < toSkip) {
  1767.                 final int cnt = (int) Math.min(skipBuffer.length, toSkip - n);
  1768.                 final int r = read(skipBuffer, 0, cnt);
  1769.                 if (r <= 0)
  1770.                     break;
  1771.                 n += r;
  1772.             }
  1773.             return n;
  1774.         }

  1776.         @Override
  1777.         public int read() throws IOException {
  1778.             int n = read(skipBuffer, 0, 1);
  1779.             return n == 1 ? skipBuffer[0] & 0xff : -1;
  1780.         }

  1782.         @Override
  1783.         public int read(byte[] dst, int pos, int cnt) throws IOException {
  1784.             try {
  1785.                 int n = 0;
  1786.                 while (n < cnt) {
  1787.                     int r = inf.inflate(dst, pos + n, cnt - n);
  1788.                     n += r;
  1789.                     if (inf.finished())
  1790.                         break;
  1791.                     if (inf.needsInput()) {
  1792.                         onObjectData(src, buf, p, bAvail);
  1793.                         use(bAvail);

  1795.                         p = fill(src, 1);
  1796.                         inf.setInput(buf, p, bAvail);
  1797.                     } else if (r == 0) {
  1798.                         throw new CorruptObjectException(MessageFormat.format(
  1799.                                 JGitText.get().packfileCorruptionDetected,
  1800.                                 JGitText.get().unknownZlibError));
  1801.                     }
  1802.                 }
  1803.                 actualSize += n;
  1804.                 return 0 < n ? n : -1;
  1805.             } catch (DataFormatException dfe) {
  1806.                 throw new CorruptObjectException(MessageFormat.format(JGitText
  1807.                         .get().packfileCorruptionDetected, dfe.getMessage()));
  1808.             }
  1809.         }

  1811.         @Override
  1812.         public void close() throws IOException {
  1813.             // We need to read here to enter the loop above and pump the
  1814.             // trailing checksum into the Inflater. It should return -1 as the
  1815.             // caller was supposed to consume all content.
  1816.             //
  1817.             if (read(skipBuffer) != -1 || actualSize != expectedSize) {
  1818.                 throw new CorruptObjectException(MessageFormat.format(JGitText
  1819.                         .get().packfileCorruptionDetected,
  1820.                         JGitText.get().wrongDecompressedLength));
  1821.             }

  1823.             int used = bAvail - inf.getRemaining();
  1824.             if (0 < used) {
  1825.                 onObjectData(src, buf, p, used);
  1826.                 use(used);
  1827.             }

  1829.             inf.reset();
  1830.         }
  1831.     }
  1832. }
Created with JaCoCo 0.8.6.202009150832