FsckPackParser.java

  1. /*
  2.  * Copyright (C) 2017, Google Inc.
  3.  * and other copyright owners as documented in the project's IP log.
  4.  *
  5.  * This program and the accompanying materials are made available
  6.  * under the terms of the Eclipse Distribution License v1.0 which
  7.  * accompanies this distribution, is reproduced below, and is
  8.  * available at http://www.eclipse.org/org/documents/edl-v10.php
  9.  *
  10.  * All rights reserved.
  11.  *
  12.  * Redistribution and use in source and binary forms, with or
  13.  * without modification, are permitted provided that the following
  14.  * conditions are met:
  15.  *
  16.  * - Redistributions of source code must retain the above copyright
  17.  *   notice, this list of conditions and the following disclaimer.
  18.  *
  19.  * - Redistributions in binary form must reproduce the above
  20.  *   copyright notice, this list of conditions and the following
  21.  *   disclaimer in the documentation and/or other materials provided
  22.  *   with the distribution.
  23.  *
  24.  * - Neither the name of the Eclipse Foundation, Inc. nor the
  25.  *   names of its contributors may be used to endorse or promote
  26.  *   products derived from this software without specific prior
  27.  *   written permission.
  28.  *
  29.  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  30.  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  31.  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  32.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33.  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  34.  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  35.  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  36.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  37.  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  38.  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  39.  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  40.  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  41.  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42.  */

  44. package org.eclipse.jgit.internal.fsck;

  46. import java.io.IOException;
  47. import java.nio.ByteBuffer;
  48. import java.nio.channels.Channels;
  49. import java.text.MessageFormat;
  50. import java.util.Arrays;
  51. import java.util.HashSet;
  52. import java.util.Set;
  53. import java.util.zip.CRC32;

  55. import org.eclipse.jgit.errors.CorruptObjectException;
  56. import org.eclipse.jgit.errors.CorruptPackIndexException;
  57. import org.eclipse.jgit.errors.CorruptPackIndexException.ErrorType;
  58. import org.eclipse.jgit.errors.MissingObjectException;
  59. import org.eclipse.jgit.internal.JGitText;
  60. import org.eclipse.jgit.internal.fsck.FsckError.CorruptObject;
  61. import org.eclipse.jgit.internal.storage.dfs.ReadableChannel;
  62. import org.eclipse.jgit.internal.storage.file.PackIndex;
  63. import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
  64. import org.eclipse.jgit.lib.AnyObjectId;
  65. import org.eclipse.jgit.lib.ObjectDatabase;
  66. import org.eclipse.jgit.lib.ObjectIdOwnerMap;
  67. import org.eclipse.jgit.transport.PackParser;
  68. import org.eclipse.jgit.transport.PackedObjectInfo;

  70. /**
  71.  * A read-only pack parser for object validity checking.
  72.  */
  73. public class FsckPackParser extends PackParser {
  74.     private final CRC32 crc;

  76.     private final ReadableChannel channel;

  78.     private final Set<CorruptObject> corruptObjects = new HashSet<>();

  80.     private long expectedObjectCount = -1L;

  82.     private long offset;

  84.     private int blockSize;

  86.     /**
  87.      * Constructor for FsckPackParser
  88.      *
  89.      * @param db
  90.      *            the object database which stores repository's data.
  91.      * @param channel
  92.      *            readable channel of the pack file.
  93.      */
  94.     public FsckPackParser(ObjectDatabase db, ReadableChannel channel) {
  95.         super(db, Channels.newInputStream(channel));
  96.         this.channel = channel;
  97.         setCheckObjectCollisions(false);
  98.         this.crc = new CRC32();
  99.         this.blockSize = channel.blockSize() > 0 ? channel.blockSize() : 65536;
  100.     }

  102.     /** {@inheritDoc} */
  103.     @Override
  104.     protected void onPackHeader(long objCnt) throws IOException {
  105.         if (expectedObjectCount >= 0) {
  106.             // Some DFS pack files don't contain the correct object count, e.g.
  107.             // INSERT/RECEIVE packs don't always contain the correct object
  108.             // count in their headers. Overwrite the expected object count
  109.             // after parsing the pack header.
  110.             setExpectedObjectCount(expectedObjectCount);
  111.         }
  112.     }

  114.     /** {@inheritDoc} */
  115.     @Override
  116.     protected void onBeginWholeObject(long streamPosition, int type,
  117.             long inflatedSize) throws IOException {
  118.         crc.reset();
  119.     }

  121.     /** {@inheritDoc} */
  122.     @Override
  123.     protected void onObjectHeader(Source src, byte[] raw, int pos, int len)
  124.             throws IOException {
  125.         crc.update(raw, pos, len);
  126.     }

  128.     /** {@inheritDoc} */
  129.     @Override
  130.     protected void onObjectData(Source src, byte[] raw, int pos, int len)
  131.             throws IOException {
  132.         crc.update(raw, pos, len);
  133.     }

  135.     /** {@inheritDoc} */
  136.     @Override
  137.     protected void onEndWholeObject(PackedObjectInfo info) throws IOException {
  138.         info.setCRC((int) crc.getValue());
  139.     }

  141.     /** {@inheritDoc} */
  142.     @Override
  143.     protected void onBeginOfsDelta(long deltaStreamPosition,
  144.             long baseStreamPosition, long inflatedSize) throws IOException {
  145.         crc.reset();
  146.     }

  148.     /** {@inheritDoc} */
  149.     @Override
  150.     protected void onBeginRefDelta(long deltaStreamPosition, AnyObjectId baseId,
  151.             long inflatedSize) throws IOException {
  152.         crc.reset();
  153.     }

  155.     /** {@inheritDoc} */
  156.     @Override
  157.     protected UnresolvedDelta onEndDelta() throws IOException {
  158.         UnresolvedDelta delta = new UnresolvedDelta();
  159.         delta.setCRC((int) crc.getValue());
  160.         return delta;
  161.     }

  163.     /** {@inheritDoc} */
  164.     @Override
  165.     protected void onInflatedObjectData(PackedObjectInfo obj, int typeCode,
  166.             byte[] data) throws IOException {
  167.         // FsckPackParser ignores this event.
  168.     }

  170.     /** {@inheritDoc} */
  171.     @Override
  172.     protected void verifySafeObject(final AnyObjectId id, final int type,
  173.             final byte[] data) {
  174.         try {
  175.             super.verifySafeObject(id, type, data);
  176.         } catch (CorruptObjectException e) {
  177.             corruptObjects.add(
  178.                     new CorruptObject(id.toObjectId(), type, e.getErrorType()));
  179.         }
  180.     }

  182.     /** {@inheritDoc} */
  183.     @Override
  184.     protected void onPackFooter(byte[] hash) throws IOException {
  185.         // Do nothing.
  186.     }

  188.     /** {@inheritDoc} */
  189.     @Override
  190.     protected boolean onAppendBase(int typeCode, byte[] data,
  191.             PackedObjectInfo info) throws IOException {
  192.         // Do nothing.
  193.         return false;
  194.     }

  196.     /** {@inheritDoc} */
  197.     @Override
  198.     protected void onEndThinPack() throws IOException {
  199.         // Do nothing.
  200.     }

  202.     /** {@inheritDoc} */
  203.     @Override
  204.     protected ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
  205.             ObjectTypeAndSize info) throws IOException {
  206.         crc.reset();
  207.         offset = obj.getOffset();
  208.         return readObjectHeader(info);
  209.     }

  211.     /** {@inheritDoc} */
  212.     @Override
  213.     protected ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
  214.             ObjectTypeAndSize info) throws IOException {
  215.         crc.reset();
  216.         offset = delta.getOffset();
  217.         return readObjectHeader(info);
  218.     }

  220.     /** {@inheritDoc} */
  221.     @Override
  222.     protected int readDatabase(byte[] dst, int pos, int cnt)
  223.             throws IOException {
  224.         // read from input instead of database.
  225.         int n = read(offset, dst, pos, cnt);
  226.         if (n > 0) {
  227.             offset += n;
  228.         }
  229.         return n;
  230.     }

  232.     int read(long channelPosition, byte[] dst, int pos, int cnt)
  233.             throws IOException {
  234.         long block = channelPosition / blockSize;
  235.         byte[] bytes = readFromChannel(block);
  236.         if (bytes == null) {
  237.             return -1;
  238.         }
  239.         int offs = (int) (channelPosition - block * blockSize);
  240.         int bytesToCopy = Math.min(cnt, bytes.length - offs);
  241.         if (bytesToCopy < 1) {
  242.             return -1;
  243.         }
  244.         System.arraycopy(bytes, offs, dst, pos, bytesToCopy);
  245.         return bytesToCopy;
  246.     }

  248.     private byte[] readFromChannel(long block) throws IOException {
  249.         channel.position(block * blockSize);
  250.         ByteBuffer buf = ByteBuffer.allocate(blockSize);
  251.         int totalBytesRead = 0;
  252.         while (totalBytesRead < blockSize) {
  253.             int bytesRead = channel.read(buf);
  254.             if (bytesRead == -1) {
  255.                 if (totalBytesRead == 0) {
  256.                     return null;
  257.                 }
  258.                 return Arrays.copyOf(buf.array(), totalBytesRead);
  259.             }
  260.             totalBytesRead += bytesRead;
  261.         }
  262.         return buf.array();
  263.     }

  265.     /** {@inheritDoc} */
  266.     @Override
  267.     protected boolean checkCRC(int oldCRC) {
  268.         return oldCRC == (int) crc.getValue();
  269.     }

  271.     /** {@inheritDoc} */
  272.     @Override
  273.     protected void onStoreStream(byte[] raw, int pos, int len)
  274.             throws IOException {
  275.         // Do nothing.
  276.     }

  278.     /**
  279.      * Get corrupt objects reported by
  280.      * {@link org.eclipse.jgit.lib.ObjectChecker}
  281.      *
  282.      * @return corrupt objects that are reported by
  283.      *         {@link org.eclipse.jgit.lib.ObjectChecker}.
  284.      */
  285.     public Set<CorruptObject> getCorruptObjects() {
  286.         return corruptObjects;
  287.     }

  289.     /**
  290.      * Verify the existing index file with all objects from the pack.
  291.      *
  292.      * @param idx
  293.      *            index file associate with the pack
  294.      * @throws org.eclipse.jgit.errors.CorruptPackIndexException
  295.      *             when the index file is corrupt.
  296.      */
  297.     public void verifyIndex(PackIndex idx)
  298.             throws CorruptPackIndexException {
  299.         ObjectIdOwnerMap<ObjFromPack> inPack = new ObjectIdOwnerMap<>();
  300.         for (int i = 0; i < getObjectCount(); i++) {
  301.             PackedObjectInfo entry = getObject(i);
  302.             inPack.add(new ObjFromPack(entry));

  304.             long offs = idx.findOffset(entry);
  305.             if (offs == -1) {
  306.                 throw new CorruptPackIndexException(
  307.                         MessageFormat.format(JGitText.get().missingObject,
  308.                                 Integer.valueOf(entry.getType()),
  309.                                 entry.getName()),
  310.                         ErrorType.MISSING_OBJ);
  311.             } else if (offs != entry.getOffset()) {
  312.                 throw new CorruptPackIndexException(MessageFormat
  313.                         .format(JGitText.get().mismatchOffset, entry.getName()),
  314.                         ErrorType.MISMATCH_OFFSET);
  315.             }

  317.             try {
  318.                 if (idx.hasCRC32Support()
  319.                         && (int) idx.findCRC32(entry) != entry.getCRC()) {
  320.                     throw new CorruptPackIndexException(
  321.                             MessageFormat.format(JGitText.get().mismatchCRC,
  322.                                     entry.getName()),
  323.                             ErrorType.MISMATCH_CRC);
  324.                 }
  325.             } catch (MissingObjectException e) {
  326.                 throw new CorruptPackIndexException(MessageFormat
  327.                         .format(JGitText.get().missingCRC, entry.getName()),
  328.                         ErrorType.MISSING_CRC);
  329.             }
  330.         }

  332.         for (MutableEntry entry : idx) {
  333.             if (!inPack.contains(entry.toObjectId())) {
  334.                 throw new CorruptPackIndexException(MessageFormat.format(
  335.                         JGitText.get().unknownObjectInIndex, entry.name()),
  336.                         ErrorType.UNKNOWN_OBJ);
  337.             }
  338.         }
  339.     }

  341.     /**
  342.      * Set the object count for overwriting the expected object count from pack
  343.      * header.
  344.      *
  345.      * @param objectCount
  346.      *            the actual expected object count.
  347.      */
  348.     public void overwriteObjectCount(long objectCount) {
  349.         this.expectedObjectCount = objectCount;
  350.     }

  352.     static class ObjFromPack extends ObjectIdOwnerMap.Entry {
  353.         ObjFromPack(AnyObjectId id) {
  354.             super(id);
  355.         }
  356.     }
  357. }
Created with JaCoCo 0.8.4.201905082037