View Javadoc
1   /*
2    * Copyright (C) 2017, Google Inc. and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  
11  package org.eclipse.jgit.internal.fsck;
12  
13  import java.io.IOException;
14  import java.nio.ByteBuffer;
15  import java.nio.channels.Channels;
16  import java.text.MessageFormat;
17  import java.util.Arrays;
18  import java.util.HashSet;
19  import java.util.Set;
20  import java.util.zip.CRC32;
21  
22  import org.eclipse.jgit.errors.CorruptObjectException;
23  import org.eclipse.jgit.errors.CorruptPackIndexException;
24  import org.eclipse.jgit.errors.CorruptPackIndexException.ErrorType;
25  import org.eclipse.jgit.errors.MissingObjectException;
26  import org.eclipse.jgit.internal.JGitText;
27  import org.eclipse.jgit.internal.fsck.FsckError.CorruptObject;
28  import org.eclipse.jgit.internal.storage.dfs.ReadableChannel;
29  import org.eclipse.jgit.internal.storage.file.PackIndex;
30  import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
31  import org.eclipse.jgit.lib.AnyObjectId;
32  import org.eclipse.jgit.lib.ObjectDatabase;
33  import org.eclipse.jgit.lib.ObjectIdOwnerMap;
34  import org.eclipse.jgit.transport.PackParser;
35  import org.eclipse.jgit.transport.PackedObjectInfo;
36  
37  /**
38   * A read-only pack parser for object validity checking.
39   */
40  public class FsckPackParser extends PackParser {
41  	private final CRC32 crc;
42  
43  	private final ReadableChannel channel;
44  
45  	private final Set<CorruptObject> corruptObjects = new HashSet<>();
46  
47  	private long expectedObjectCount = -1L;
48  
49  	private long offset;
50  
51  	private int blockSize;
52  
53  	/**
54  	 * Constructor for FsckPackParser
55  	 *
56  	 * @param db
57  	 *            the object database which stores repository's data.
58  	 * @param channel
59  	 *            readable channel of the pack file.
60  	 */
61  	public FsckPackParser(ObjectDatabase db, ReadableChannel channel) {
62  		super(db, Channels.newInputStream(channel));
63  		this.channel = channel;
64  		setCheckObjectCollisions(false);
65  		this.crc = new CRC32();
66  		this.blockSize = channel.blockSize() > 0 ? channel.blockSize() : 65536;
67  	}
68  
69  	/** {@inheritDoc} */
70  	@Override
71  	protected void onPackHeader(long objCnt) throws IOException {
72  		if (expectedObjectCount >= 0) {
73  			// Some DFS pack files don't contain the correct object count, e.g.
74  			// INSERT/RECEIVE packs don't always contain the correct object
75  			// count in their headers. Overwrite the expected object count
76  			// after parsing the pack header.
77  			setExpectedObjectCount(expectedObjectCount);
78  		}
79  	}
80  
81  	/** {@inheritDoc} */
82  	@Override
83  	protected void onBeginWholeObject(long streamPosition, int type,
84  			long inflatedSize) throws IOException {
85  		crc.reset();
86  	}
87  
88  	/** {@inheritDoc} */
89  	@Override
90  	protected void onObjectHeader(Source src, byte[] raw, int pos, int len)
91  			throws IOException {
92  		crc.update(raw, pos, len);
93  	}
94  
95  	/** {@inheritDoc} */
96  	@Override
97  	protected void onObjectData(Source src, byte[] raw, int pos, int len)
98  			throws IOException {
99  		crc.update(raw, pos, len);
100 	}
101 
102 	/** {@inheritDoc} */
103 	@Override
104 	protected void onEndWholeObject(PackedObjectInfo info) throws IOException {
105 		info.setCRC((int) crc.getValue());
106 	}
107 
108 	/** {@inheritDoc} */
109 	@Override
110 	protected void onBeginOfsDelta(long deltaStreamPosition,
111 			long baseStreamPosition, long inflatedSize) throws IOException {
112 		crc.reset();
113 	}
114 
115 	/** {@inheritDoc} */
116 	@Override
117 	protected void onBeginRefDelta(long deltaStreamPosition, AnyObjectId baseId,
118 			long inflatedSize) throws IOException {
119 		crc.reset();
120 	}
121 
122 	/** {@inheritDoc} */
123 	@Override
124 	protected UnresolvedDelta onEndDelta() throws IOException {
125 		UnresolvedDelta delta = new UnresolvedDelta();
126 		delta.setCRC((int) crc.getValue());
127 		return delta;
128 	}
129 
130 	/** {@inheritDoc} */
131 	@Override
132 	protected void onInflatedObjectData(PackedObjectInfo obj, int typeCode,
133 			byte[] data) throws IOException {
134 		// FsckPackParser ignores this event.
135 	}
136 
137 	/** {@inheritDoc} */
138 	@Override
139 	protected void verifySafeObject(final AnyObjectId id, final int type,
140 			final byte[] data) {
141 		try {
142 			super.verifySafeObject(id, type, data);
143 		} catch (CorruptObjectException e) {
144 			corruptObjects.add(
145 					new CorruptObject(id.toObjectId(), type, e.getErrorType()));
146 		}
147 	}
148 
149 	/** {@inheritDoc} */
150 	@Override
151 	protected void onPackFooter(byte[] hash) throws IOException {
152 		// Do nothing.
153 	}
154 
155 	/** {@inheritDoc} */
156 	@Override
157 	protected boolean onAppendBase(int typeCode, byte[] data,
158 			PackedObjectInfo info) throws IOException {
159 		// Do nothing.
160 		return false;
161 	}
162 
163 	/** {@inheritDoc} */
164 	@Override
165 	protected void onEndThinPack() throws IOException {
166 		// Do nothing.
167 	}
168 
169 	/** {@inheritDoc} */
170 	@Override
171 	protected ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
172 			ObjectTypeAndSize info) throws IOException {
173 		crc.reset();
174 		offset = obj.getOffset();
175 		return readObjectHeader(info);
176 	}
177 
178 	/** {@inheritDoc} */
179 	@Override
180 	protected ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
181 			ObjectTypeAndSize info) throws IOException {
182 		crc.reset();
183 		offset = delta.getOffset();
184 		return readObjectHeader(info);
185 	}
186 
187 	/** {@inheritDoc} */
188 	@Override
189 	protected int readDatabase(byte[] dst, int pos, int cnt)
190 			throws IOException {
191 		// read from input instead of database.
192 		int n = read(offset, dst, pos, cnt);
193 		if (n > 0) {
194 			offset += n;
195 		}
196 		return n;
197 	}
198 
199 	int read(long channelPosition, byte[] dst, int pos, int cnt)
200 			throws IOException {
201 		long block = channelPosition / blockSize;
202 		byte[] bytes = readFromChannel(block);
203 		if (bytes == null) {
204 			return -1;
205 		}
206 		int offs = (int) (channelPosition - block * blockSize);
207 		int bytesToCopy = Math.min(cnt, bytes.length - offs);
208 		if (bytesToCopy < 1) {
209 			return -1;
210 		}
211 		System.arraycopy(bytes, offs, dst, pos, bytesToCopy);
212 		return bytesToCopy;
213 	}
214 
215 	private byte[] readFromChannel(long block) throws IOException {
216 		channel.position(block * blockSize);
217 		ByteBuffer buf = ByteBuffer.allocate(blockSize);
218 		int totalBytesRead = 0;
219 		while (totalBytesRead < blockSize) {
220 			int bytesRead = channel.read(buf);
221 			if (bytesRead == -1) {
222 				if (totalBytesRead == 0) {
223 					return null;
224 				}
225 				return Arrays.copyOf(buf.array(), totalBytesRead);
226 			}
227 			totalBytesRead += bytesRead;
228 		}
229 		return buf.array();
230 	}
231 
232 	/** {@inheritDoc} */
233 	@Override
234 	protected boolean checkCRC(int oldCRC) {
235 		return oldCRC == (int) crc.getValue();
236 	}
237 
238 	/** {@inheritDoc} */
239 	@Override
240 	protected void onStoreStream(byte[] raw, int pos, int len)
241 			throws IOException {
242 		// Do nothing.
243 	}
244 
245 	/**
246 	 * Get corrupt objects reported by
247 	 * {@link org.eclipse.jgit.lib.ObjectChecker}
248 	 *
249 	 * @return corrupt objects that are reported by
250 	 *         {@link org.eclipse.jgit.lib.ObjectChecker}.
251 	 */
252 	public Set<CorruptObject> getCorruptObjects() {
253 		return corruptObjects;
254 	}
255 
256 	/**
257 	 * Verify the existing index file with all objects from the pack.
258 	 *
259 	 * @param idx
260 	 *            index file associate with the pack
261 	 * @throws org.eclipse.jgit.errors.CorruptPackIndexException
262 	 *             when the index file is corrupt.
263 	 */
264 	public void verifyIndex(PackIndex idx)
265 			throws CorruptPackIndexException {
266 		ObjectIdOwnerMap<ObjFromPack> inPack = new ObjectIdOwnerMap<>();
267 		for (int i = 0; i < getObjectCount(); i++) {
268 			PackedObjectInfo entry = getObject(i);
269 			inPack.add(new ObjFromPack(entry));
270 
271 			long offs = idx.findOffset(entry);
272 			if (offs == -1) {
273 				throw new CorruptPackIndexException(
274 						MessageFormat.format(JGitText.get().missingObject,
275 								Integer.valueOf(entry.getType()),
276 								entry.getName()),
277 						ErrorType.MISSING_OBJ);
278 			} else if (offs != entry.getOffset()) {
279 				throw new CorruptPackIndexException(MessageFormat
280 						.format(JGitText.get().mismatchOffset, entry.getName()),
281 						ErrorType.MISMATCH_OFFSET);
282 			}
283 
284 			try {
285 				if (idx.hasCRC32Support()
286 						&& (int) idx.findCRC32(entry) != entry.getCRC()) {
287 					throw new CorruptPackIndexException(
288 							MessageFormat.format(JGitText.get().mismatchCRC,
289 									entry.getName()),
290 							ErrorType.MISMATCH_CRC);
291 				}
292 			} catch (MissingObjectException e) {
293 				CorruptPackIndexException cpe = new CorruptPackIndexException(
294 						MessageFormat.format(JGitText.get().missingCRC,
295 								entry.getName()),
296 						ErrorType.MISSING_CRC);
297 				cpe.initCause(e);
298 				throw cpe;
299 			}
300 		}
301 
302 		for (MutableEntry entry : idx) {
303 			if (!inPack.contains(entry.toObjectId())) {
304 				throw new CorruptPackIndexException(MessageFormat.format(
305 						JGitText.get().unknownObjectInIndex, entry.name()),
306 						ErrorType.UNKNOWN_OBJ);
307 			}
308 		}
309 	}
310 
311 	/**
312 	 * Set the object count for overwriting the expected object count from pack
313 	 * header.
314 	 *
315 	 * @param objectCount
316 	 *            the actual expected object count.
317 	 */
318 	public void overwriteObjectCount(long objectCount) {
319 		this.expectedObjectCount = objectCount;
320 	}
321 
322 	static class ObjFromPack extends ObjectIdOwnerMap.Entry {
323 		ObjFromPack(AnyObjectId id) {
324 			super(id);
325 		}
326 	}
327 }