1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.internal.storage.file;
45
46 import static java.util.Comparator.comparing;
47 import static java.util.stream.Collectors.toList;
48
49 import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
50 import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
51 import static org.hamcrest.Matchers.greaterThan;
52 import static org.hamcrest.Matchers.lessThan;
53 import static org.junit.Assert.assertArrayEquals;
54 import static org.junit.Assert.assertEquals;
55 import static org.junit.Assert.assertNotEquals;
56 import static org.junit.Assert.assertThat;
57 import static org.junit.Assert.fail;
58
59 import java.io.ByteArrayInputStream;
60 import java.io.File;
61 import java.io.IOException;
62 import java.nio.file.FileVisitResult;
63 import java.nio.file.Files;
64 import java.nio.file.Path;
65 import java.nio.file.SimpleFileVisitor;
66 import java.nio.file.attribute.BasicFileAttributes;
67 import java.util.ArrayList;
68 import java.util.Collection;
69 import java.util.List;
70 import java.util.Random;
71 import java.util.function.Predicate;
72 import java.util.regex.Matcher;
73 import java.util.regex.Pattern;
74
75 import org.eclipse.jgit.dircache.DirCache;
76 import org.eclipse.jgit.dircache.DirCacheBuilder;
77 import org.eclipse.jgit.dircache.DirCacheEntry;
78 import org.eclipse.jgit.errors.MissingObjectException;
79 import org.eclipse.jgit.junit.RepositoryTestCase;
80 import org.eclipse.jgit.lib.CommitBuilder;
81 import org.eclipse.jgit.lib.Constants;
82 import org.eclipse.jgit.lib.FileMode;
83 import org.eclipse.jgit.lib.ObjectId;
84 import org.eclipse.jgit.lib.ObjectLoader;
85 import org.eclipse.jgit.lib.ObjectReader;
86 import org.eclipse.jgit.lib.ObjectStream;
87 import org.eclipse.jgit.storage.file.WindowCacheConfig;
88 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
89 import org.eclipse.jgit.util.IO;
90 import org.junit.After;
91 import org.junit.Before;
92 import org.junit.Test;
93
94 @SuppressWarnings("boxing")
95 public class PackInserterTest extends RepositoryTestCase {
96 private WindowCacheConfig origWindowCacheConfig;
97
98 private static final Random random = new Random(0);
99
100 @Before
101 public void setWindowCacheConfig() {
102 origWindowCacheConfig = new WindowCacheConfig();
103 origWindowCacheConfig.install();
104 }
105
106 @After
107 public void resetWindowCacheConfig() {
108 origWindowCacheConfig.install();
109 }
110
111 @Before
112 public void emptyAtSetUp() throws Exception {
113 assertEquals(0, listPacks().size());
114 assertNoObjects();
115 }
116
117 @Test
118 public void noFlush() throws Exception {
119 try (PackInserter ins = newInserter()) {
120 ins.insert(OBJ_BLOB, Constants.encode("foo contents"));
121
122 }
123 assertNoObjects();
124 }
125
126 @Test
127 public void flushEmptyPack() throws Exception {
128 try (PackInserter ins = newInserter()) {
129 ins.flush();
130 }
131 assertNoObjects();
132 }
133
134 @Test
135 public void singlePack() throws Exception {
136 ObjectId blobId;
137 byte[] blob = Constants.encode("foo contents");
138 ObjectId treeId;
139 ObjectId commitId;
140 byte[] commit;
141 try (PackInserter ins = newInserter()) {
142 blobId = ins.insert(OBJ_BLOB, blob);
143
144 DirCache dc = DirCache.newInCore();
145 DirCacheBuilder b = dc.builder();
146 DirCacheEntry dce = new DirCacheEntry("foo");
147 dce.setFileMode(FileMode.REGULAR_FILE);
148 dce.setObjectId(blobId);
149 b.add(dce);
150 b.finish();
151 treeId = dc.writeTree(ins);
152
153 CommitBuilder cb = new CommitBuilder();
154 cb.setTreeId(treeId);
155 cb.setAuthor(author);
156 cb.setCommitter(committer);
157 cb.setMessage("Commit message");
158 commit = cb.toByteArray();
159 commitId = ins.insert(cb);
160 ins.flush();
161 }
162
163 assertPacksOnly();
164 List<PackFile> packs = listPacks();
165 assertEquals(1, packs.size());
166 assertEquals(3, packs.get(0).getObjectCount());
167
168 try (ObjectReader reader = db.newObjectReader()) {
169 assertBlob(reader, blobId, blob);
170
171 CanonicalTreeParser treeParser =
172 new CanonicalTreeParser(null, reader, treeId);
173 assertEquals("foo", treeParser.getEntryPathString());
174 assertEquals(blobId, treeParser.getEntryObjectId());
175
176 ObjectLoader commitLoader = reader.open(commitId);
177 assertEquals(OBJ_COMMIT, commitLoader.getType());
178 assertArrayEquals(commit, commitLoader.getBytes());
179 }
180 }
181
182 @Test
183 public void multiplePacks() throws Exception {
184 ObjectId blobId1;
185 ObjectId blobId2;
186 byte[] blob1 = Constants.encode("blob1");
187 byte[] blob2 = Constants.encode("blob2");
188
189 try (PackInserter ins = newInserter()) {
190 blobId1 = ins.insert(OBJ_BLOB, blob1);
191 ins.flush();
192 blobId2 = ins.insert(OBJ_BLOB, blob2);
193 ins.flush();
194 }
195
196 assertPacksOnly();
197 List<PackFile> packs = listPacks();
198 assertEquals(2, packs.size());
199 assertEquals(1, packs.get(0).getObjectCount());
200 assertEquals(1, packs.get(1).getObjectCount());
201
202 try (ObjectReader reader = db.newObjectReader()) {
203 assertBlob(reader, blobId1, blob1);
204 assertBlob(reader, blobId2, blob2);
205 }
206 }
207
208 @Test
209 public void largeBlob() throws Exception {
210 ObjectId blobId;
211 byte[] blob = newLargeBlob();
212 try (PackInserter ins = newInserter()) {
213 assertThat(blob.length, greaterThan(ins.getBufferSize()));
214 blobId =
215 ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob));
216 ins.flush();
217 }
218
219 assertPacksOnly();
220 Collection<PackFile> packs = listPacks();
221 assertEquals(1, packs.size());
222 PackFile p = packs.iterator().next();
223 assertEquals(1, p.getObjectCount());
224
225 try (ObjectReader reader = db.newObjectReader()) {
226 assertBlob(reader, blobId, blob);
227 }
228 }
229
230 @Test
231 public void overwriteExistingPack() throws Exception {
232 ObjectId blobId;
233 byte[] blob = Constants.encode("foo contents");
234
235 try (PackInserter ins = newInserter()) {
236 blobId = ins.insert(OBJ_BLOB, blob);
237 ins.flush();
238 }
239
240 assertPacksOnly();
241 List<PackFile> packs = listPacks();
242 assertEquals(1, packs.size());
243 PackFile pack = packs.get(0);
244 assertEquals(1, pack.getObjectCount());
245
246 String inode = getInode(pack.getPackFile());
247
248 try (PackInserter ins = newInserter()) {
249 ins.checkExisting(false);
250 assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
251 ins.flush();
252 }
253
254 assertPacksOnly();
255 packs = listPacks();
256 assertEquals(1, packs.size());
257 pack = packs.get(0);
258 assertEquals(1, pack.getObjectCount());
259
260 if (inode != null) {
261
262
263 assertNotEquals(inode, getInode(pack.getPackFile()));
264 }
265 }
266
267 @Test
268 public void checkExisting() throws Exception {
269 ObjectId blobId;
270 byte[] blob = Constants.encode("foo contents");
271
272 try (PackInserter ins = newInserter()) {
273 blobId = ins.insert(OBJ_BLOB, blob);
274 ins.insert(OBJ_BLOB, Constants.encode("another blob"));
275 ins.flush();
276 }
277
278 assertPacksOnly();
279 assertEquals(1, listPacks().size());
280
281 try (PackInserter ins = newInserter()) {
282 assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
283 ins.flush();
284 }
285
286 assertPacksOnly();
287 assertEquals(1, listPacks().size());
288
289 try (PackInserter ins = newInserter()) {
290 ins.checkExisting(false);
291 assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
292 ins.flush();
293 }
294
295 assertPacksOnly();
296 assertEquals(2, listPacks().size());
297
298 try (ObjectReader reader = db.newObjectReader()) {
299 assertBlob(reader, blobId, blob);
300 }
301 }
302
303 @Test
304 public void insertSmallInputStreamRespectsCheckExisting() throws Exception {
305 ObjectId blobId;
306 byte[] blob = Constants.encode("foo contents");
307 try (PackInserter ins = newInserter()) {
308 assertThat(blob.length, lessThan(ins.getBufferSize()));
309 blobId = ins.insert(OBJ_BLOB, blob);
310 ins.insert(OBJ_BLOB, Constants.encode("another blob"));
311 ins.flush();
312 }
313
314 assertPacksOnly();
315 assertEquals(1, listPacks().size());
316
317 try (PackInserter ins = newInserter()) {
318 assertEquals(blobId,
319 ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)));
320 ins.flush();
321 }
322
323 assertPacksOnly();
324 assertEquals(1, listPacks().size());
325 }
326
327 @Test
328 public void insertLargeInputStreamBypassesCheckExisting() throws Exception {
329 ObjectId blobId;
330 byte[] blob = newLargeBlob();
331
332 try (PackInserter ins = newInserter()) {
333 assertThat(blob.length, greaterThan(ins.getBufferSize()));
334 blobId = ins.insert(OBJ_BLOB, blob);
335 ins.insert(OBJ_BLOB, Constants.encode("another blob"));
336 ins.flush();
337 }
338
339 assertPacksOnly();
340 assertEquals(1, listPacks().size());
341
342 try (PackInserter ins = newInserter()) {
343 assertEquals(blobId,
344 ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)));
345 ins.flush();
346 }
347
348 assertPacksOnly();
349 assertEquals(2, listPacks().size());
350 }
351
352 @Test
353 public void readBackSmallFiles() throws Exception {
354 ObjectId blobId1;
355 ObjectId blobId2;
356 ObjectId blobId3;
357 byte[] blob1 = Constants.encode("blob1");
358 byte[] blob2 = Constants.encode("blob2");
359 byte[] blob3 = Constants.encode("blob3");
360 try (PackInserter ins = newInserter()) {
361 assertThat(blob1.length, lessThan(ins.getBufferSize()));
362 blobId1 = ins.insert(OBJ_BLOB, blob1);
363
364 try (ObjectReader reader = ins.newReader()) {
365 assertBlob(reader, blobId1, blob1);
366 }
367
368
369 blobId2 = ins.insert(OBJ_BLOB, blob2);
370 ins.flush();
371
372 blobId3 = ins.insert(OBJ_BLOB, blob3);
373 }
374
375 assertPacksOnly();
376 List<PackFile> packs = listPacks();
377 assertEquals(1, packs.size());
378 assertEquals(2, packs.get(0).getObjectCount());
379
380 try (ObjectReader reader = db.newObjectReader()) {
381 assertBlob(reader, blobId1, blob1);
382 assertBlob(reader, blobId2, blob2);
383
384 try {
385 reader.open(blobId3);
386 fail("Expected MissingObjectException");
387 } catch (MissingObjectException expected) {
388
389 }
390 }
391 }
392
393 @Test
394 public void readBackLargeFile() throws Exception {
395 ObjectId blobId;
396 byte[] blob = newLargeBlob();
397
398 WindowCacheConfig wcc = new WindowCacheConfig();
399 wcc.setStreamFileThreshold(1024);
400 wcc.install();
401 try (ObjectReader reader = db.newObjectReader()) {
402 assertThat(blob.length, greaterThan(reader.getStreamFileThreshold()));
403 }
404
405 try (PackInserter ins = newInserter()) {
406 blobId = ins.insert(OBJ_BLOB, blob);
407
408 try (ObjectReader reader = ins.newReader()) {
409
410 assertThat(blob.length, greaterThan(reader.getStreamFileThreshold()));
411 assertBlob(reader, blobId, blob);
412 }
413 }
414
415 assertPacksOnly();
416
417
418 assertEquals(0, listPacks().size());
419
420 try (ObjectReader reader = db.newObjectReader()) {
421 try {
422 reader.open(blobId);
423 fail("Expected MissingObjectException");
424 } catch (MissingObjectException expected) {
425
426 }
427 }
428 }
429
430 @Test
431 public void readBackFallsBackToRepo() throws Exception {
432 ObjectId blobId;
433 byte[] blob = Constants.encode("foo contents");
434 try (PackInserter ins = newInserter()) {
435 assertThat(blob.length, lessThan(ins.getBufferSize()));
436 blobId = ins.insert(OBJ_BLOB, blob);
437 ins.flush();
438 }
439
440 try (PackInserter ins = newInserter();
441 ObjectReader reader = ins.newReader()) {
442 assertBlob(reader, blobId, blob);
443 }
444 }
445
446 @Test
447 public void readBackSmallObjectBeforeLargeObject() throws Exception {
448 WindowCacheConfig wcc = new WindowCacheConfig();
449 wcc.setStreamFileThreshold(1024);
450 wcc.install();
451
452 ObjectId blobId1;
453 ObjectId blobId2;
454 ObjectId largeId;
455 byte[] blob1 = Constants.encode("blob1");
456 byte[] blob2 = Constants.encode("blob2");
457 byte[] largeBlob = newLargeBlob();
458 try (PackInserter ins = newInserter()) {
459 assertThat(blob1.length, lessThan(ins.getBufferSize()));
460 assertThat(largeBlob.length, greaterThan(ins.getBufferSize()));
461
462 blobId1 = ins.insert(OBJ_BLOB, blob1);
463 largeId = ins.insert(OBJ_BLOB, largeBlob);
464
465 try (ObjectReader reader = ins.newReader()) {
466
467
468
469
470
471
472 assertBlob(reader, blobId1, blob1);
473 }
474
475 blobId2 = ins.insert(OBJ_BLOB, blob2);
476
477 try (ObjectReader reader = ins.newReader()) {
478 assertBlob(reader, blobId1, blob1);
479 assertBlob(reader, blobId2, blob2);
480 assertBlob(reader, largeId, largeBlob);
481 }
482
483 ins.flush();
484 }
485
486 try (ObjectReader reader = db.newObjectReader()) {
487 assertBlob(reader, blobId1, blob1);
488 assertBlob(reader, blobId2, blob2);
489 assertBlob(reader, largeId, largeBlob);
490 }
491 }
492
493 private List<PackFile> listPacks() throws Exception {
494 List<PackFile> fromOpenDb = listPacks(db);
495 List<PackFile> reopened;
496 try (FileRepository db2 = new FileRepository(db.getDirectory())) {
497 reopened = listPacks(db2);
498 }
499 assertEquals(fromOpenDb.size(), reopened.size());
500 for (int i = 0 ; i < fromOpenDb.size(); i++) {
501 PackFile a = fromOpenDb.get(i);
502 PackFile b = reopened.get(i);
503 assertEquals(a.getPackName(), b.getPackName());
504 assertEquals(
505 a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath());
506 assertEquals(a.getObjectCount(), b.getObjectCount());
507 a.getObjectCount();
508 }
509 return fromOpenDb;
510 }
511
512 private static List<PackFile> listPacks(FileRepository db) throws Exception {
513 return db.getObjectDatabase().getPacks().stream()
514 .sorted(comparing(PackFile::getPackName)).collect(toList());
515 }
516
517 private PackInserter newInserter() {
518 return db.getObjectDatabase().newPackInserter();
519 }
520
521 private static byte[] newLargeBlob() {
522 byte[] blob = new byte[10240];
523 random.nextBytes(blob);
524 return blob;
525 }
526
527 private static String getInode(File f) throws Exception {
528 BasicFileAttributes attrs = Files.readAttributes(
529 f.toPath(), BasicFileAttributes.class);
530 Object k = attrs.fileKey();
531 if (k == null) {
532 return null;
533 }
534 Pattern p = Pattern.compile("^\\(dev=[^,]*,ino=(\\d+)\\)$");
535 Matcher m = p.matcher(k.toString());
536 return m.matches() ? m.group(1) : null;
537 }
538
539 private static void assertBlob(ObjectReader reader, ObjectId id,
540 byte[] expected) throws Exception {
541 ObjectLoader loader = reader.open(id);
542 assertEquals(OBJ_BLOB, loader.getType());
543 assertEquals(expected.length, loader.getSize());
544 try (ObjectStream s = loader.openStream()) {
545 int n = (int) s.getSize();
546 byte[] actual = new byte[n];
547 assertEquals(n, IO.readFully(s, actual, 0));
548 assertArrayEquals(expected, actual);
549 }
550 }
551
552 private void assertPacksOnly() throws Exception {
553 new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx"))
554 .assertNoBadFiles(db.getObjectDatabase().getDirectory());
555 }
556
557 private void assertNoObjects() throws Exception {
558 new BadFileCollector(f -> true)
559 .assertNoBadFiles(db.getObjectDatabase().getDirectory());
560 }
561
562 private static class BadFileCollector extends SimpleFileVisitor<Path> {
563 private final Predicate<String> badName;
564 private List<String> bad;
565
566 BadFileCollector(Predicate<String> badName) {
567 this.badName = badName;
568 }
569
570 void assertNoBadFiles(File f) throws IOException {
571 bad = new ArrayList<>();
572 Files.walkFileTree(f.toPath(), this);
573 if (!bad.isEmpty()) {
574 fail("unexpected files in object directory: " + bad);
575 }
576 }
577
578 @Override
579 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
580 Path fileName = file.getFileName();
581 if (fileName != null) {
582 String name = fileName.toString();
583 if (!attrs.isDirectory() && badName.test(name)) {
584 bad.add(name);
585 }
586 }
587 return FileVisitResult.CONTINUE;
588 }
589 }
590 }