1 package org.eclipse.jgit.internal.storage.dfs;
2
3 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
4 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST;
5 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
6 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
7 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
8 import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
9 import static org.junit.Assert.assertEquals;
10 import static org.junit.Assert.assertFalse;
11 import static org.junit.Assert.assertNotNull;
12 import static org.junit.Assert.assertNull;
13 import static org.junit.Assert.assertSame;
14 import static org.junit.Assert.assertTrue;
15 import static org.junit.Assert.fail;
16
17 import java.io.IOException;
18 import java.util.Collections;
19 import java.util.concurrent.TimeUnit;
20
21 import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
22 import org.eclipse.jgit.internal.storage.dfs.DfsRefDatabase;
23 import org.eclipse.jgit.internal.storage.reftable.RefCursor;
24 import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
25 import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
26 import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
27 import org.eclipse.jgit.junit.MockSystemReader;
28 import org.eclipse.jgit.junit.TestRepository;
29 import org.eclipse.jgit.lib.AnyObjectId;
30 import org.eclipse.jgit.lib.BatchRefUpdate;
31 import org.eclipse.jgit.lib.NullProgressMonitor;
32 import org.eclipse.jgit.lib.ObjectId;
33 import org.eclipse.jgit.lib.ObjectIdRef;
34 import org.eclipse.jgit.lib.Ref;
35 import org.eclipse.jgit.lib.Repository;
36 import org.eclipse.jgit.revwalk.RevBlob;
37 import org.eclipse.jgit.revwalk.RevCommit;
38 import org.eclipse.jgit.revwalk.RevWalk;
39 import org.eclipse.jgit.storage.pack.PackConfig;
40 import org.eclipse.jgit.transport.ReceiveCommand;
41 import org.eclipse.jgit.util.SystemReader;
42 import org.junit.After;
43 import org.junit.Before;
44 import org.junit.Test;
45
46
47 public class DfsGarbageCollectorTest {
48 private TestRepository<InMemoryRepository> git;
49 private InMemoryRepository repo;
50 private DfsObjDatabase odb;
51 private MockSystemReader mockSystemReader;
52
53 @Before
54 public void setUp() throws IOException {
55 DfsRepositoryDescription desc = new DfsRepositoryDescription("test");
56 git = new TestRepository<>(new InMemoryRepository(desc));
57 repo = git.getRepository();
58 odb = repo.getObjectDatabase();
59 mockSystemReader = new MockSystemReader();
60 SystemReader.setInstance(mockSystemReader);
61 }
62
63 @After
64 public void tearDown() {
65 SystemReader.setInstance(null);
66 }
67
68 @Test
69 public void testCollectionWithNoGarbage() throws Exception {
70 RevCommit commit0 = commit().message("0").create();
71 RevCommit commit1 = commit().message("1").parent(commit0).create();
72 git.update("master", commit1);
73
74 assertTrue("commit0 reachable", isReachable(repo, commit0));
75 assertTrue("commit1 reachable", isReachable(repo, commit1));
76
77
78 assertEquals(2, odb.getPacks().length);
79 for (DfsPackFile pack : odb.getPacks()) {
80 assertEquals(INSERT, pack.getPackDescription().getPackSource());
81 }
82
83 gcNoTtl();
84
85
86 assertEquals(1, odb.getPacks().length);
87 DfsPackFile pack = odb.getPacks()[0];
88 assertEquals(GC, pack.getPackDescription().getPackSource());
89 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
90 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
91 }
92
93 @Test
94 public void testRacyNoReusePrefersSmaller() throws Exception {
95 StringBuilder msg = new StringBuilder();
96 for (int i = 0; i < 100; i++) {
97 msg.append(i).append(": i am a teapot\n");
98 }
99 RevBlob a = git.blob(msg.toString());
100 RevCommit c0 = git.commit()
101 .add("tea", a)
102 .message("0")
103 .create();
104
105 msg.append("short and stout\n");
106 RevBlob b = git.blob(msg.toString());
107 RevCommit c1 = git.commit().parent(c0).tick(1)
108 .add("tea", b)
109 .message("1")
110 .create();
111 git.update("master", c1);
112
113 PackConfig cfg = new PackConfig();
114 cfg.setReuseObjects(false);
115 cfg.setReuseDeltas(false);
116 cfg.setDeltaCompress(false);
117 cfg.setThreads(1);
118 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
119 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
120 gc.setPackConfig(cfg);
121 run(gc);
122
123 assertEquals(1, odb.getPacks().length);
124 DfsPackDescription large = odb.getPacks()[0].getPackDescription();
125 assertSame(PackSource.GC, large.getPackSource());
126
127 cfg.setDeltaCompress(true);
128 gc = new DfsGarbageCollector(repo);
129 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
130 gc.setPackConfig(cfg);
131 run(gc);
132
133 assertEquals(1, odb.getPacks().length);
134 DfsPackDescription small = odb.getPacks()[0].getPackDescription();
135 assertSame(PackSource.GC, small.getPackSource());
136 assertTrue(
137 "delta compression pack is smaller",
138 small.getFileSize(PACK) < large.getFileSize(PACK));
139 assertTrue(
140 "large pack is older",
141 large.getLastModified() < small.getLastModified());
142
143
144 odb.commitPack(Collections.singleton(large), null);
145 odb.clearCache();
146 assertEquals(2, odb.getPacks().length);
147
148 gc = new DfsGarbageCollector(repo);
149 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
150 run(gc);
151
152 assertEquals(1, odb.getPacks().length);
153 DfsPackDescription rebuilt = odb.getPacks()[0].getPackDescription();
154 assertEquals(small.getFileSize(PACK), rebuilt.getFileSize(PACK));
155 }
156
157 @Test
158 public void testCollectionWithGarbage() throws Exception {
159 RevCommit commit0 = commit().message("0").create();
160 RevCommit commit1 = commit().message("1").parent(commit0).create();
161 git.update("master", commit0);
162
163 assertTrue("commit0 reachable", isReachable(repo, commit0));
164 assertFalse("commit1 garbage", isReachable(repo, commit1));
165 gcNoTtl();
166
167 assertEquals(2, odb.getPacks().length);
168 DfsPackFile gc = null;
169 DfsPackFile garbage = null;
170 for (DfsPackFile pack : odb.getPacks()) {
171 DfsPackDescription d = pack.getPackDescription();
172 if (d.getPackSource() == GC) {
173 gc = pack;
174 } else if (d.getPackSource() == UNREACHABLE_GARBAGE) {
175 garbage = pack;
176 } else {
177 fail("unexpected " + d.getPackSource());
178 }
179 }
180
181 assertNotNull("created GC pack", gc);
182 assertTrue(isObjectInPack(commit0, gc));
183
184 assertNotNull("created UNREACHABLE_GARBAGE pack", garbage);
185 assertTrue(isObjectInPack(commit1, garbage));
186 }
187
188 @Test
189 public void testCollectionWithGarbageAndGarbagePacksPurged()
190 throws Exception {
191 RevCommit commit0 = commit().message("0").create();
192 RevCommit commit1 = commit().message("1").parent(commit0).create();
193 git.update("master", commit0);
194
195 gcWithTtl();
196
197
198 assertEquals(2, odb.getPacks().length);
199 boolean gcPackFound = false;
200 boolean garbagePackFound = false;
201 for (DfsPackFile pack : odb.getPacks()) {
202 DfsPackDescription d = pack.getPackDescription();
203 if (d.getPackSource() == GC) {
204 gcPackFound = true;
205 assertTrue("has commit0", isObjectInPack(commit0, pack));
206 assertFalse("no commit1", isObjectInPack(commit1, pack));
207 } else if (d.getPackSource() == UNREACHABLE_GARBAGE) {
208 garbagePackFound = true;
209 assertFalse("no commit0", isObjectInPack(commit0, pack));
210 assertTrue("has commit1", isObjectInPack(commit1, pack));
211 } else {
212 fail("unexpected " + d.getPackSource());
213 }
214 }
215 assertTrue("gc pack found", gcPackFound);
216 assertTrue("garbage pack found", garbagePackFound);
217
218 gcWithTtl();
219
220 DfsPackFile[] packs = odb.getPacks();
221 assertEquals(1, packs.length);
222
223 assertEquals(GC, packs[0].getPackDescription().getPackSource());
224 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
225 assertFalse("no commit1", isObjectInPack(commit1, packs[0]));
226 }
227
228 @Test
229 public void testCollectionWithGarbageAndRereferencingGarbage()
230 throws Exception {
231 RevCommit commit0 = commit().message("0").create();
232 RevCommit commit1 = commit().message("1").parent(commit0).create();
233 git.update("master", commit0);
234
235 gcWithTtl();
236
237
238 assertEquals(2, odb.getPacks().length);
239 boolean gcPackFound = false;
240 boolean garbagePackFound = false;
241 for (DfsPackFile pack : odb.getPacks()) {
242 DfsPackDescription d = pack.getPackDescription();
243 if (d.getPackSource() == GC) {
244 gcPackFound = true;
245 assertTrue("has commit0", isObjectInPack(commit0, pack));
246 assertFalse("no commit1", isObjectInPack(commit1, pack));
247 } else if (d.getPackSource() == UNREACHABLE_GARBAGE) {
248 garbagePackFound = true;
249 assertFalse("no commit0", isObjectInPack(commit0, pack));
250 assertTrue("has commit1", isObjectInPack(commit1, pack));
251 } else {
252 fail("unexpected " + d.getPackSource());
253 }
254 }
255 assertTrue("gc pack found", gcPackFound);
256 assertTrue("garbage pack found", garbagePackFound);
257
258 git.update("master", commit1);
259
260 gcWithTtl();
261
262
263 DfsPackFile[] packs = odb.getPacks();
264 assertEquals(1, packs.length);
265
266 assertEquals(GC, packs[0].getPackDescription().getPackSource());
267 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
268 assertTrue("has commit1", isObjectInPack(commit1, packs[0]));
269 }
270
271 @Test
272 public void testCollectionWithPureGarbageAndGarbagePacksPurged()
273 throws Exception {
274 RevCommit commit0 = commit().message("0").create();
275 RevCommit commit1 = commit().message("1").parent(commit0).create();
276
277 gcWithTtl();
278
279
280 DfsPackFile[] packs = odb.getPacks();
281 assertEquals(1, packs.length);
282
283 assertEquals(UNREACHABLE_GARBAGE, packs[0].getPackDescription().getPackSource());
284 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
285 assertTrue("has commit1", isObjectInPack(commit1, packs[0]));
286
287 gcWithTtl();
288
289
290 assertEquals(0, odb.getPacks().length);
291 }
292
293 @Test
294 public void testCollectionWithPureGarbageAndRereferencingGarbage()
295 throws Exception {
296 RevCommit commit0 = commit().message("0").create();
297 RevCommit commit1 = commit().message("1").parent(commit0).create();
298
299 gcWithTtl();
300
301
302 DfsPackFile[] packs = odb.getPacks();
303 assertEquals(1, packs.length);
304
305 DfsPackDescription pack = packs[0].getPackDescription();
306 assertEquals(UNREACHABLE_GARBAGE, pack.getPackSource());
307 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
308 assertTrue("has commit1", isObjectInPack(commit1, packs[0]));
309
310 git.update("master", commit0);
311
312 gcWithTtl();
313
314
315 packs = odb.getPacks();
316 assertEquals(1, packs.length);
317
318 pack = packs[0].getPackDescription();
319 assertEquals(GC, pack.getPackSource());
320 assertTrue("has commit0", isObjectInPack(commit0, packs[0]));
321 assertFalse("no commit1", isObjectInPack(commit1, packs[0]));
322 }
323
324 @Test
325 public void testCollectionWithGarbageCoalescence() throws Exception {
326 RevCommit commit0 = commit().message("0").create();
327 RevCommit commit1 = commit().message("1").parent(commit0).create();
328 git.update("master", commit0);
329
330 for (int i = 0; i < 3; i++) {
331 commit1 = commit().message("g" + i).parent(commit1).create();
332
333
334
335 gcNoTtl();
336 assertEquals(1, countPacks(UNREACHABLE_GARBAGE));
337 }
338 }
339
340 @Test
341 public void testCollectionWithGarbageNoCoalescence() throws Exception {
342 RevCommit commit0 = commit().message("0").create();
343 RevCommit commit1 = commit().message("1").parent(commit0).create();
344 git.update("master", commit0);
345
346 for (int i = 0; i < 3; i++) {
347 commit1 = commit().message("g" + i).parent(commit1).create();
348
349 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
350 gc.setCoalesceGarbageLimit(0);
351 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
352 run(gc);
353 assertEquals(1 + i, countPacks(UNREACHABLE_GARBAGE));
354 }
355 }
356
357 @Test
358 public void testCollectionWithGarbageCoalescenceWithShortTtl()
359 throws Exception {
360 RevCommit commit0 = commit().message("0").create();
361 RevCommit commit1 = commit().message("1").parent(commit0).create();
362 git.update("master", commit0);
363
364
365 for (int i = 0; i < 100; i++) {
366 mockSystemReader.tick(60);
367 commit1 = commit().message("g" + i).parent(commit1).create();
368
369 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
370 gc.setGarbageTtl(1, TimeUnit.HOURS);
371 run(gc);
372
373
374
375
376
377 int count = countPacks(UNREACHABLE_GARBAGE);
378 assertTrue("Garbage pack count should not exceed 4, but found "
379 + count, count <= 4);
380 }
381 }
382
383 @Test
384 public void testCollectionWithGarbageCoalescenceWithLongTtl()
385 throws Exception {
386 RevCommit commit0 = commit().message("0").create();
387 RevCommit commit1 = commit().message("1").parent(commit0).create();
388 git.update("master", commit0);
389
390
391 for (int i = 0; i < 100; i++) {
392 mockSystemReader.tick(3600);
393 commit1 = commit().message("g" + i).parent(commit1).create();
394
395 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
396 gc.setGarbageTtl(2, TimeUnit.DAYS);
397 run(gc);
398
399
400
401
402
403 int count = countPacks(UNREACHABLE_GARBAGE);
404 assertTrue("Garbage pack count should not exceed 3, but found "
405 + count, count <= 3);
406 }
407 }
408
409 @Test
410 public void testEstimateGcPackSizeInNewRepo() throws Exception {
411 RevCommit commit0 = commit().message("0").create();
412 RevCommit commit1 = commit().message("1").parent(commit0).create();
413 git.update("master", commit1);
414
415
416 long inputPacksSize = 32;
417 assertEquals(2, odb.getPacks().length);
418 for (DfsPackFile pack : odb.getPacks()) {
419 assertEquals(INSERT, pack.getPackDescription().getPackSource());
420 inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32;
421 }
422
423 gcNoTtl();
424
425
426 assertEquals(1, odb.getPacks().length);
427 DfsPackFile pack = odb.getPacks()[0];
428 assertEquals(GC, pack.getPackDescription().getPackSource());
429 assertEquals(inputPacksSize,
430 pack.getPackDescription().getEstimatedPackSize());
431 }
432
433 @Test
434 public void testEstimateGcPackSizeWithAnExistingGcPack() throws Exception {
435 RevCommit commit0 = commit().message("0").create();
436 RevCommit commit1 = commit().message("1").parent(commit0).create();
437 git.update("master", commit1);
438
439 gcNoTtl();
440
441 RevCommit commit2 = commit().message("2").parent(commit1).create();
442 git.update("master", commit2);
443
444
445 assertEquals(2, odb.getPacks().length);
446 boolean gcPackFound = false;
447 boolean insertPackFound = false;
448 long inputPacksSize = 32;
449 for (DfsPackFile pack : odb.getPacks()) {
450 DfsPackDescription d = pack.getPackDescription();
451 if (d.getPackSource() == GC) {
452 gcPackFound = true;
453 } else if (d.getPackSource() == INSERT) {
454 insertPackFound = true;
455 } else {
456 fail("unexpected " + d.getPackSource());
457 }
458 inputPacksSize += d.getFileSize(PACK) - 32;
459 }
460 assertTrue(gcPackFound);
461 assertTrue(insertPackFound);
462
463 gcNoTtl();
464
465
466 DfsPackFile pack = odb.getPacks()[0];
467 assertEquals(GC, pack.getPackDescription().getPackSource());
468 assertEquals(inputPacksSize,
469 pack.getPackDescription().getEstimatedPackSize());
470 }
471
472 @Test
473 public void testEstimateGcRestPackSizeInNewRepo() throws Exception {
474 RevCommit commit0 = commit().message("0").create();
475 RevCommit commit1 = commit().message("1").parent(commit0).create();
476 git.update("refs/notes/note1", commit1);
477
478
479 long inputPacksSize = 32;
480 assertEquals(2, odb.getPacks().length);
481 for (DfsPackFile pack : odb.getPacks()) {
482 assertEquals(INSERT, pack.getPackDescription().getPackSource());
483 inputPacksSize += pack.getPackDescription().getFileSize(PACK) - 32;
484 }
485
486 gcNoTtl();
487
488
489 assertEquals(1, odb.getPacks().length);
490 DfsPackFile pack = odb.getPacks()[0];
491 assertEquals(GC_REST, pack.getPackDescription().getPackSource());
492 assertEquals(inputPacksSize,
493 pack.getPackDescription().getEstimatedPackSize());
494 }
495
496 @Test
497 public void testEstimateGcRestPackSizeWithAnExistingGcPack()
498 throws Exception {
499 RevCommit commit0 = commit().message("0").create();
500 RevCommit commit1 = commit().message("1").parent(commit0).create();
501 git.update("refs/notes/note1", commit1);
502
503 gcNoTtl();
504
505 RevCommit commit2 = commit().message("2").parent(commit1).create();
506 git.update("refs/notes/note2", commit2);
507
508
509 assertEquals(2, odb.getPacks().length);
510 boolean gcRestPackFound = false;
511 boolean insertPackFound = false;
512 long inputPacksSize = 32;
513 for (DfsPackFile pack : odb.getPacks()) {
514 DfsPackDescription d = pack.getPackDescription();
515 if (d.getPackSource() == GC_REST) {
516 gcRestPackFound = true;
517 } else if (d.getPackSource() == INSERT) {
518 insertPackFound = true;
519 } else {
520 fail("unexpected " + d.getPackSource());
521 }
522 inputPacksSize += d.getFileSize(PACK) - 32;
523 }
524 assertTrue(gcRestPackFound);
525 assertTrue(insertPackFound);
526
527 gcNoTtl();
528
529
530 DfsPackFile pack = odb.getPacks()[0];
531 assertEquals(GC_REST, pack.getPackDescription().getPackSource());
532 assertEquals(inputPacksSize,
533 pack.getPackDescription().getEstimatedPackSize());
534 }
535
536 @Test
537 public void testEstimateGcPackSizesWithGcAndGcRestPacks() throws Exception {
538 RevCommit commit0 = commit().message("0").create();
539 git.update("head", commit0);
540 RevCommit commit1 = commit().message("1").parent(commit0).create();
541 git.update("refs/notes/note1", commit1);
542
543 gcNoTtl();
544
545 RevCommit commit2 = commit().message("2").parent(commit1).create();
546 git.update("refs/notes/note2", commit2);
547
548
549 assertEquals(3, odb.getPacks().length);
550 boolean gcPackFound = false;
551 boolean gcRestPackFound = false;
552 boolean insertPackFound = false;
553 long gcPackSize = 0;
554 long gcRestPackSize = 0;
555 long insertPackSize = 0;
556 for (DfsPackFile pack : odb.getPacks()) {
557 DfsPackDescription d = pack.getPackDescription();
558 if (d.getPackSource() == GC) {
559 gcPackFound = true;
560 gcPackSize = d.getFileSize(PACK);
561 } else if (d.getPackSource() == GC_REST) {
562 gcRestPackFound = true;
563 gcRestPackSize = d.getFileSize(PACK);
564 } else if (d.getPackSource() == INSERT) {
565 insertPackFound = true;
566 insertPackSize = d.getFileSize(PACK);
567 } else {
568 fail("unexpected " + d.getPackSource());
569 }
570 }
571 assertTrue(gcPackFound);
572 assertTrue(gcRestPackFound);
573 assertTrue(insertPackFound);
574
575 gcNoTtl();
576
577
578
579
580
581 assertEquals(2, odb.getPacks().length);
582 gcPackFound = false;
583 gcRestPackFound = false;
584 for (DfsPackFile pack : odb.getPacks()) {
585 DfsPackDescription d = pack.getPackDescription();
586 if (d.getPackSource() == GC) {
587 gcPackFound = true;
588 assertEquals(gcPackSize + insertPackSize - 32,
589 pack.getPackDescription().getEstimatedPackSize());
590 } else if (d.getPackSource() == GC_REST) {
591 gcRestPackFound = true;
592 assertEquals(gcRestPackSize + insertPackSize - 32,
593 pack.getPackDescription().getEstimatedPackSize());
594 } else {
595 fail("unexpected " + d.getPackSource());
596 }
597 }
598 assertTrue(gcPackFound);
599 assertTrue(gcRestPackFound);
600 }
601
602 @Test
603 public void testEstimateUnreachableGarbagePackSize() throws Exception {
604 RevCommit commit0 = commit().message("0").create();
605 RevCommit commit1 = commit().message("1").parent(commit0).create();
606 git.update("master", commit0);
607
608 assertTrue("commit0 reachable", isReachable(repo, commit0));
609 assertFalse("commit1 garbage", isReachable(repo, commit1));
610
611
612 long packSize0 = 0;
613 long packSize1 = 0;
614 assertEquals(2, odb.getPacks().length);
615 for (DfsPackFile pack : odb.getPacks()) {
616 DfsPackDescription d = pack.getPackDescription();
617 assertEquals(INSERT, d.getPackSource());
618 if (isObjectInPack(commit0, pack)) {
619 packSize0 = d.getFileSize(PACK);
620 } else if (isObjectInPack(commit1, pack)) {
621 packSize1 = d.getFileSize(PACK);
622 } else {
623 fail("expected object not found in the pack");
624 }
625 }
626
627 gcNoTtl();
628
629 assertEquals(2, odb.getPacks().length);
630 for (DfsPackFile pack : odb.getPacks()) {
631 DfsPackDescription d = pack.getPackDescription();
632 if (d.getPackSource() == GC) {
633
634
635
636
637 assertEquals(packSize0 + packSize1 - 32,
638 d.getEstimatedPackSize());
639 } else if (d.getPackSource() == UNREACHABLE_GARBAGE) {
640
641 assertEquals(packSize1, d.getEstimatedPackSize());
642 } else {
643 fail("unexpected " + d.getPackSource());
644 }
645 }
646 }
647
648 @Test
649 public void testSinglePackForAllRefs() throws Exception {
650 RevCommit commit0 = commit().message("0").create();
651 git.update("head", commit0);
652 RevCommit commit1 = commit().message("1").parent(commit0).create();
653 git.update("refs/notes/note1", commit1);
654
655 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
656 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
657 gc.getPackConfig().setSinglePack(true);
658 run(gc);
659 assertEquals(1, odb.getPacks().length);
660
661 gc = new DfsGarbageCollector(repo);
662 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
663 gc.getPackConfig().setSinglePack(false);
664 run(gc);
665 assertEquals(2, odb.getPacks().length);
666 }
667
668 @SuppressWarnings("boxing")
669 @Test
670 public void producesNewReftable() throws Exception {
671 String master = "refs/heads/master";
672 RevCommit commit0 = commit().message("0").create();
673 RevCommit commit1 = commit().message("1").parent(commit0).create();
674
675 BatchRefUpdate bru = git.getRepository().getRefDatabase()
676 .newBatchUpdate();
677 bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit1, master));
678 for (int i = 1; i <= 5100; i++) {
679 bru.addCommand(new ReceiveCommand(ObjectId.zeroId(), commit0,
680 String.format("refs/pulls/%04d", i)));
681 }
682 try (RevWalk rw = new RevWalk(git.getRepository())) {
683 bru.execute(rw, NullProgressMonitor.INSTANCE);
684 }
685
686 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
687 gc.setReftableConfig(new ReftableConfig());
688 run(gc);
689
690
691 assertEquals(1, odb.getPacks().length);
692 DfsPackFile pack = odb.getPacks()[0];
693 DfsPackDescription desc = pack.getPackDescription();
694 assertEquals(GC, desc.getPackSource());
695 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
696 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
697
698
699 assertTrue(desc.hasFileExt(REFTABLE));
700 ReftableWriter.Stats stats = desc.getReftableStats();
701 assertNotNull(stats);
702 assertTrue(stats.totalBytes() > 0);
703 assertEquals(5101, stats.refCount());
704 assertEquals(1, stats.minUpdateIndex());
705 assertEquals(1, stats.maxUpdateIndex());
706
707 DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
708 try (DfsReader ctx = odb.newReader();
709 ReftableReader rr = table.open(ctx);
710 RefCursor rc = rr.seekRef("refs/pulls/5100")) {
711 assertTrue(rc.next());
712 assertEquals(commit0, rc.getRef().getObjectId());
713 assertFalse(rc.next());
714 }
715 }
716
717 @Test
718 public void leavesNonGcReftablesIfNotConfigured() throws Exception {
719 String master = "refs/heads/master";
720 RevCommit commit0 = commit().message("0").create();
721 RevCommit commit1 = commit().message("1").parent(commit0).create();
722 git.update(master, commit1);
723
724 DfsPackDescription t1 = odb.newPack(INSERT);
725 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
726 new ReftableWriter().begin(out).finish();
727 t1.addFileExt(REFTABLE);
728 }
729 odb.commitPack(Collections.singleton(t1), null);
730
731 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
732 gc.setReftableConfig(null);
733 run(gc);
734
735
736 assertEquals(1, odb.getPacks().length);
737 DfsPackFile pack = odb.getPacks()[0];
738 DfsPackDescription desc = pack.getPackDescription();
739 assertEquals(GC, desc.getPackSource());
740 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
741 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
742
743
744 DfsReftable[] tables = odb.getReftables();
745 assertEquals(2, tables.length);
746 assertEquals(t1, tables[0].getPackDescription());
747 }
748
749 @Test
750 public void prunesNonGcReftables() throws Exception {
751 String master = "refs/heads/master";
752 RevCommit commit0 = commit().message("0").create();
753 RevCommit commit1 = commit().message("1").parent(commit0).create();
754 git.update(master, commit1);
755
756 DfsPackDescription t1 = odb.newPack(INSERT);
757 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
758 new ReftableWriter().begin(out).finish();
759 t1.addFileExt(REFTABLE);
760 }
761 odb.commitPack(Collections.singleton(t1), null);
762 odb.clearCache();
763
764 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
765 gc.setReftableConfig(new ReftableConfig());
766 run(gc);
767
768
769 assertEquals(1, odb.getPacks().length);
770 DfsPackFile pack = odb.getPacks()[0];
771 DfsPackDescription desc = pack.getPackDescription();
772 assertEquals(GC, desc.getPackSource());
773 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
774 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
775
776
777 DfsReftable[] tables = odb.getReftables();
778 assertEquals(1, tables.length);
779 assertEquals(desc, tables[0].getPackDescription());
780 assertTrue(desc.hasFileExt(REFTABLE));
781 }
782
783 @Test
784 public void compactsReftables() throws Exception {
785 String master = "refs/heads/master";
786 RevCommit commit0 = commit().message("0").create();
787 RevCommit commit1 = commit().message("1").parent(commit0).create();
788 git.update(master, commit1);
789
790 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
791 gc.setReftableConfig(new ReftableConfig());
792 run(gc);
793
794 DfsPackDescription t1 = odb.newPack(INSERT);
795 Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE,
796 "refs/heads/next", commit0.copy());
797 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
798 ReftableWriter w = new ReftableWriter();
799 w.setMinUpdateIndex(42);
800 w.setMaxUpdateIndex(42);
801 w.begin(out);
802 w.sortAndWriteRefs(Collections.singleton(next));
803 w.finish();
804 t1.addFileExt(REFTABLE);
805 t1.setReftableStats(w.getStats());
806 }
807 odb.commitPack(Collections.singleton(t1), null);
808
809 gc = new DfsGarbageCollector(repo);
810 gc.setReftableConfig(new ReftableConfig());
811 run(gc);
812
813
814 assertEquals(1, odb.getPacks().length);
815 DfsPackFile pack = odb.getPacks()[0];
816 DfsPackDescription desc = pack.getPackDescription();
817 assertEquals(GC, desc.getPackSource());
818 assertTrue("commit0 in pack", isObjectInPack(commit0, pack));
819 assertTrue("commit1 in pack", isObjectInPack(commit1, pack));
820
821
822 DfsReftable[] tables = odb.getReftables();
823 assertEquals(1, tables.length);
824 assertEquals(desc, tables[0].getPackDescription());
825 assertTrue(desc.hasFileExt(REFTABLE));
826
827
828 DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc);
829 try (DfsReader ctx = odb.newReader();
830 ReftableReader rr = table.open(ctx);
831 RefCursor rc = rr.allRefs()) {
832 assertEquals(1, rr.minUpdateIndex());
833 assertEquals(42, rr.maxUpdateIndex());
834
835 assertTrue(rc.next());
836 assertEquals(master, rc.getRef().getName());
837 assertEquals(commit1, rc.getRef().getObjectId());
838
839 assertTrue(rc.next());
840 assertEquals(next.getName(), rc.getRef().getName());
841 assertEquals(commit0, rc.getRef().getObjectId());
842
843 assertFalse(rc.next());
844 }
845 }
846
847 @Test
848 public void reftableWithoutTombstoneResurrected() throws Exception {
849 RevCommit commit0 = commit().message("0").create();
850 String NEXT = "refs/heads/next";
851 DfsRefDatabase refdb = (DfsRefDatabase)repo.getRefDatabase();
852 git.update(NEXT, commit0);
853 Ref next = refdb.exactRef(NEXT);
854 assertNotNull(next);
855 assertEquals(commit0, next.getObjectId());
856
857 git.delete(NEXT);
858 refdb.clearCache();
859 assertNull(refdb.exactRef(NEXT));
860
861 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
862 gc.setReftableConfig(new ReftableConfig());
863 gc.setIncludeDeletes(false);
864 gc.setConvertToReftable(false);
865 run(gc);
866 assertEquals(1, odb.getReftables().length);
867 try (DfsReader ctx = odb.newReader();
868 ReftableReader rr = odb.getReftables()[0].open(ctx)) {
869 rr.setIncludeDeletes(true);
870 assertEquals(1, rr.minUpdateIndex());
871 assertEquals(2, rr.maxUpdateIndex());
872 assertNull(rr.exactRef(NEXT));
873 }
874
875 RevCommit commit1 = commit().message("1").create();
876 DfsPackDescription t1 = odb.newPack(INSERT);
877 Ref newNext = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, NEXT,
878 commit1);
879 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
880 ReftableWriter w = new ReftableWriter();
881 w.setMinUpdateIndex(1);
882 w.setMaxUpdateIndex(1);
883 w.begin(out);
884 w.writeRef(newNext, 1);
885 w.finish();
886 t1.addFileExt(REFTABLE);
887 t1.setReftableStats(w.getStats());
888 }
889 odb.commitPack(Collections.singleton(t1), null);
890 assertEquals(2, odb.getReftables().length);
891 refdb.clearCache();
892 newNext = refdb.exactRef(NEXT);
893 assertNotNull(newNext);
894 assertEquals(commit1, newNext.getObjectId());
895 }
896
897 @Test
898 public void reftableWithTombstoneNotResurrected() throws Exception {
899 RevCommit commit0 = commit().message("0").create();
900 String NEXT = "refs/heads/next";
901 DfsRefDatabase refdb = (DfsRefDatabase)repo.getRefDatabase();
902 git.update(NEXT, commit0);
903 Ref next = refdb.exactRef(NEXT);
904 assertNotNull(next);
905 assertEquals(commit0, next.getObjectId());
906
907 git.delete(NEXT);
908 refdb.clearCache();
909 assertNull(refdb.exactRef(NEXT));
910
911 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
912 gc.setReftableConfig(new ReftableConfig());
913 gc.setIncludeDeletes(true);
914 gc.setConvertToReftable(false);
915 run(gc);
916 assertEquals(1, odb.getReftables().length);
917 try (DfsReader ctx = odb.newReader();
918 ReftableReader rr = odb.getReftables()[0].open(ctx)) {
919 rr.setIncludeDeletes(true);
920 assertEquals(1, rr.minUpdateIndex());
921 assertEquals(2, rr.maxUpdateIndex());
922 next = rr.exactRef(NEXT);
923 assertNotNull(next);
924 assertNull(next.getObjectId());
925 }
926
927 RevCommit commit1 = commit().message("1").create();
928 DfsPackDescription t1 = odb.newPack(INSERT);
929 Ref newNext = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, NEXT,
930 commit1);
931 try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) {
932 ReftableWriter w = new ReftableWriter();
933 w.setMinUpdateIndex(1);
934 w.setMaxUpdateIndex(1);
935 w.begin(out);
936 w.writeRef(newNext, 1);
937 w.finish();
938 t1.addFileExt(REFTABLE);
939 t1.setReftableStats(w.getStats());
940 }
941 odb.commitPack(Collections.singleton(t1), null);
942 assertEquals(2, odb.getReftables().length);
943 refdb.clearCache();
944 assertNull(refdb.exactRef(NEXT));
945 }
946
947 private TestRepository<InMemoryRepository>.CommitBuilder commit() {
948 return git.commit();
949 }
950
951 private void gcNoTtl() throws IOException {
952 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
953 gc.setGarbageTtl(0, TimeUnit.MILLISECONDS);
954 run(gc);
955 }
956
957 private void gcWithTtl() throws IOException {
958
959 mockSystemReader.tick(60);
960 DfsGarbageCollector gc = new DfsGarbageCollector(repo);
961 gc.setGarbageTtl(1, TimeUnit.MINUTES);
962 run(gc);
963 }
964
965 private void run(DfsGarbageCollector gc) throws IOException {
966
967 mockSystemReader.tick(1);
968 assertTrue("gc repacked", gc.pack(null));
969 odb.clearCache();
970 }
971
972 private static boolean isReachable(Repository repo, AnyObjectId id)
973 throws IOException {
974 try (RevWalk rw = new RevWalk(repo)) {
975 for (Ref ref : repo.getRefDatabase().getRefs()) {
976 rw.markStart(rw.parseCommit(ref.getObjectId()));
977 }
978 for (RevCommit next; (next = rw.next()) != null;) {
979 if (AnyObjectId.equals(next, id)) {
980 return true;
981 }
982 }
983 }
984 return false;
985 }
986
987 private boolean isObjectInPack(AnyObjectId id, DfsPackFile pack)
988 throws IOException {
989 try (DfsReader reader = odb.newReader()) {
990 return pack.hasObject(reader, id);
991 }
992 }
993
994 private int countPacks(PackSource source) throws IOException {
995 int cnt = 0;
996 for (DfsPackFile pack : odb.getPacks()) {
997 if (pack.getPackDescription().getPackSource() == source) {
998 cnt++;
999 }
1000 }
1001 return cnt;
1002 }
1003 }