1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.internal.storage.file;
45
46 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
47 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
48
49 import java.io.File;
50 import java.io.FileOutputStream;
51 import java.io.IOException;
52 import java.io.OutputStream;
53 import java.io.PrintWriter;
54 import java.io.StringWriter;
55 import java.nio.channels.Channels;
56 import java.nio.channels.FileChannel;
57 import java.nio.file.DirectoryNotEmptyException;
58 import java.nio.file.DirectoryStream;
59 import java.nio.file.Files;
60 import java.nio.file.Path;
61 import java.nio.file.Paths;
62 import java.nio.file.StandardCopyOption;
63 import java.text.MessageFormat;
64 import java.text.ParseException;
65 import java.time.Instant;
66 import java.time.temporal.ChronoUnit;
67 import java.util.ArrayList;
68 import java.util.Collection;
69 import java.util.Collections;
70 import java.util.Comparator;
71 import java.util.Date;
72 import java.util.HashMap;
73 import java.util.HashSet;
74 import java.util.Iterator;
75 import java.util.LinkedList;
76 import java.util.List;
77 import java.util.Map;
78 import java.util.Objects;
79 import java.util.Set;
80 import java.util.TreeMap;
81 import java.util.concurrent.Callable;
82 import java.util.concurrent.ExecutorService;
83 import java.util.regex.Pattern;
84 import java.util.stream.Collectors;
85 import java.util.stream.Stream;
86
87 import org.eclipse.jgit.annotations.NonNull;
88 import org.eclipse.jgit.dircache.DirCacheIterator;
89 import org.eclipse.jgit.errors.CancelledException;
90 import org.eclipse.jgit.errors.CorruptObjectException;
91 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
92 import org.eclipse.jgit.errors.MissingObjectException;
93 import org.eclipse.jgit.errors.NoWorkTreeException;
94 import org.eclipse.jgit.internal.JGitText;
95 import org.eclipse.jgit.internal.storage.pack.PackExt;
96 import org.eclipse.jgit.internal.storage.pack.PackWriter;
97 import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
98 import org.eclipse.jgit.lib.ConfigConstants;
99 import org.eclipse.jgit.lib.Constants;
100 import org.eclipse.jgit.lib.FileMode;
101 import org.eclipse.jgit.lib.NullProgressMonitor;
102 import org.eclipse.jgit.lib.ObjectId;
103 import org.eclipse.jgit.lib.ObjectIdSet;
104 import org.eclipse.jgit.lib.ObjectLoader;
105 import org.eclipse.jgit.lib.ObjectReader;
106 import org.eclipse.jgit.lib.ProgressMonitor;
107 import org.eclipse.jgit.lib.Ref;
108 import org.eclipse.jgit.lib.Ref.Storage;
109 import org.eclipse.jgit.lib.RefDatabase;
110 import org.eclipse.jgit.lib.ReflogEntry;
111 import org.eclipse.jgit.lib.ReflogReader;
112 import org.eclipse.jgit.lib.internal.WorkQueue;
113 import org.eclipse.jgit.revwalk.ObjectWalk;
114 import org.eclipse.jgit.revwalk.RevObject;
115 import org.eclipse.jgit.revwalk.RevWalk;
116 import org.eclipse.jgit.storage.pack.PackConfig;
117 import org.eclipse.jgit.treewalk.TreeWalk;
118 import org.eclipse.jgit.treewalk.filter.TreeFilter;
119 import org.eclipse.jgit.util.FileUtils;
120 import org.eclipse.jgit.util.GitDateParser;
121 import org.eclipse.jgit.util.SystemReader;
122 import org.slf4j.Logger;
123 import org.slf4j.LoggerFactory;
124
125
126
127
128
129
130
131
132 public class GC {
133 private final static Logger LOG = LoggerFactory
134 .getLogger(GC.class);
135
136 private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago";
137
138 private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago";
139
140 private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
141 .compile("[0-9a-fA-F]{38}");
142
143 private static final String PACK_EXT = "." + PackExt.PACK.getExtension();
144
145 private static final String BITMAP_EXT = "."
146 + PackExt.BITMAP_INDEX.getExtension();
147
148 private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension();
149
150 private static final int DEFAULT_AUTOPACKLIMIT = 50;
151
152 private static final int DEFAULT_AUTOLIMIT = 6700;
153
154 private static volatile ExecutorService executor;
155
156
157
158
159
160
161
162
163
164 public static void setExecutor(ExecutorService e) {
165 executor = e;
166 }
167
168 private final FileRepository repo;
169
170 private ProgressMonitor pm;
171
172 private long expireAgeMillis = -1;
173
174 private Date expire;
175
176 private long packExpireAgeMillis = -1;
177
178 private Date packExpire;
179
180 private PackConfig pconfig = null;
181
182
183
184
185
186
187
188 private Collection<Ref> lastPackedRefs;
189
190
191
192
193
194
195 private long lastRepackTime;
196
197
198
199
200 private boolean automatic;
201
202
203
204
205 private boolean background;
206
207
208
209
210
211
212
213
214 public GC(FileRepository repo) {
215 this.repo = repo;
216 this.pm = NullProgressMonitor.INSTANCE;
217 }
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244 public Collection<PackFile> gc() throws IOException, ParseException {
245 if (!background) {
246 return doGc();
247 }
248 final GcLog gcLog = new GcLog(repo);
249 if (!gcLog.lock()) {
250
251 return Collections.emptyList();
252 }
253
254 Callable<Collection<PackFile>> gcTask = () -> {
255 try {
256 Collection<PackFile> newPacks = doGc();
257 if (automatic && tooManyLooseObjects()) {
258 String message = JGitText.get().gcTooManyUnpruned;
259 gcLog.write(message);
260 gcLog.commit();
261 }
262 return newPacks;
263 } catch (IOException | ParseException e) {
264 try {
265 gcLog.write(e.getMessage());
266 StringWriter sw = new StringWriter();
267 e.printStackTrace(new PrintWriter(sw));
268 gcLog.write(sw.toString());
269 gcLog.commit();
270 } catch (IOException e2) {
271 e2.addSuppressed(e);
272 LOG.error(e2.getMessage(), e2);
273 }
274 } finally {
275 gcLog.unlock();
276 }
277 return Collections.emptyList();
278 };
279
280 executor().submit(gcTask);
281 return Collections.emptyList();
282 }
283
284 private ExecutorService executor() {
285 return (executor != null) ? executor : WorkQueue.getExecutor();
286 }
287
288 private Collection<PackFile> doGc() throws IOException, ParseException {
289 if (automatic && !needGc()) {
290 return Collections.emptyList();
291 }
292 pm.start(6 );
293 packRefs();
294
295 Collection<PackFile> newPacks = repack();
296 prune(Collections.<ObjectId> emptySet());
297
298 return newPacks;
299 }
300
301
302
303
304
305
306
307
308
309
310
311 private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
312 throws IOException {
313 for (PackIndex.MutableEntry entry : pack) {
314 ObjectId oid = entry.toObjectId();
315 if (existing.contains(oid)) {
316 continue;
317 }
318 existing.add(oid);
319 ObjectLoader loader = reader.open(oid);
320 inserter.insert(loader.getType(),
321 loader.getSize(),
322 loader.openStream(),
323 true );
324 }
325 }
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343 private void deleteOldPacks(Collection<PackFile> oldPacks,
344 Collection<PackFile> newPacks) throws ParseException, IOException {
345 HashSet<ObjectId> ids = new HashSet<>();
346 for (PackFile pack : newPacks) {
347 for (PackIndex.MutableEntry entry : pack) {
348 ids.add(entry.toObjectId());
349 }
350 }
351 ObjectReader reader = repo.newObjectReader();
352 ObjectDirectory dir = repo.getObjectDatabase();
353 ObjectDirectoryInserter inserter = dir.newInserter();
354 boolean shouldLoosen = !"now".equals(getPruneExpireStr()) &&
355 getExpireDate() < Long.MAX_VALUE;
356
357 prunePreserved();
358 long packExpireDate = getPackExpireDate();
359 oldPackLoop: for (PackFile oldPack : oldPacks) {
360 checkCancelled();
361 String oldName = oldPack.getPackName();
362
363
364 for (PackFile newPack : newPacks)
365 if (oldName.equals(newPack.getPackName()))
366 continue oldPackLoop;
367
368 if (!oldPack.shouldBeKept()
369 && repo.getFS().lastModified(
370 oldPack.getPackFile()) < packExpireDate) {
371 oldPack.close();
372 if (shouldLoosen) {
373 loosen(inserter, reader, oldPack, ids);
374 }
375 prunePack(oldName);
376 }
377 }
378
379
380
381 repo.getObjectDatabase().close();
382 }
383
384
385
386
387
388
389
390
391
392
393
394 private void removeOldPack(File packFile, String packName, PackExt ext,
395 int deleteOptions) throws IOException {
396 if (pconfig != null && pconfig.isPreserveOldPacks()) {
397 File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
398 FileUtils.mkdir(oldPackDir, true);
399
400 String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();
401 File oldPackFile = new File(oldPackDir, oldPackName);
402 FileUtils.rename(packFile, oldPackFile);
403 } else {
404 FileUtils.delete(packFile, deleteOptions);
405 }
406 }
407
408
409
410
411 private void prunePreserved() {
412 if (pconfig != null && pconfig.isPrunePreserved()) {
413 try {
414 FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
415 FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
416 } catch (IOException e) {
417
418 }
419 }
420 }
421
422
423
424
425
426
427
428
429
430
431
432 private void prunePack(String packName) {
433 PackExt[] extensions = PackExt.values();
434 try {
435
436
437 int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
438 for (PackExt ext : extensions)
439 if (PackExt.PACK.equals(ext)) {
440 File f = nameFor(packName, "." + ext.getExtension());
441 removeOldPack(f, packName, ext, deleteOptions);
442 break;
443 }
444
445
446 deleteOptions |= FileUtils.IGNORE_ERRORS;
447 for (PackExt ext : extensions) {
448 if (!PackExt.PACK.equals(ext)) {
449 File f = nameFor(packName, "." + ext.getExtension());
450 removeOldPack(f, packName, ext, deleteOptions);
451 }
452 }
453 } catch (IOException e) {
454
455 }
456 }
457
458
459
460
461
462
463
464
465 public void prunePacked() throws IOException {
466 ObjectDirectory objdb = repo.getObjectDatabase();
467 Collection<PackFile> packs = objdb.getPacks();
468 File objects = repo.getObjectsDirectory();
469 String[] fanout = objects.list();
470
471 if (fanout != null && fanout.length > 0) {
472 pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
473 try {
474 for (String d : fanout) {
475 checkCancelled();
476 pm.update(1);
477 if (d.length() != 2)
478 continue;
479 String[] entries = new File(objects, d).list();
480 if (entries == null)
481 continue;
482 for (String e : entries) {
483 checkCancelled();
484 if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
485 continue;
486 ObjectId id;
487 try {
488 id = ObjectId.fromString(d + e);
489 } catch (IllegalArgumentException notAnObject) {
490
491
492 continue;
493 }
494 boolean found = false;
495 for (PackFile p : packs) {
496 checkCancelled();
497 if (p.hasObject(id)) {
498 found = true;
499 break;
500 }
501 }
502 if (found)
503 FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
504 | FileUtils.SKIP_MISSING
505 | FileUtils.IGNORE_ERRORS);
506 }
507 }
508 } finally {
509 pm.endTask();
510 }
511 }
512 }
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527 public void prune(Set<ObjectId> objectsToKeep) throws IOException,
528 ParseException {
529 long expireDate = getExpireDate();
530
531
532
533 Map<ObjectId, File> deletionCandidates = new HashMap<>();
534 Set<ObjectId> indexObjects = null;
535 File objects = repo.getObjectsDirectory();
536 String[] fanout = objects.list();
537 if (fanout == null || fanout.length == 0) {
538 return;
539 }
540 pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
541 fanout.length);
542 try {
543 for (String d : fanout) {
544 checkCancelled();
545 pm.update(1);
546 if (d.length() != 2)
547 continue;
548 File[] entries = new File(objects, d).listFiles();
549 if (entries == null)
550 continue;
551 for (File f : entries) {
552 checkCancelled();
553 String fName = f.getName();
554 if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
555 continue;
556 if (repo.getFS().lastModified(f) >= expireDate)
557 continue;
558 try {
559 ObjectId id = ObjectId.fromString(d + fName);
560 if (objectsToKeep.contains(id))
561 continue;
562 if (indexObjects == null)
563 indexObjects = listNonHEADIndexObjects();
564 if (indexObjects.contains(id))
565 continue;
566 deletionCandidates.put(id, f);
567 } catch (IllegalArgumentException notAnObject) {
568
569
570 continue;
571 }
572 }
573 }
574 } finally {
575 pm.endTask();
576 }
577
578 if (deletionCandidates.isEmpty()) {
579 return;
580 }
581
582 checkCancelled();
583
584
585
586
587
588 Collection<Ref> newRefs;
589 if (lastPackedRefs == null || lastPackedRefs.isEmpty())
590 newRefs = getAllRefs();
591 else {
592 Map<String, Ref> last = new HashMap<>();
593 for (Ref r : lastPackedRefs) {
594 last.put(r.getName(), r);
595 }
596 newRefs = new ArrayList<>();
597 for (Ref r : getAllRefs()) {
598 Ref old = last.get(r.getName());
599 if (!equals(r, old)) {
600 newRefs.add(r);
601 }
602 }
603 }
604
605 if (!newRefs.isEmpty()) {
606
607
608
609
610
611 ObjectWalk w = new ObjectWalk(repo);
612 try {
613 for (Ref cr : newRefs) {
614 checkCancelled();
615 w.markStart(w.parseAny(cr.getObjectId()));
616 }
617 if (lastPackedRefs != null)
618 for (Ref lpr : lastPackedRefs) {
619 w.markUninteresting(w.parseAny(lpr.getObjectId()));
620 }
621 removeReferenced(deletionCandidates, w);
622 } finally {
623 w.dispose();
624 }
625 }
626
627 if (deletionCandidates.isEmpty())
628 return;
629
630
631
632
633
634
635 ObjectWalk w = new ObjectWalk(repo);
636 try {
637 for (Ref ar : getAllRefs())
638 for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
639 checkCancelled();
640 w.markStart(w.parseAny(id));
641 }
642 if (lastPackedRefs != null)
643 for (Ref lpr : lastPackedRefs) {
644 checkCancelled();
645 w.markUninteresting(w.parseAny(lpr.getObjectId()));
646 }
647 removeReferenced(deletionCandidates, w);
648 } finally {
649 w.dispose();
650 }
651
652 if (deletionCandidates.isEmpty())
653 return;
654
655 checkCancelled();
656
657
658
659
660
661 Set<File> touchedFanout = new HashSet<>();
662 for (File f : deletionCandidates.values()) {
663 if (f.lastModified() < expireDate) {
664 f.delete();
665 touchedFanout.add(f.getParentFile());
666 }
667 }
668
669 for (File f : touchedFanout) {
670 FileUtils.delete(f,
671 FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
672 }
673
674 repo.getObjectDatabase().close();
675 }
676
677 private long getExpireDate() throws ParseException {
678 long expireDate = Long.MAX_VALUE;
679
680 if (expire == null && expireAgeMillis == -1) {
681 String pruneExpireStr = getPruneExpireStr();
682 if (pruneExpireStr == null)
683 pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
684 expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
685 .getInstance().getLocale());
686 expireAgeMillis = -1;
687 }
688 if (expire != null)
689 expireDate = expire.getTime();
690 if (expireAgeMillis != -1)
691 expireDate = System.currentTimeMillis() - expireAgeMillis;
692 return expireDate;
693 }
694
695 private String getPruneExpireStr() {
696 return repo.getConfig().getString(
697 ConfigConstants.CONFIG_GC_SECTION, null,
698 ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
699 }
700
701 private long getPackExpireDate() throws ParseException {
702 long packExpireDate = Long.MAX_VALUE;
703
704 if (packExpire == null && packExpireAgeMillis == -1) {
705 String prunePackExpireStr = repo.getConfig().getString(
706 ConfigConstants.CONFIG_GC_SECTION, null,
707 ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
708 if (prunePackExpireStr == null)
709 prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
710 packExpire = GitDateParser.parse(prunePackExpireStr, null,
711 SystemReader.getInstance().getLocale());
712 packExpireAgeMillis = -1;
713 }
714 if (packExpire != null)
715 packExpireDate = packExpire.getTime();
716 if (packExpireAgeMillis != -1)
717 packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
718 return packExpireDate;
719 }
720
721
722
723
724
725
726
727
728
729
730
731 private void removeReferenced(Map<ObjectId, File> id2File,
732 ObjectWalk w) throws MissingObjectException,
733 IncorrectObjectTypeException, IOException {
734 RevObject ro = w.next();
735 while (ro != null) {
736 checkCancelled();
737 if (id2File.remove(ro.getId()) != null)
738 if (id2File.isEmpty())
739 return;
740 ro = w.next();
741 }
742 ro = w.nextObject();
743 while (ro != null) {
744 checkCancelled();
745 if (id2File.remove(ro.getId()) != null)
746 if (id2File.isEmpty())
747 return;
748 ro = w.nextObject();
749 }
750 }
751
752 private static boolean equals(Ref r1, Ref r2) {
753 if (r1 == null || r2 == null)
754 return false;
755 if (r1.isSymbolic()) {
756 if (!r2.isSymbolic())
757 return false;
758 return r1.getTarget().getName().equals(r2.getTarget().getName());
759 } else {
760 if (r2.isSymbolic()) {
761 return false;
762 }
763 return Objects.equals(r1.getObjectId(), r2.getObjectId());
764 }
765 }
766
767
768
769
770
771
772 public void packRefs() throws IOException {
773 Collection<Ref> refs = repo.getRefDatabase().getRefs(Constants.R_REFS).values();
774 List<String> refsToBePacked = new ArrayList<>(refs.size());
775 pm.beginTask(JGitText.get().packRefs, refs.size());
776 try {
777 for (Ref ref : refs) {
778 checkCancelled();
779 if (!ref.isSymbolic() && ref.getStorage().isLoose())
780 refsToBePacked.add(ref.getName());
781 pm.update(1);
782 }
783 ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
784 } finally {
785 pm.endTask();
786 }
787 }
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803 public Collection<PackFile> repack() throws IOException {
804 Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
805
806 long time = System.currentTimeMillis();
807 Collection<Ref> refsBefore = getAllRefs();
808
809 Set<ObjectId> allHeadsAndTags = new HashSet<>();
810 Set<ObjectId> allHeads = new HashSet<>();
811 Set<ObjectId> allTags = new HashSet<>();
812 Set<ObjectId> nonHeads = new HashSet<>();
813 Set<ObjectId> txnHeads = new HashSet<>();
814 Set<ObjectId> tagTargets = new HashSet<>();
815 Set<ObjectId> indexObjects = listNonHEADIndexObjects();
816 RefDatabase refdb = repo.getRefDatabase();
817
818 for (Ref ref : refsBefore) {
819 checkCancelled();
820 nonHeads.addAll(listRefLogObjects(ref, 0));
821 if (ref.isSymbolic() || ref.getObjectId() == null) {
822 continue;
823 }
824 if (isHead(ref)) {
825 allHeads.add(ref.getObjectId());
826 } else if (isTag(ref)) {
827 allTags.add(ref.getObjectId());
828 } else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
829 txnHeads.add(ref.getObjectId());
830 } else {
831 nonHeads.add(ref.getObjectId());
832 }
833 if (ref.getPeeledObjectId() != null) {
834 tagTargets.add(ref.getPeeledObjectId());
835 }
836 }
837
838 List<ObjectIdSet> excluded = new LinkedList<>();
839 for (final PackFile f : repo.getObjectDatabase().getPacks()) {
840 checkCancelled();
841 if (f.shouldBeKept())
842 excluded.add(f.getIndex());
843 }
844
845
846 allTags.removeAll(allHeads);
847 allHeadsAndTags.addAll(allHeads);
848 allHeadsAndTags.addAll(allTags);
849
850
851 tagTargets.addAll(allHeadsAndTags);
852 nonHeads.addAll(indexObjects);
853
854
855 if (pconfig != null && pconfig.getSinglePack()) {
856 allHeadsAndTags.addAll(nonHeads);
857 nonHeads.clear();
858 }
859
860 List<PackFile> ret = new ArrayList<>(2);
861 PackFile heads = null;
862 if (!allHeadsAndTags.isEmpty()) {
863 heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
864 tagTargets, excluded);
865 if (heads != null) {
866 ret.add(heads);
867 excluded.add(0, heads.getIndex());
868 }
869 }
870 if (!nonHeads.isEmpty()) {
871 PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
872 tagTargets, excluded);
873 if (rest != null)
874 ret.add(rest);
875 }
876 if (!txnHeads.isEmpty()) {
877 PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
878 null, excluded);
879 if (txn != null)
880 ret.add(txn);
881 }
882 try {
883 deleteOldPacks(toBeDeleted, ret);
884 } catch (ParseException e) {
885
886
887
888 throw new IOException(e);
889 }
890 prunePacked();
891 deleteEmptyRefsFolders();
892 deleteOrphans();
893 deleteTempPacksIdx();
894
895 lastPackedRefs = refsBefore;
896 lastRepackTime = time;
897 return ret;
898 }
899
900 private static boolean isHead(Ref ref) {
901 return ref.getName().startsWith(Constants.R_HEADS);
902 }
903
904 private static boolean isTag(Ref ref) {
905 return ref.getName().startsWith(Constants.R_TAGS);
906 }
907
908 private void deleteEmptyRefsFolders() throws IOException {
909 Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
910
911
912 Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
913 try (Stream<Path> entries = Files.list(refs)) {
914 Iterator<Path> iterator = entries.iterator();
915 while (iterator.hasNext()) {
916 try (Stream<Path> s = Files.list(iterator.next())) {
917 s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
918 }
919 }
920 }
921 }
922
923 private boolean canBeSafelyDeleted(Path path, Instant threshold) {
924 try {
925 return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
926 }
927 catch (IOException e) {
928 LOG.warn(MessageFormat.format(
929 JGitText.get().cannotAccessLastModifiedForSafeDeletion,
930 path), e);
931 return false;
932 }
933 }
934
935 private void deleteDir(Path dir) {
936 try (Stream<Path> dirs = Files.walk(dir)) {
937 dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
938 .forEach(this::delete);
939 } catch (IOException e) {
940 LOG.error(e.getMessage(), e);
941 }
942 }
943
944 private boolean isDirectory(Path p) {
945 return p.toFile().isDirectory();
946 }
947
948 private void delete(Path d) {
949 try {
950 Files.delete(d);
951 } catch (DirectoryNotEmptyException e) {
952
953 } catch (IOException e) {
954 LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
955 e);
956 }
957 }
958
959
960
961
962
963
964
965
966 private void deleteOrphans() {
967 Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
968 "pack");
969 List<String> fileNames = null;
970 try (Stream<Path> files = Files.list(packDir)) {
971 fileNames = files.map(path -> path.getFileName().toString())
972 .filter(name -> {
973 return (name.endsWith(PACK_EXT)
974 || name.endsWith(BITMAP_EXT)
975 || name.endsWith(INDEX_EXT));
976 }).sorted(Collections.reverseOrder())
977 .collect(Collectors.toList());
978 } catch (IOException e1) {
979
980 }
981 if (fileNames == null) {
982 return;
983 }
984
985 String base = null;
986 for (String n : fileNames) {
987 if (n.endsWith(PACK_EXT)) {
988 base = n.substring(0, n.lastIndexOf('.'));
989 } else {
990 if (base == null || !n.startsWith(base)) {
991 try {
992 Files.delete(new File(packDir.toFile(), n).toPath());
993 } catch (IOException e) {
994 LOG.error(e.getMessage(), e);
995 }
996 }
997 }
998 }
999 }
1000
1001 private void deleteTempPacksIdx() {
1002 Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
1003 "pack");
1004 Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
1005 try (DirectoryStream<Path> stream =
1006 Files.newDirectoryStream(packDir, "gc_*_tmp")) {
1007 stream.forEach(t -> {
1008 try {
1009 Instant lastModified = Files.getLastModifiedTime(t)
1010 .toInstant();
1011 if (lastModified.isBefore(threshold)) {
1012 Files.deleteIfExists(t);
1013 }
1014 } catch (IOException e) {
1015 LOG.error(e.getMessage(), e);
1016 }
1017 });
1018 } catch (IOException e) {
1019 LOG.error(e.getMessage(), e);
1020 }
1021 }
1022
1023
1024
1025
1026
1027
1028
1029
1030 private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
1031 ReflogReader reflogReader = repo.getReflogReader(ref.getName());
1032 if (reflogReader == null) {
1033 return Collections.emptySet();
1034 }
1035 List<ReflogEntry> rlEntries = reflogReader
1036 .getReverseEntries();
1037 if (rlEntries == null || rlEntries.isEmpty())
1038 return Collections.<ObjectId> emptySet();
1039 Set<ObjectId> ret = new HashSet<>();
1040 for (ReflogEntry e : rlEntries) {
1041 if (e.getWho().getWhen().getTime() < minTime)
1042 break;
1043 ObjectId newId = e.getNewId();
1044 if (newId != null && !ObjectId.zeroId().equals(newId))
1045 ret.add(newId);
1046 ObjectId oldId = e.getOldId();
1047 if (oldId != null && !ObjectId.zeroId().equals(oldId))
1048 ret.add(oldId);
1049 }
1050 return ret;
1051 }
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064 private Collection<Ref> getAllRefs() throws IOException {
1065 RefDatabase refdb = repo.getRefDatabase();
1066 Collection<Ref> refs = refdb.getRefs(RefDatabase.ALL).values();
1067 List<Ref> addl = refdb.getAdditionalRefs();
1068 if (!addl.isEmpty()) {
1069 List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1070 all.addAll(refs);
1071
1072 for (Ref r : addl) {
1073 checkCancelled();
1074 if (r.getName().startsWith(Constants.R_REFS)) {
1075 all.add(r);
1076 }
1077 }
1078 return all;
1079 }
1080 return refs;
1081 }
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092 private Set<ObjectId> listNonHEADIndexObjects()
1093 throws CorruptObjectException, IOException {
1094 if (repo.isBare()) {
1095 return Collections.emptySet();
1096 }
1097 try (TreeWalk treeWalk = new TreeWalk(repo)) {
1098 treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1099 ObjectId headID = repo.resolve(Constants.HEAD);
1100 if (headID != null) {
1101 try (RevWalk revWalk = new RevWalk(repo)) {
1102 treeWalk.addTree(revWalk.parseTree(headID));
1103 }
1104 }
1105
1106 treeWalk.setFilter(TreeFilter.ANY_DIFF);
1107 treeWalk.setRecursive(true);
1108 Set<ObjectId> ret = new HashSet<>();
1109
1110 while (treeWalk.next()) {
1111 checkCancelled();
1112 ObjectId objectId = treeWalk.getObjectId(0);
1113 switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1114 case FileMode.TYPE_MISSING:
1115 case FileMode.TYPE_GITLINK:
1116 continue;
1117 case FileMode.TYPE_TREE:
1118 case FileMode.TYPE_FILE:
1119 case FileMode.TYPE_SYMLINK:
1120 ret.add(objectId);
1121 continue;
1122 default:
1123 throw new IOException(MessageFormat.format(
1124 JGitText.get().corruptObjectInvalidMode3,
1125 String.format("%o",
1126 Integer.valueOf(treeWalk.getRawMode(0))),
1127 (objectId == null) ? "null" : objectId.name(),
1128 treeWalk.getPathString(),
1129 repo.getIndexFile()));
1130 }
1131 }
1132 return ret;
1133 }
1134 }
1135
1136 private PackFile writePack(@NonNull Set<? extends ObjectId> want,
1137 @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1138 Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1139 throws IOException {
1140 checkCancelled();
1141 File tmpPack = null;
1142 Map<PackExt, File> tmpExts = new TreeMap<>(
1143 new Comparator<PackExt>() {
1144 @Override
1145 public int compare(PackExt o1, PackExt o2) {
1146
1147
1148
1149 if (o1 == o2)
1150 return 0;
1151 if (o1 == PackExt.INDEX)
1152 return 1;
1153 if (o2 == PackExt.INDEX)
1154 return -1;
1155 return Integer.signum(o1.hashCode() - o2.hashCode());
1156 }
1157
1158 });
1159 try (PackWriter pw = new PackWriter(
1160 (pconfig == null) ? new PackConfig(repo) : pconfig,
1161 repo.newObjectReader())) {
1162
1163 pw.setDeltaBaseAsOffset(true);
1164 pw.setReuseDeltaCommits(false);
1165 if (tagTargets != null) {
1166 pw.setTagTargets(tagTargets);
1167 }
1168 if (excludeObjects != null)
1169 for (ObjectIdSet idx : excludeObjects)
1170 pw.excludeObjects(idx);
1171 pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1172 if (pw.getObjectCount() == 0)
1173 return null;
1174 checkCancelled();
1175
1176
1177 String id = pw.computeName().getName();
1178 File packdir = new File(repo.getObjectsDirectory(), "pack");
1179 tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir);
1180 final String tmpBase = tmpPack.getName()
1181 .substring(0, tmpPack.getName().lastIndexOf('.'));
1182 File tmpIdx = new File(packdir, tmpBase + ".idx_tmp");
1183 tmpExts.put(INDEX, tmpIdx);
1184
1185 if (!tmpIdx.createNewFile())
1186 throw new IOException(MessageFormat.format(
1187 JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1188
1189
1190 FileOutputStream fos = new FileOutputStream(tmpPack);
1191 FileChannel channel = fos.getChannel();
1192 OutputStream channelStream = Channels.newOutputStream(channel);
1193 try {
1194 pw.writePack(pm, pm, channelStream);
1195 } finally {
1196 channel.force(true);
1197 channelStream.close();
1198 fos.close();
1199 }
1200
1201
1202 fos = new FileOutputStream(tmpIdx);
1203 FileChannel idxChannel = fos.getChannel();
1204 OutputStream idxStream = Channels.newOutputStream(idxChannel);
1205 try {
1206 pw.writeIndex(idxStream);
1207 } finally {
1208 idxChannel.force(true);
1209 idxStream.close();
1210 fos.close();
1211 }
1212
1213 if (pw.prepareBitmapIndex(pm)) {
1214 File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp");
1215 tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1216
1217 if (!tmpBitmapIdx.createNewFile())
1218 throw new IOException(MessageFormat.format(
1219 JGitText.get().cannotCreateIndexfile,
1220 tmpBitmapIdx.getPath()));
1221
1222 fos = new FileOutputStream(tmpBitmapIdx);
1223 idxChannel = fos.getChannel();
1224 idxStream = Channels.newOutputStream(idxChannel);
1225 try {
1226 pw.writeBitmapIndex(idxStream);
1227 } finally {
1228 idxChannel.force(true);
1229 idxStream.close();
1230 fos.close();
1231 }
1232 }
1233
1234
1235 File realPack = nameFor(id, ".pack");
1236
1237 repo.getObjectDatabase().closeAllPackHandles(realPack);
1238 tmpPack.setReadOnly();
1239
1240 FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1241 for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1242 File tmpExt = tmpEntry.getValue();
1243 tmpExt.setReadOnly();
1244
1245 File realExt = nameFor(id,
1246 "." + tmpEntry.getKey().getExtension());
1247 try {
1248 FileUtils.rename(tmpExt, realExt,
1249 StandardCopyOption.ATOMIC_MOVE);
1250 } catch (IOException e) {
1251 File newExt = new File(realExt.getParentFile(),
1252 realExt.getName() + ".new");
1253 try {
1254 FileUtils.rename(tmpExt, newExt,
1255 StandardCopyOption.ATOMIC_MOVE);
1256 } catch (IOException e2) {
1257 newExt = tmpExt;
1258 e = e2;
1259 }
1260 throw new IOException(MessageFormat.format(
1261 JGitText.get().panicCantRenameIndexFile, newExt,
1262 realExt), e);
1263 }
1264 }
1265
1266 return repo.getObjectDatabase().openPack(realPack);
1267 } finally {
1268 if (tmpPack != null && tmpPack.exists())
1269 tmpPack.delete();
1270 for (File tmpExt : tmpExts.values()) {
1271 if (tmpExt.exists())
1272 tmpExt.delete();
1273 }
1274 }
1275 }
1276
1277 private File nameFor(String name, String ext) {
1278 File packdir = new File(repo.getObjectsDirectory(), "pack");
1279 return new File(packdir, "pack-" + name + ext);
1280 }
1281
1282 private void checkCancelled() throws CancelledException {
1283 if (pm.isCancelled()) {
1284 throw new CancelledException(JGitText.get().operationCanceled);
1285 }
1286 }
1287
1288
1289
1290
1291
1292 public static class RepoStatistics {
1293
1294
1295
1296
1297
1298 public long numberOfPackedObjects;
1299
1300
1301
1302
1303 public long numberOfPackFiles;
1304
1305
1306
1307
1308 public long numberOfLooseObjects;
1309
1310
1311
1312
1313 public long sizeOfLooseObjects;
1314
1315
1316
1317
1318 public long sizeOfPackedObjects;
1319
1320
1321
1322
1323 public long numberOfLooseRefs;
1324
1325
1326
1327
1328 public long numberOfPackedRefs;
1329
1330
1331
1332
1333 public long numberOfBitmaps;
1334
1335 @Override
1336 public String toString() {
1337 final StringBuilder b = new StringBuilder();
1338 b.append("numberOfPackedObjects=").append(numberOfPackedObjects);
1339 b.append(", numberOfPackFiles=").append(numberOfPackFiles);
1340 b.append(", numberOfLooseObjects=").append(numberOfLooseObjects);
1341 b.append(", numberOfLooseRefs=").append(numberOfLooseRefs);
1342 b.append(", numberOfPackedRefs=").append(numberOfPackedRefs);
1343 b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects);
1344 b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects);
1345 b.append(", numberOfBitmaps=").append(numberOfBitmaps);
1346 return b.toString();
1347 }
1348 }
1349
1350
1351
1352
1353
1354
1355
1356 public RepoStatistics getStatistics() throws IOException {
1357 RepoStatistics ret = new RepoStatistics();
1358 Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
1359 for (PackFile f : packs) {
1360 ret.numberOfPackedObjects += f.getIndex().getObjectCount();
1361 ret.numberOfPackFiles++;
1362 ret.sizeOfPackedObjects += f.getPackFile().length();
1363 if (f.getBitmapIndex() != null)
1364 ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
1365 }
1366 File objDir = repo.getObjectsDirectory();
1367 String[] fanout = objDir.list();
1368 if (fanout != null && fanout.length > 0) {
1369 for (String d : fanout) {
1370 if (d.length() != 2)
1371 continue;
1372 File[] entries = new File(objDir, d).listFiles();
1373 if (entries == null)
1374 continue;
1375 for (File f : entries) {
1376 if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1377 continue;
1378 ret.numberOfLooseObjects++;
1379 ret.sizeOfLooseObjects += f.length();
1380 }
1381 }
1382 }
1383
1384 RefDatabase refDb = repo.getRefDatabase();
1385 for (Ref r : refDb.getRefs(RefDatabase.ALL).values()) {
1386 Storage storage = r.getStorage();
1387 if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1388 ret.numberOfLooseRefs++;
1389 if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1390 ret.numberOfPackedRefs++;
1391 }
1392
1393 return ret;
1394 }
1395
1396
1397
1398
1399
1400
1401
1402 public GC setProgressMonitor(ProgressMonitor pm) {
1403 this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1404 return this;
1405 }
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416 public void setExpireAgeMillis(long expireAgeMillis) {
1417 this.expireAgeMillis = expireAgeMillis;
1418 expire = null;
1419 }
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430 public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1431 this.packExpireAgeMillis = packExpireAgeMillis;
1432 expire = null;
1433 }
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443 public void setPackConfig(PackConfig pconfig) {
1444 this.pconfig = pconfig;
1445 }
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459 public void setExpire(Date expire) {
1460 this.expire = expire;
1461 expireAgeMillis = -1;
1462 }
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473 public void setPackExpire(Date packExpire) {
1474 this.packExpire = packExpire;
1475 packExpireAgeMillis = -1;
1476 }
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513 public void setAuto(boolean auto) {
1514 this.automatic = auto;
1515 }
1516
1517
1518
1519
1520
1521 void setBackground(boolean background) {
1522 this.background = background;
1523 }
1524
1525 private boolean needGc() {
1526 if (tooManyPacks()) {
1527 addRepackAllOption();
1528 } else if (!tooManyLooseObjects()) {
1529 return false;
1530 }
1531
1532 return true;
1533 }
1534
1535 private void addRepackAllOption() {
1536
1537
1538 }
1539
1540
1541
1542
1543 boolean tooManyPacks() {
1544 int autopacklimit = repo.getConfig().getInt(
1545 ConfigConstants.CONFIG_GC_SECTION,
1546 ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1547 DEFAULT_AUTOPACKLIMIT);
1548 if (autopacklimit <= 0) {
1549 return false;
1550 }
1551
1552
1553 return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1554 }
1555
1556
1557
1558
1559
1560
1561
1562 boolean tooManyLooseObjects() {
1563 int auto = getLooseObjectLimit();
1564 if (auto <= 0) {
1565 return false;
1566 }
1567 int n = 0;
1568 int threshold = (auto + 255) / 256;
1569 Path dir = repo.getObjectsDirectory().toPath().resolve("17");
1570 if (!Files.exists(dir)) {
1571 return false;
1572 }
1573 try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir,
1574 new DirectoryStream.Filter<Path>() {
1575
1576 @Override
1577 public boolean accept(Path file) throws IOException {
1578 Path fileName = file.getFileName();
1579 return Files.isRegularFile(file) && fileName != null
1580 && PATTERN_LOOSE_OBJECT
1581 .matcher(fileName.toString()).matches();
1582 }
1583 })) {
1584 for (Iterator<Path> iter = stream.iterator(); iter.hasNext();
1585 iter.next()) {
1586 if (++n > threshold) {
1587 return true;
1588 }
1589 }
1590 } catch (IOException e) {
1591 LOG.error(e.getMessage(), e);
1592 }
1593 return false;
1594 }
1595
1596 private int getLooseObjectLimit() {
1597 return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1598 ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1599 }
1600 }