1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.internal.storage.file;
45
46 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
47 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
48
49 import java.io.File;
50 import java.io.FileOutputStream;
51 import java.io.IOException;
52 import java.io.OutputStream;
53 import java.io.PrintWriter;
54 import java.io.StringWriter;
55 import java.nio.channels.Channels;
56 import java.nio.channels.FileChannel;
57 import java.nio.file.DirectoryNotEmptyException;
58 import java.nio.file.DirectoryStream;
59 import java.nio.file.Files;
60 import java.nio.file.Path;
61 import java.nio.file.StandardCopyOption;
62 import java.text.MessageFormat;
63 import java.text.ParseException;
64 import java.time.Instant;
65 import java.time.temporal.ChronoUnit;
66 import java.util.ArrayList;
67 import java.util.Collection;
68 import java.util.Collections;
69 import java.util.Comparator;
70 import java.util.Date;
71 import java.util.HashMap;
72 import java.util.HashSet;
73 import java.util.Iterator;
74 import java.util.LinkedList;
75 import java.util.List;
76 import java.util.Map;
77 import java.util.Objects;
78 import java.util.Set;
79 import java.util.TreeMap;
80 import java.util.concurrent.Callable;
81 import java.util.concurrent.ExecutorService;
82 import java.util.regex.Pattern;
83 import java.util.stream.Collectors;
84 import java.util.stream.Stream;
85
86 import org.eclipse.jgit.annotations.NonNull;
87 import org.eclipse.jgit.dircache.DirCacheIterator;
88 import org.eclipse.jgit.errors.CancelledException;
89 import org.eclipse.jgit.errors.CorruptObjectException;
90 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
91 import org.eclipse.jgit.errors.MissingObjectException;
92 import org.eclipse.jgit.errors.NoWorkTreeException;
93 import org.eclipse.jgit.internal.JGitText;
94 import org.eclipse.jgit.internal.storage.pack.PackExt;
95 import org.eclipse.jgit.internal.storage.pack.PackWriter;
96 import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
97 import org.eclipse.jgit.lib.ConfigConstants;
98 import org.eclipse.jgit.lib.Constants;
99 import org.eclipse.jgit.lib.FileMode;
100 import org.eclipse.jgit.lib.NullProgressMonitor;
101 import org.eclipse.jgit.lib.ObjectId;
102 import org.eclipse.jgit.lib.ObjectIdSet;
103 import org.eclipse.jgit.lib.ObjectLoader;
104 import org.eclipse.jgit.lib.ObjectReader;
105 import org.eclipse.jgit.lib.ProgressMonitor;
106 import org.eclipse.jgit.lib.Ref;
107 import org.eclipse.jgit.lib.Ref.Storage;
108 import org.eclipse.jgit.lib.RefDatabase;
109 import org.eclipse.jgit.lib.ReflogEntry;
110 import org.eclipse.jgit.lib.ReflogReader;
111 import org.eclipse.jgit.lib.internal.WorkQueue;
112 import org.eclipse.jgit.revwalk.ObjectWalk;
113 import org.eclipse.jgit.revwalk.RevObject;
114 import org.eclipse.jgit.revwalk.RevWalk;
115 import org.eclipse.jgit.storage.pack.PackConfig;
116 import org.eclipse.jgit.treewalk.TreeWalk;
117 import org.eclipse.jgit.treewalk.filter.TreeFilter;
118 import org.eclipse.jgit.util.FileUtils;
119 import org.eclipse.jgit.util.GitDateParser;
120 import org.eclipse.jgit.util.SystemReader;
121 import org.slf4j.Logger;
122 import org.slf4j.LoggerFactory;
123
124
125
126
127
128
129
130
131
132
133 public class GC {
134 private final static Logger LOG = LoggerFactory
135 .getLogger(GC.class);
136
137 private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago";
138
139 private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago";
140
141 private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
142 .compile("[0-9a-fA-F]{38}");
143
144 private static final String PACK_EXT = "." + PackExt.PACK.getExtension();
145
146 private static final String BITMAP_EXT = "."
147 + PackExt.BITMAP_INDEX.getExtension();
148
149 private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension();
150
151 private static final int DEFAULT_AUTOPACKLIMIT = 50;
152
153 private static final int DEFAULT_AUTOLIMIT = 6700;
154
155 private static volatile ExecutorService executor;
156
157
158
159
160
161
162
163
164 public static void setExecutor(ExecutorService e) {
165 executor = e;
166 }
167
168 private final FileRepository repo;
169
170 private ProgressMonitor pm;
171
172 private long expireAgeMillis = -1;
173
174 private Date expire;
175
176 private long packExpireAgeMillis = -1;
177
178 private Date packExpire;
179
180 private PackConfig pconfig = null;
181
182
183
184
185
186
187
188 private Collection<Ref> lastPackedRefs;
189
190
191
192
193
194
195 private long lastRepackTime;
196
197
198
199
200 private boolean automatic;
201
202
203
204
205 private boolean background;
206
207
208
209
210
211
212
213
214 public GC(FileRepository repo) {
215 this.repo = repo;
216 this.pm = NullProgressMonitor.INSTANCE;
217 }
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247 @SuppressWarnings("FutureReturnValueIgnored")
248 public Collection<PackFile> gc() throws IOException, ParseException {
249 if (!background) {
250 return doGc();
251 }
252 final GcLogernal/storage/file/GcLog.html#GcLog">GcLog gcLog = new GcLog(repo);
253 if (!gcLog.lock()) {
254
255 return Collections.emptyList();
256 }
257
258 Callable<Collection<PackFile>> gcTask = () -> {
259 try {
260 Collection<PackFile> newPacks = doGc();
261 if (automatic && tooManyLooseObjects()) {
262 String message = JGitText.get().gcTooManyUnpruned;
263 gcLog.write(message);
264 gcLog.commit();
265 }
266 return newPacks;
267 } catch (IOException | ParseException e) {
268 try {
269 gcLog.write(e.getMessage());
270 StringWriter sw = new StringWriter();
271 e.printStackTrace(new PrintWriter(sw));
272 gcLog.write(sw.toString());
273 gcLog.commit();
274 } catch (IOException e2) {
275 e2.addSuppressed(e);
276 LOG.error(e2.getMessage(), e2);
277 }
278 } finally {
279 gcLog.unlock();
280 }
281 return Collections.emptyList();
282 };
283
284 executor().submit(gcTask);
285 return Collections.emptyList();
286 }
287
288 private ExecutorService executor() {
289 return (executor != null) ? executor : WorkQueue.getExecutor();
290 }
291
292 private Collection<PackFile> doGc() throws IOException, ParseException {
293 if (automatic && !needGc()) {
294 return Collections.emptyList();
295 }
296 pm.start(6 );
297 packRefs();
298
299 Collection<PackFile> newPacks = repack();
300 prune(Collections.emptySet());
301
302 return newPacks;
303 }
304
305
306
307
308
309
310
311
312
313
314
315 private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
316 throws IOException {
317 for (PackIndex.MutableEntry entry : pack) {
318 ObjectId oid = entry.toObjectId();
319 if (existing.contains(oid)) {
320 continue;
321 }
322 existing.add(oid);
323 ObjectLoader loader = reader.open(oid);
324 inserter.insert(loader.getType(),
325 loader.getSize(),
326 loader.openStream(),
327 true );
328 }
329 }
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347 private void deleteOldPacks(Collection<PackFile> oldPacks,
348 Collection<PackFile> newPacks) throws ParseException, IOException {
349 HashSet<ObjectId> ids = new HashSet<>();
350 for (PackFile pack : newPacks) {
351 for (PackIndex.MutableEntry entry : pack) {
352 ids.add(entry.toObjectId());
353 }
354 }
355 ObjectReader reader = repo.newObjectReader();
356 ObjectDirectory dir = repo.getObjectDatabase();
357 ObjectDirectoryInserter inserter = dir.newInserter();
358 boolean shouldLoosen = !"now".equals(getPruneExpireStr()) &&
359 getExpireDate() < Long.MAX_VALUE;
360
361 prunePreserved();
362 long packExpireDate = getPackExpireDate();
363 oldPackLoop: for (PackFile oldPack : oldPacks) {
364 checkCancelled();
365 String oldName = oldPack.getPackName();
366
367
368 for (PackFile newPack : newPacks)
369 if (oldName.equals(newPack.getPackName()))
370 continue oldPackLoop;
371
372 if (!oldPack.shouldBeKept()
373 && repo.getFS().lastModified(
374 oldPack.getPackFile()) < packExpireDate) {
375 oldPack.close();
376 if (shouldLoosen) {
377 loosen(inserter, reader, oldPack, ids);
378 }
379 prunePack(oldName);
380 }
381 }
382
383
384
385 repo.getObjectDatabase().close();
386 }
387
388
389
390
391
392
393
394
395
396
397
398 private void removeOldPack(File packFile, String packName, PackExt ext,
399 int deleteOptions) throws IOException {
400 if (pconfig != null && pconfig.isPreserveOldPacks()) {
401 File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
402 FileUtils.mkdir(oldPackDir, true);
403
404 String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();
405 File oldPackFile = new File(oldPackDir, oldPackName);
406 FileUtils.rename(packFile, oldPackFile);
407 } else {
408 FileUtils.delete(packFile, deleteOptions);
409 }
410 }
411
412
413
414
415 private void prunePreserved() {
416 if (pconfig != null && pconfig.isPrunePreserved()) {
417 try {
418 FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
419 FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
420 } catch (IOException e) {
421
422 }
423 }
424 }
425
426
427
428
429
430
431
432
433
434
435
436 private void prunePack(String packName) {
437 PackExt[] extensions = PackExt.values();
438 try {
439
440
441 int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
442 for (PackExt ext : extensions)
443 if (PackExt.PACK.equals(ext)) {
444 File f = nameFor(packName, "." + ext.getExtension());
445 removeOldPack(f, packName, ext, deleteOptions);
446 break;
447 }
448
449
450 deleteOptions |= FileUtils.IGNORE_ERRORS;
451 for (PackExt ext : extensions) {
452 if (!PackExt.PACK.equals(ext)) {
453 File f = nameFor(packName, "." + ext.getExtension());
454 removeOldPack(f, packName, ext, deleteOptions);
455 }
456 }
457 } catch (IOException e) {
458
459 }
460 }
461
462
463
464
465
466
467
468
469 public void prunePacked() throws IOException {
470 ObjectDirectory objdb = repo.getObjectDatabase();
471 Collection<PackFile> packs = objdb.getPacks();
472 File objects = repo.getObjectsDirectory();
473 String[] fanout = objects.list();
474
475 if (fanout != null && fanout.length > 0) {
476 pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
477 try {
478 for (String d : fanout) {
479 checkCancelled();
480 pm.update(1);
481 if (d.length() != 2)
482 continue;
483 String[] entries = new File(objects, d).list();
484 if (entries == null)
485 continue;
486 for (String e : entries) {
487 checkCancelled();
488 if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
489 continue;
490 ObjectId id;
491 try {
492 id = ObjectId.fromString(d + e);
493 } catch (IllegalArgumentException notAnObject) {
494
495
496 continue;
497 }
498 boolean found = false;
499 for (PackFile p : packs) {
500 checkCancelled();
501 if (p.hasObject(id)) {
502 found = true;
503 break;
504 }
505 }
506 if (found)
507 FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
508 | FileUtils.SKIP_MISSING
509 | FileUtils.IGNORE_ERRORS);
510 }
511 }
512 } finally {
513 pm.endTask();
514 }
515 }
516 }
517
518
519
520
521
522
523
524
525
526
527
528
529
530 public void prune(Set<ObjectId> objectsToKeep) throws IOException,
531 ParseException {
532 long expireDate = getExpireDate();
533
534
535
536 Map<ObjectId, File> deletionCandidates = new HashMap<>();
537 Set<ObjectId> indexObjects = null;
538 File objects = repo.getObjectsDirectory();
539 String[] fanout = objects.list();
540 if (fanout == null || fanout.length == 0) {
541 return;
542 }
543 pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
544 fanout.length);
545 try {
546 for (String d : fanout) {
547 checkCancelled();
548 pm.update(1);
549 if (d.length() != 2)
550 continue;
551 File[] entries = new File(objects, d).listFiles();
552 if (entries == null)
553 continue;
554 for (File f : entries) {
555 checkCancelled();
556 String fName = f.getName();
557 if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
558 continue;
559 if (repo.getFS().lastModified(f) >= expireDate)
560 continue;
561 try {
562 ObjectId id = ObjectId.fromString(d + fName);
563 if (objectsToKeep.contains(id))
564 continue;
565 if (indexObjects == null)
566 indexObjects = listNonHEADIndexObjects();
567 if (indexObjects.contains(id))
568 continue;
569 deletionCandidates.put(id, f);
570 } catch (IllegalArgumentException notAnObject) {
571
572
573 }
574 }
575 }
576 } finally {
577 pm.endTask();
578 }
579
580 if (deletionCandidates.isEmpty()) {
581 return;
582 }
583
584 checkCancelled();
585
586
587
588
589
590 Collection<Ref> newRefs;
591 if (lastPackedRefs == null || lastPackedRefs.isEmpty())
592 newRefs = getAllRefs();
593 else {
594 Map<String, Ref> last = new HashMap<>();
595 for (Ref r : lastPackedRefs) {
596 last.put(r.getName(), r);
597 }
598 newRefs = new ArrayList<>();
599 for (Ref r : getAllRefs()) {
600 Ref old = last.get(r.getName());
601 if (!equals(r, old)) {
602 newRefs.add(r);
603 }
604 }
605 }
606
607 if (!newRefs.isEmpty()) {
608
609
610
611
612
613 ObjectWalk w = new ObjectWalk(repo);
614 try {
615 for (Ref cr : newRefs) {
616 checkCancelled();
617 w.markStart(w.parseAny(cr.getObjectId()));
618 }
619 if (lastPackedRefs != null)
620 for (Ref lpr : lastPackedRefs) {
621 w.markUninteresting(w.parseAny(lpr.getObjectId()));
622 }
623 removeReferenced(deletionCandidates, w);
624 } finally {
625 w.dispose();
626 }
627 }
628
629 if (deletionCandidates.isEmpty())
630 return;
631
632
633
634
635
636
637 ObjectWalk w = new ObjectWalk(repo);
638 try {
639 for (Ref ar : getAllRefs())
640 for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
641 checkCancelled();
642 w.markStart(w.parseAny(id));
643 }
644 if (lastPackedRefs != null)
645 for (Ref lpr : lastPackedRefs) {
646 checkCancelled();
647 w.markUninteresting(w.parseAny(lpr.getObjectId()));
648 }
649 removeReferenced(deletionCandidates, w);
650 } finally {
651 w.dispose();
652 }
653
654 if (deletionCandidates.isEmpty())
655 return;
656
657 checkCancelled();
658
659
660
661
662
663 Set<File> touchedFanout = new HashSet<>();
664 for (File f : deletionCandidates.values()) {
665 if (f.lastModified() < expireDate) {
666 f.delete();
667 touchedFanout.add(f.getParentFile());
668 }
669 }
670
671 for (File f : touchedFanout) {
672 FileUtils.delete(f,
673 FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
674 }
675
676 repo.getObjectDatabase().close();
677 }
678
679 private long getExpireDate() throws ParseException {
680 long expireDate = Long.MAX_VALUE;
681
682 if (expire == null && expireAgeMillis == -1) {
683 String pruneExpireStr = getPruneExpireStr();
684 if (pruneExpireStr == null)
685 pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
686 expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
687 .getInstance().getLocale());
688 expireAgeMillis = -1;
689 }
690 if (expire != null)
691 expireDate = expire.getTime();
692 if (expireAgeMillis != -1)
693 expireDate = System.currentTimeMillis() - expireAgeMillis;
694 return expireDate;
695 }
696
697 private String getPruneExpireStr() {
698 return repo.getConfig().getString(
699 ConfigConstants.CONFIG_GC_SECTION, null,
700 ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
701 }
702
703 private long getPackExpireDate() throws ParseException {
704 long packExpireDate = Long.MAX_VALUE;
705
706 if (packExpire == null && packExpireAgeMillis == -1) {
707 String prunePackExpireStr = repo.getConfig().getString(
708 ConfigConstants.CONFIG_GC_SECTION, null,
709 ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
710 if (prunePackExpireStr == null)
711 prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
712 packExpire = GitDateParser.parse(prunePackExpireStr, null,
713 SystemReader.getInstance().getLocale());
714 packExpireAgeMillis = -1;
715 }
716 if (packExpire != null)
717 packExpireDate = packExpire.getTime();
718 if (packExpireAgeMillis != -1)
719 packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
720 return packExpireDate;
721 }
722
723
724
725
726
727
728
729
730
731
732
733 private void removeReferenced(Map<ObjectId, File> id2File,
734 ObjectWalk w) throws MissingObjectException,
735 IncorrectObjectTypeException, IOException {
736 RevObject ro = w.next();
737 while (ro != null) {
738 checkCancelled();
739 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
740 return;
741 }
742 ro = w.next();
743 }
744 ro = w.nextObject();
745 while (ro != null) {
746 checkCancelled();
747 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
748 return;
749 }
750 ro = w.nextObject();
751 }
752 }
753
754 private static boolean equals(Reff" href="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref r1, Ref r2) {
755 if (r1 == null || r2 == null) {
756 return false;
757 }
758 if (r1.isSymbolic()) {
759 return r2.isSymbolic() && r1.getTarget().getName()
760 .equals(r2.getTarget().getName());
761 }
762 return !r2.isSymbolic()
763 && Objects.equals(r1.getObjectId(), r2.getObjectId());
764 }
765
766
767
768
769
770
771 public void packRefs() throws IOException {
772 Collection<Ref> refs = repo.getRefDatabase()
773 .getRefsByPrefix(Constants.R_REFS);
774 List<String> refsToBePacked = new ArrayList<>(refs.size());
775 pm.beginTask(JGitText.get().packRefs, refs.size());
776 try {
777 for (Ref ref : refs) {
778 checkCancelled();
779 if (!ref.isSymbolic() && ref.getStorage().isLoose())
780 refsToBePacked.add(ref.getName());
781 pm.update(1);
782 }
783 ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
784 } finally {
785 pm.endTask();
786 }
787 }
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803 public Collection<PackFile> repack() throws IOException {
804 Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
805
806 long time = System.currentTimeMillis();
807 Collection<Ref> refsBefore = getAllRefs();
808
809 Set<ObjectId> allHeadsAndTags = new HashSet<>();
810 Set<ObjectId> allHeads = new HashSet<>();
811 Set<ObjectId> allTags = new HashSet<>();
812 Set<ObjectId> nonHeads = new HashSet<>();
813 Set<ObjectId> txnHeads = new HashSet<>();
814 Set<ObjectId> tagTargets = new HashSet<>();
815 Set<ObjectId> indexObjects = listNonHEADIndexObjects();
816 RefDatabase refdb = repo.getRefDatabase();
817
818 for (Ref ref : refsBefore) {
819 checkCancelled();
820 nonHeads.addAll(listRefLogObjects(ref, 0));
821 if (ref.isSymbolic() || ref.getObjectId() == null) {
822 continue;
823 }
824 if (isHead(ref)) {
825 allHeads.add(ref.getObjectId());
826 } else if (isTag(ref)) {
827 allTags.add(ref.getObjectId());
828 } else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
829 txnHeads.add(ref.getObjectId());
830 } else {
831 nonHeads.add(ref.getObjectId());
832 }
833 if (ref.getPeeledObjectId() != null) {
834 tagTargets.add(ref.getPeeledObjectId());
835 }
836 }
837
838 List<ObjectIdSet> excluded = new LinkedList<>();
839 for (PackFile f : repo.getObjectDatabase().getPacks()) {
840 checkCancelled();
841 if (f.shouldBeKept())
842 excluded.add(f.getIndex());
843 }
844
845
846 allTags.removeAll(allHeads);
847 allHeadsAndTags.addAll(allHeads);
848 allHeadsAndTags.addAll(allTags);
849
850
851 tagTargets.addAll(allHeadsAndTags);
852 nonHeads.addAll(indexObjects);
853
854
855 if (pconfig != null && pconfig.getSinglePack()) {
856 allHeadsAndTags.addAll(nonHeads);
857 nonHeads.clear();
858 }
859
860 List<PackFile> ret = new ArrayList<>(2);
861 PackFile heads = null;
862 if (!allHeadsAndTags.isEmpty()) {
863 heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
864 tagTargets, excluded);
865 if (heads != null) {
866 ret.add(heads);
867 excluded.add(0, heads.getIndex());
868 }
869 }
870 if (!nonHeads.isEmpty()) {
871 PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
872 tagTargets, excluded);
873 if (rest != null)
874 ret.add(rest);
875 }
876 if (!txnHeads.isEmpty()) {
877 PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
878 null, excluded);
879 if (txn != null)
880 ret.add(txn);
881 }
882 try {
883 deleteOldPacks(toBeDeleted, ret);
884 } catch (ParseException e) {
885
886
887
888 throw new IOException(e);
889 }
890 prunePacked();
891 deleteEmptyRefsFolders();
892 deleteOrphans();
893 deleteTempPacksIdx();
894
895 lastPackedRefs = refsBefore;
896 lastRepackTime = time;
897 return ret;
898 }
899
900 private static boolean isHead(Ref ref) {
901 return ref.getName().startsWith(Constants.R_HEADS);
902 }
903
904 private static boolean isTag(Ref ref) {
905 return ref.getName().startsWith(Constants.R_TAGS);
906 }
907
908 private void deleteEmptyRefsFolders() throws IOException {
909 Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
910
911
912 Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
913 try (Stream<Path> entries = Files.list(refs)
914 .filter(Files::isDirectory)) {
915 Iterator<Path> iterator = entries.iterator();
916 while (iterator.hasNext()) {
917 try (Stream<Path> s = Files.list(iterator.next())) {
918 s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
919 }
920 }
921 }
922 }
923
924 private boolean canBeSafelyDeleted(Path path, Instant threshold) {
925 try {
926 return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
927 }
928 catch (IOException e) {
929 LOG.warn(MessageFormat.format(
930 JGitText.get().cannotAccessLastModifiedForSafeDeletion,
931 path), e);
932 return false;
933 }
934 }
935
936 private void deleteDir(Path dir) {
937 try (Stream<Path> dirs = Files.walk(dir)) {
938 dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
939 .forEach(this::delete);
940 } catch (IOException e) {
941 LOG.error(e.getMessage(), e);
942 }
943 }
944
945 private boolean isDirectory(Path p) {
946 return p.toFile().isDirectory();
947 }
948
949 private void delete(Path d) {
950 try {
951 Files.delete(d);
952 } catch (DirectoryNotEmptyException e) {
953
954 } catch (IOException e) {
955 LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
956 e);
957 }
958 }
959
960
961
962
963
964
965
966
967 private void deleteOrphans() {
968 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
969 List<String> fileNames = null;
970 try (Stream<Path> files = Files.list(packDir)) {
971 fileNames = files.map(path -> path.getFileName().toString())
972 .filter(name -> (name.endsWith(PACK_EXT)
973 || name.endsWith(BITMAP_EXT)
974 || name.endsWith(INDEX_EXT)))
975 .sorted(Collections.reverseOrder())
976 .collect(Collectors.toList());
977 } catch (IOException e1) {
978
979 }
980 if (fileNames == null) {
981 return;
982 }
983
984 String base = null;
985 for (String n : fileNames) {
986 if (n.endsWith(PACK_EXT)) {
987 base = n.substring(0, n.lastIndexOf('.'));
988 } else {
989 if (base == null || !n.startsWith(base)) {
990 try {
991 Files.delete(packDir.resolve(n));
992 } catch (IOException e) {
993 LOG.error(e.getMessage(), e);
994 }
995 }
996 }
997 }
998 }
999
1000 private void deleteTempPacksIdx() {
1001 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
1002 Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
1003 if (!Files.exists(packDir)) {
1004 return;
1005 }
1006 try (DirectoryStream<Path> stream =
1007 Files.newDirectoryStream(packDir, "gc_*_tmp")) {
1008 stream.forEach(t -> {
1009 try {
1010 Instant lastModified = Files.getLastModifiedTime(t)
1011 .toInstant();
1012 if (lastModified.isBefore(threshold)) {
1013 Files.deleteIfExists(t);
1014 }
1015 } catch (IOException e) {
1016 LOG.error(e.getMessage(), e);
1017 }
1018 });
1019 } catch (IOException e) {
1020 LOG.error(e.getMessage(), e);
1021 }
1022 }
1023
1024
1025
1026
1027
1028
1029
1030
1031 private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
1032 ReflogReader reflogReader = repo.getReflogReader(ref.getName());
1033 if (reflogReader == null) {
1034 return Collections.emptySet();
1035 }
1036 List<ReflogEntry> rlEntries = reflogReader
1037 .getReverseEntries();
1038 if (rlEntries == null || rlEntries.isEmpty())
1039 return Collections.emptySet();
1040 Set<ObjectId> ret = new HashSet<>();
1041 for (ReflogEntry e : rlEntries) {
1042 if (e.getWho().getWhen().getTime() < minTime)
1043 break;
1044 ObjectId newId = e.getNewId();
1045 if (newId != null && !ObjectId.zeroId().equals(newId))
1046 ret.add(newId);
1047 ObjectId oldId = e.getOldId();
1048 if (oldId != null && !ObjectId.zeroId().equals(oldId))
1049 ret.add(oldId);
1050 }
1051 return ret;
1052 }
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065 private Collection<Ref> getAllRefs() throws IOException {
1066 RefDatabase refdb = repo.getRefDatabase();
1067 Collection<Ref> refs = refdb.getRefs();
1068 List<Ref> addl = refdb.getAdditionalRefs();
1069 if (!addl.isEmpty()) {
1070 List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1071 all.addAll(refs);
1072
1073 for (Ref r : addl) {
1074 checkCancelled();
1075 if (r.getName().startsWith(Constants.R_REFS)) {
1076 all.add(r);
1077 }
1078 }
1079 return all;
1080 }
1081 return refs;
1082 }
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093 private Set<ObjectId> listNonHEADIndexObjects()
1094 throws CorruptObjectException, IOException {
1095 if (repo.isBare()) {
1096 return Collections.emptySet();
1097 }
1098 try (TreeWalkTreeWalk.html#TreeWalk">TreeWalk treeWalk = new TreeWalk(repo)) {
1099 treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1100 ObjectId headID = repo.resolve(Constants.HEAD);
1101 if (headID != null) {
1102 try (RevWalk/RevWalk.html#RevWalk">RevWalk revWalk = new RevWalk(repo)) {
1103 treeWalk.addTree(revWalk.parseTree(headID));
1104 }
1105 }
1106
1107 treeWalk.setFilter(TreeFilter.ANY_DIFF);
1108 treeWalk.setRecursive(true);
1109 Set<ObjectId> ret = new HashSet<>();
1110
1111 while (treeWalk.next()) {
1112 checkCancelled();
1113 ObjectId objectId = treeWalk.getObjectId(0);
1114 switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1115 case FileMode.TYPE_MISSING:
1116 case FileMode.TYPE_GITLINK:
1117 continue;
1118 case FileMode.TYPE_TREE:
1119 case FileMode.TYPE_FILE:
1120 case FileMode.TYPE_SYMLINK:
1121 ret.add(objectId);
1122 continue;
1123 default:
1124 throw new IOException(MessageFormat.format(
1125 JGitText.get().corruptObjectInvalidMode3,
1126 String.format("%o",
1127 Integer.valueOf(treeWalk.getRawMode(0))),
1128 (objectId == null) ? "null" : objectId.name(),
1129 treeWalk.getPathString(),
1130 repo.getIndexFile()));
1131 }
1132 }
1133 return ret;
1134 }
1135 }
1136
1137 private PackFile writePack(@NonNull Set<? extends ObjectId> want,
1138 @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1139 Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1140 throws IOException {
1141 checkCancelled();
1142 File tmpPack = null;
1143 Map<PackExt, File> tmpExts = new TreeMap<>((o1, o2) -> {
1144
1145
1146
1147 if (o1 == o2) {
1148 return 0;
1149 }
1150 if (o1 == PackExt.INDEX) {
1151 return 1;
1152 }
1153 if (o2 == PackExt.INDEX) {
1154 return -1;
1155 }
1156 return Integer.signum(o1.hashCode() - o2.hashCode());
1157 });
1158 try (PackWriternal/storage/pack/PackWriter.html#PackWriter">PackWriter pw = new PackWriter(
1159 (pconfig == null) ? new PackConfig(repo) : pconfig,
1160 repo.newObjectReader())) {
1161
1162 pw.setDeltaBaseAsOffset(true);
1163 pw.setReuseDeltaCommits(false);
1164 if (tagTargets != null) {
1165 pw.setTagTargets(tagTargets);
1166 }
1167 if (excludeObjects != null)
1168 for (ObjectIdSet idx : excludeObjects)
1169 pw.excludeObjects(idx);
1170 pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1171 if (pw.getObjectCount() == 0)
1172 return null;
1173 checkCancelled();
1174
1175
1176 String id = pw.computeName().getName();
1177 File packdir = repo.getObjectDatabase().getPackDirectory();
1178 tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir);
1179 final String tmpBase = tmpPack.getName()
1180 .substring(0, tmpPack.getName().lastIndexOf('.'));
1181 File tmpIdx = new File(packdir, tmpBase + ".idx_tmp");
1182 tmpExts.put(INDEX, tmpIdx);
1183
1184 if (!tmpIdx.createNewFile())
1185 throw new IOException(MessageFormat.format(
1186 JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1187
1188
1189 try (FileOutputStream fos = new FileOutputStream(tmpPack);
1190 FileChannel channel = fos.getChannel();
1191 OutputStream channelStream = Channels
1192 .newOutputStream(channel)) {
1193 pw.writePack(pm, pm, channelStream);
1194 channel.force(true);
1195 }
1196
1197
1198 try (FileOutputStream fos = new FileOutputStream(tmpIdx);
1199 FileChannel idxChannel = fos.getChannel();
1200 OutputStream idxStream = Channels
1201 .newOutputStream(idxChannel)) {
1202 pw.writeIndex(idxStream);
1203 idxChannel.force(true);
1204 }
1205
1206 if (pw.prepareBitmapIndex(pm)) {
1207 File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp");
1208 tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1209
1210 if (!tmpBitmapIdx.createNewFile())
1211 throw new IOException(MessageFormat.format(
1212 JGitText.get().cannotCreateIndexfile,
1213 tmpBitmapIdx.getPath()));
1214
1215 try (FileOutputStream fos = new FileOutputStream(tmpBitmapIdx);
1216 FileChannel idxChannel = fos.getChannel();
1217 OutputStream idxStream = Channels
1218 .newOutputStream(idxChannel)) {
1219 pw.writeBitmapIndex(idxStream);
1220 idxChannel.force(true);
1221 }
1222 }
1223
1224
1225 File realPack = nameFor(id, ".pack");
1226
1227 repo.getObjectDatabase().closeAllPackHandles(realPack);
1228 tmpPack.setReadOnly();
1229
1230 FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1231 for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1232 File tmpExt = tmpEntry.getValue();
1233 tmpExt.setReadOnly();
1234
1235 File realExt = nameFor(id,
1236 "." + tmpEntry.getKey().getExtension());
1237 try {
1238 FileUtils.rename(tmpExt, realExt,
1239 StandardCopyOption.ATOMIC_MOVE);
1240 } catch (IOException e) {
1241 File newExt = new File(realExt.getParentFile(),
1242 realExt.getName() + ".new");
1243 try {
1244 FileUtils.rename(tmpExt, newExt,
1245 StandardCopyOption.ATOMIC_MOVE);
1246 } catch (IOException e2) {
1247 newExt = tmpExt;
1248 e = e2;
1249 }
1250 throw new IOException(MessageFormat.format(
1251 JGitText.get().panicCantRenameIndexFile, newExt,
1252 realExt), e);
1253 }
1254 }
1255
1256 return repo.getObjectDatabase().openPack(realPack);
1257 } finally {
1258 if (tmpPack != null && tmpPack.exists())
1259 tmpPack.delete();
1260 for (File tmpExt : tmpExts.values()) {
1261 if (tmpExt.exists())
1262 tmpExt.delete();
1263 }
1264 }
1265 }
1266
1267 private File nameFor(String name, String ext) {
1268 File packdir = repo.getObjectDatabase().getPackDirectory();
1269 return new File(packdir, "pack-" + name + ext);
1270 }
1271
1272 private void checkCancelled() throws CancelledException {
1273 if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
1274 throw new CancelledException(JGitText.get().operationCanceled);
1275 }
1276 }
1277
1278
1279
1280
1281
1282 public static class RepoStatistics {
1283
1284
1285
1286
1287
1288 public long numberOfPackedObjects;
1289
1290
1291
1292
1293 public long numberOfPackFiles;
1294
1295
1296
1297
1298 public long numberOfLooseObjects;
1299
1300
1301
1302
1303 public long sizeOfLooseObjects;
1304
1305
1306
1307
1308 public long sizeOfPackedObjects;
1309
1310
1311
1312
1313 public long numberOfLooseRefs;
1314
1315
1316
1317
1318 public long numberOfPackedRefs;
1319
1320
1321
1322
1323 public long numberOfBitmaps;
1324
1325 @Override
1326 public String toString() {
1327 final StringBuilder b = new StringBuilder();
1328 b.append("numberOfPackedObjects=").append(numberOfPackedObjects);
1329 b.append(", numberOfPackFiles=").append(numberOfPackFiles);
1330 b.append(", numberOfLooseObjects=").append(numberOfLooseObjects);
1331 b.append(", numberOfLooseRefs=").append(numberOfLooseRefs);
1332 b.append(", numberOfPackedRefs=").append(numberOfPackedRefs);
1333 b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects);
1334 b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects);
1335 b.append(", numberOfBitmaps=").append(numberOfBitmaps);
1336 return b.toString();
1337 }
1338 }
1339
1340
1341
1342
1343
1344
1345
1346 public RepoStatistics getStatistics() throws IOException {
1347 RepoStatistics ret = new RepoStatistics();
1348 Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
1349 for (PackFile f : packs) {
1350 ret.numberOfPackedObjects += f.getIndex().getObjectCount();
1351 ret.numberOfPackFiles++;
1352 ret.sizeOfPackedObjects += f.getPackFile().length();
1353 if (f.getBitmapIndex() != null)
1354 ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
1355 }
1356 File objDir = repo.getObjectsDirectory();
1357 String[] fanout = objDir.list();
1358 if (fanout != null && fanout.length > 0) {
1359 for (String d : fanout) {
1360 if (d.length() != 2)
1361 continue;
1362 File[] entries = new File(objDir, d).listFiles();
1363 if (entries == null)
1364 continue;
1365 for (File f : entries) {
1366 if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1367 continue;
1368 ret.numberOfLooseObjects++;
1369 ret.sizeOfLooseObjects += f.length();
1370 }
1371 }
1372 }
1373
1374 RefDatabase refDb = repo.getRefDatabase();
1375 for (Ref r : refDb.getRefs()) {
1376 Storage storage = r.getStorage();
1377 if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1378 ret.numberOfLooseRefs++;
1379 if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1380 ret.numberOfPackedRefs++;
1381 }
1382
1383 return ret;
1384 }
1385
1386
1387
1388
1389
1390
1391
1392 public GC setProgressMonitor(ProgressMonitor pm) {
1393 this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1394 return this;
1395 }
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406 public void setExpireAgeMillis(long expireAgeMillis) {
1407 this.expireAgeMillis = expireAgeMillis;
1408 expire = null;
1409 }
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420 public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1421 this.packExpireAgeMillis = packExpireAgeMillis;
1422 expire = null;
1423 }
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434 public void setPackConfig(PackConfig pconfig) {
1435 this.pconfig = pconfig;
1436 }
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450 public void setExpire(Date expire) {
1451 this.expire = expire;
1452 expireAgeMillis = -1;
1453 }
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464 public void setPackExpire(Date packExpire) {
1465 this.packExpire = packExpire;
1466 packExpireAgeMillis = -1;
1467 }
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504 public void setAuto(boolean auto) {
1505 this.automatic = auto;
1506 }
1507
1508
1509
1510
1511
1512 void setBackground(boolean background) {
1513 this.background = background;
1514 }
1515
1516 private boolean needGc() {
1517 if (tooManyPacks()) {
1518 addRepackAllOption();
1519 } else {
1520 return tooManyLooseObjects();
1521 }
1522
1523 return true;
1524 }
1525
1526 private void addRepackAllOption() {
1527
1528
1529 }
1530
1531
1532
1533
1534 boolean tooManyPacks() {
1535 int autopacklimit = repo.getConfig().getInt(
1536 ConfigConstants.CONFIG_GC_SECTION,
1537 ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1538 DEFAULT_AUTOPACKLIMIT);
1539 if (autopacklimit <= 0) {
1540 return false;
1541 }
1542
1543
1544 return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1545 }
1546
1547
1548
1549
1550
1551
1552
1553 boolean tooManyLooseObjects() {
1554 int auto = getLooseObjectLimit();
1555 if (auto <= 0) {
1556 return false;
1557 }
1558 int n = 0;
1559 int threshold = (auto + 255) / 256;
1560 Path dir = repo.getObjectsDirectory().toPath().resolve("17");
1561 if (!dir.toFile().exists()) {
1562 return false;
1563 }
1564 try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, file -> {
1565 Path fileName = file.getFileName();
1566 return file.toFile().isFile() && fileName != null
1567 && PATTERN_LOOSE_OBJECT.matcher(fileName.toString())
1568 .matches();
1569 })) {
1570 for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); iter
1571 .next()) {
1572 if (++n > threshold) {
1573 return true;
1574 }
1575 }
1576 } catch (IOException e) {
1577 LOG.error(e.getMessage(), e);
1578 }
1579 return false;
1580 }
1581
1582 private int getLooseObjectLimit() {
1583 return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1584 ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1585 }
1586 }