1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.internal.storage.file;
45
46 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
47 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
48
49 import java.io.File;
50 import java.io.FileOutputStream;
51 import java.io.IOException;
52 import java.io.OutputStream;
53 import java.io.PrintWriter;
54 import java.io.StringWriter;
55 import java.nio.channels.Channels;
56 import java.nio.channels.FileChannel;
57 import java.nio.file.DirectoryStream;
58 import java.nio.file.Files;
59 import java.nio.file.Path;
60 import java.nio.file.Paths;
61 import java.nio.file.StandardCopyOption;
62 import java.text.MessageFormat;
63 import java.text.ParseException;
64 import java.time.Instant;
65 import java.time.temporal.ChronoUnit;
66 import java.util.ArrayList;
67 import java.util.Collection;
68 import java.util.Collections;
69 import java.util.Comparator;
70 import java.util.Date;
71 import java.util.HashMap;
72 import java.util.HashSet;
73 import java.util.Iterator;
74 import java.util.LinkedList;
75 import java.util.List;
76 import java.util.Map;
77 import java.util.Objects;
78 import java.util.Set;
79 import java.util.TreeMap;
80 import java.util.concurrent.Callable;
81 import java.util.concurrent.ExecutionException;
82 import java.util.concurrent.ExecutorService;
83 import java.util.concurrent.Future;
84 import java.util.regex.Pattern;
85 import java.util.stream.Collectors;
86 import java.util.stream.Stream;
87
88 import org.eclipse.jgit.annotations.NonNull;
89 import org.eclipse.jgit.api.errors.JGitInternalException;
90 import org.eclipse.jgit.dircache.DirCacheIterator;
91 import org.eclipse.jgit.errors.CancelledException;
92 import org.eclipse.jgit.errors.CorruptObjectException;
93 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
94 import org.eclipse.jgit.errors.MissingObjectException;
95 import org.eclipse.jgit.errors.NoWorkTreeException;
96 import org.eclipse.jgit.internal.JGitText;
97 import org.eclipse.jgit.internal.storage.pack.PackExt;
98 import org.eclipse.jgit.internal.storage.pack.PackWriter;
99 import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
100 import org.eclipse.jgit.lib.ConfigConstants;
101 import org.eclipse.jgit.lib.Constants;
102 import org.eclipse.jgit.lib.FileMode;
103 import org.eclipse.jgit.lib.NullProgressMonitor;
104 import org.eclipse.jgit.lib.ObjectId;
105 import org.eclipse.jgit.lib.ObjectIdSet;
106 import org.eclipse.jgit.lib.ObjectLoader;
107 import org.eclipse.jgit.lib.ObjectReader;
108 import org.eclipse.jgit.lib.ProgressMonitor;
109 import org.eclipse.jgit.lib.Ref;
110 import org.eclipse.jgit.lib.Ref.Storage;
111 import org.eclipse.jgit.lib.RefDatabase;
112 import org.eclipse.jgit.lib.ReflogEntry;
113 import org.eclipse.jgit.lib.ReflogReader;
114 import org.eclipse.jgit.lib.internal.WorkQueue;
115 import org.eclipse.jgit.revwalk.ObjectWalk;
116 import org.eclipse.jgit.revwalk.RevObject;
117 import org.eclipse.jgit.revwalk.RevWalk;
118 import org.eclipse.jgit.storage.pack.PackConfig;
119 import org.eclipse.jgit.treewalk.TreeWalk;
120 import org.eclipse.jgit.treewalk.filter.TreeFilter;
121 import org.eclipse.jgit.util.FileUtils;
122 import org.eclipse.jgit.util.GitDateParser;
123 import org.eclipse.jgit.util.SystemReader;
124 import org.slf4j.Logger;
125 import org.slf4j.LoggerFactory;
126
127
128
129
130
131
132
133
134 public class GC {
135 private final static Logger LOG = LoggerFactory
136 .getLogger(GC.class);
137
138 private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago";
139
140 private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago";
141
142 private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
143 .compile("[0-9a-fA-F]{38}");
144
145 private static final String PACK_EXT = "." + PackExt.PACK.getExtension();
146
147 private static final String BITMAP_EXT = "."
148 + PackExt.BITMAP_INDEX.getExtension();
149
150 private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension();
151
152 private static final int DEFAULT_AUTOPACKLIMIT = 50;
153
154 private static final int DEFAULT_AUTOLIMIT = 6700;
155
156 private static volatile ExecutorService executor;
157
158
159
160
161
162
163
164
165
166 public static void setExecutor(ExecutorService e) {
167 executor = e;
168 }
169
170 private final FileRepository repo;
171
172 private ProgressMonitor pm;
173
174 private long expireAgeMillis = -1;
175
176 private Date expire;
177
178 private long packExpireAgeMillis = -1;
179
180 private Date packExpire;
181
182 private PackConfig pconfig = null;
183
184
185
186
187
188
189
190 private Collection<Ref> lastPackedRefs;
191
192
193
194
195
196
197 private long lastRepackTime;
198
199
200
201
202 private boolean automatic;
203
204
205
206
207 private boolean background;
208
209
210
211
212
213
214
215
216 public GC(FileRepository repo) {
217 this.repo = repo;
218 this.pm = NullProgressMonitor.INSTANCE;
219 }
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246 public Collection<PackFile> gc() throws IOException, ParseException {
247 final GcLog gcLog = background ? new GcLog(repo) : null;
248 if (gcLog != null && !gcLog.lock(background)) {
249
250 return Collections.emptyList();
251 }
252
253 Callable<Collection<PackFile>> gcTask = () -> {
254 try {
255 Collection<PackFile> newPacks = doGc();
256 if (automatic && tooManyLooseObjects() && gcLog != null) {
257 String message = JGitText.get().gcTooManyUnpruned;
258 gcLog.write(message);
259 gcLog.commit();
260 }
261 return newPacks;
262 } catch (IOException | ParseException e) {
263 if (background) {
264 if (gcLog == null) {
265
266 return Collections.emptyList();
267 }
268 try {
269 gcLog.write(e.getMessage());
270 StringWriter sw = new StringWriter();
271 e.printStackTrace(new PrintWriter(sw));
272 gcLog.write(sw.toString());
273 gcLog.commit();
274 } catch (IOException e2) {
275 e2.addSuppressed(e);
276 LOG.error(e2.getMessage(), e2);
277 }
278 } else {
279 throw new JGitInternalException(e.getMessage(), e);
280 }
281 } finally {
282 if (gcLog != null) {
283 gcLog.unlock();
284 }
285 }
286 return Collections.emptyList();
287 };
288 Future<Collection<PackFile>> result = executor().submit(gcTask);
289 if (background) {
290
291 return Collections.emptyList();
292 }
293 try {
294 return result.get();
295 } catch (InterruptedException | ExecutionException e) {
296 throw new IOException(e);
297 }
298 }
299
300 private ExecutorService executor() {
301 return (executor != null) ? executor : WorkQueue.getExecutor();
302 }
303
304 private Collection<PackFile> doGc() throws IOException, ParseException {
305 if (automatic && !needGc()) {
306 return Collections.emptyList();
307 }
308 pm.start(6 );
309 packRefs();
310
311 Collection<PackFile> newPacks = repack();
312 prune(Collections.<ObjectId> emptySet());
313
314 return newPacks;
315 }
316
317
318
319
320
321
322
323
324
325
326
327 private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
328 throws IOException {
329 for (PackIndex.MutableEntry entry : pack) {
330 ObjectId oid = entry.toObjectId();
331 if (existing.contains(oid)) {
332 continue;
333 }
334 existing.add(oid);
335 ObjectLoader loader = reader.open(oid);
336 inserter.insert(loader.getType(),
337 loader.getSize(),
338 loader.openStream(),
339 true );
340 }
341 }
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359 private void deleteOldPacks(Collection<PackFile> oldPacks,
360 Collection<PackFile> newPacks) throws ParseException, IOException {
361 HashSet<ObjectId> ids = new HashSet<>();
362 for (PackFile pack : newPacks) {
363 for (PackIndex.MutableEntry entry : pack) {
364 ids.add(entry.toObjectId());
365 }
366 }
367 ObjectReader reader = repo.newObjectReader();
368 ObjectDirectory dir = repo.getObjectDatabase();
369 ObjectDirectoryInserter inserter = dir.newInserter();
370 boolean shouldLoosen = !"now".equals(getPruneExpireStr()) &&
371 getExpireDate() < Long.MAX_VALUE;
372
373 prunePreserved();
374 long packExpireDate = getPackExpireDate();
375 oldPackLoop: for (PackFile oldPack : oldPacks) {
376 checkCancelled();
377 String oldName = oldPack.getPackName();
378
379
380 for (PackFile newPack : newPacks)
381 if (oldName.equals(newPack.getPackName()))
382 continue oldPackLoop;
383
384 if (!oldPack.shouldBeKept()
385 && repo.getFS().lastModified(
386 oldPack.getPackFile()) < packExpireDate) {
387 oldPack.close();
388 if (shouldLoosen) {
389 loosen(inserter, reader, oldPack, ids);
390 }
391 prunePack(oldName);
392 }
393 }
394
395
396
397 repo.getObjectDatabase().close();
398 }
399
400
401
402
403
404
405
406
407
408
409
410 private void removeOldPack(File packFile, String packName, PackExt ext,
411 int deleteOptions) throws IOException {
412 if (pconfig != null && pconfig.isPreserveOldPacks()) {
413 File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
414 FileUtils.mkdir(oldPackDir, true);
415
416 String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();
417 File oldPackFile = new File(oldPackDir, oldPackName);
418 FileUtils.rename(packFile, oldPackFile);
419 } else {
420 FileUtils.delete(packFile, deleteOptions);
421 }
422 }
423
424
425
426
427 private void prunePreserved() {
428 if (pconfig != null && pconfig.isPrunePreserved()) {
429 try {
430 FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
431 FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
432 } catch (IOException e) {
433
434 }
435 }
436 }
437
438
439
440
441
442
443
444
445
446
447
448 private void prunePack(String packName) {
449 PackExt[] extensions = PackExt.values();
450 try {
451
452
453 int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
454 for (PackExt ext : extensions)
455 if (PackExt.PACK.equals(ext)) {
456 File f = nameFor(packName, "." + ext.getExtension());
457 removeOldPack(f, packName, ext, deleteOptions);
458 break;
459 }
460
461
462 deleteOptions |= FileUtils.IGNORE_ERRORS;
463 for (PackExt ext : extensions) {
464 if (!PackExt.PACK.equals(ext)) {
465 File f = nameFor(packName, "." + ext.getExtension());
466 removeOldPack(f, packName, ext, deleteOptions);
467 }
468 }
469 } catch (IOException e) {
470
471 }
472 }
473
474
475
476
477
478
479
480
481 public void prunePacked() throws IOException {
482 ObjectDirectory objdb = repo.getObjectDatabase();
483 Collection<PackFile> packs = objdb.getPacks();
484 File objects = repo.getObjectsDirectory();
485 String[] fanout = objects.list();
486
487 if (fanout != null && fanout.length > 0) {
488 pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
489 try {
490 for (String d : fanout) {
491 checkCancelled();
492 pm.update(1);
493 if (d.length() != 2)
494 continue;
495 String[] entries = new File(objects, d).list();
496 if (entries == null)
497 continue;
498 for (String e : entries) {
499 checkCancelled();
500 if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
501 continue;
502 ObjectId id;
503 try {
504 id = ObjectId.fromString(d + e);
505 } catch (IllegalArgumentException notAnObject) {
506
507
508 continue;
509 }
510 boolean found = false;
511 for (PackFile p : packs) {
512 checkCancelled();
513 if (p.hasObject(id)) {
514 found = true;
515 break;
516 }
517 }
518 if (found)
519 FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
520 | FileUtils.SKIP_MISSING
521 | FileUtils.IGNORE_ERRORS);
522 }
523 }
524 } finally {
525 pm.endTask();
526 }
527 }
528 }
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543 public void prune(Set<ObjectId> objectsToKeep) throws IOException,
544 ParseException {
545 long expireDate = getExpireDate();
546
547
548
549 Map<ObjectId, File> deletionCandidates = new HashMap<>();
550 Set<ObjectId> indexObjects = null;
551 File objects = repo.getObjectsDirectory();
552 String[] fanout = objects.list();
553 if (fanout == null || fanout.length == 0) {
554 return;
555 }
556 pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
557 fanout.length);
558 try {
559 for (String d : fanout) {
560 checkCancelled();
561 pm.update(1);
562 if (d.length() != 2)
563 continue;
564 File[] entries = new File(objects, d).listFiles();
565 if (entries == null)
566 continue;
567 for (File f : entries) {
568 checkCancelled();
569 String fName = f.getName();
570 if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
571 continue;
572 if (repo.getFS().lastModified(f) >= expireDate)
573 continue;
574 try {
575 ObjectId id = ObjectId.fromString(d + fName);
576 if (objectsToKeep.contains(id))
577 continue;
578 if (indexObjects == null)
579 indexObjects = listNonHEADIndexObjects();
580 if (indexObjects.contains(id))
581 continue;
582 deletionCandidates.put(id, f);
583 } catch (IllegalArgumentException notAnObject) {
584
585
586 continue;
587 }
588 }
589 }
590 } finally {
591 pm.endTask();
592 }
593
594 if (deletionCandidates.isEmpty()) {
595 return;
596 }
597
598 checkCancelled();
599
600
601
602
603
604 Collection<Ref> newRefs;
605 if (lastPackedRefs == null || lastPackedRefs.isEmpty())
606 newRefs = getAllRefs();
607 else {
608 Map<String, Ref> last = new HashMap<>();
609 for (Ref r : lastPackedRefs) {
610 last.put(r.getName(), r);
611 }
612 newRefs = new ArrayList<>();
613 for (Ref r : getAllRefs()) {
614 Ref old = last.get(r.getName());
615 if (!equals(r, old)) {
616 newRefs.add(r);
617 }
618 }
619 }
620
621 if (!newRefs.isEmpty()) {
622
623
624
625
626
627 ObjectWalk w = new ObjectWalk(repo);
628 try {
629 for (Ref cr : newRefs) {
630 checkCancelled();
631 w.markStart(w.parseAny(cr.getObjectId()));
632 }
633 if (lastPackedRefs != null)
634 for (Ref lpr : lastPackedRefs) {
635 w.markUninteresting(w.parseAny(lpr.getObjectId()));
636 }
637 removeReferenced(deletionCandidates, w);
638 } finally {
639 w.dispose();
640 }
641 }
642
643 if (deletionCandidates.isEmpty())
644 return;
645
646
647
648
649
650
651 ObjectWalk w = new ObjectWalk(repo);
652 try {
653 for (Ref ar : getAllRefs())
654 for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
655 checkCancelled();
656 w.markStart(w.parseAny(id));
657 }
658 if (lastPackedRefs != null)
659 for (Ref lpr : lastPackedRefs) {
660 checkCancelled();
661 w.markUninteresting(w.parseAny(lpr.getObjectId()));
662 }
663 removeReferenced(deletionCandidates, w);
664 } finally {
665 w.dispose();
666 }
667
668 if (deletionCandidates.isEmpty())
669 return;
670
671 checkCancelled();
672
673
674
675
676
677 Set<File> touchedFanout = new HashSet<>();
678 for (File f : deletionCandidates.values()) {
679 if (f.lastModified() < expireDate) {
680 f.delete();
681 touchedFanout.add(f.getParentFile());
682 }
683 }
684
685 for (File f : touchedFanout) {
686 FileUtils.delete(f,
687 FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
688 }
689
690 repo.getObjectDatabase().close();
691 }
692
693 private long getExpireDate() throws ParseException {
694 long expireDate = Long.MAX_VALUE;
695
696 if (expire == null && expireAgeMillis == -1) {
697 String pruneExpireStr = getPruneExpireStr();
698 if (pruneExpireStr == null)
699 pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
700 expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
701 .getInstance().getLocale());
702 expireAgeMillis = -1;
703 }
704 if (expire != null)
705 expireDate = expire.getTime();
706 if (expireAgeMillis != -1)
707 expireDate = System.currentTimeMillis() - expireAgeMillis;
708 return expireDate;
709 }
710
711 private String getPruneExpireStr() {
712 return repo.getConfig().getString(
713 ConfigConstants.CONFIG_GC_SECTION, null,
714 ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
715 }
716
717 private long getPackExpireDate() throws ParseException {
718 long packExpireDate = Long.MAX_VALUE;
719
720 if (packExpire == null && packExpireAgeMillis == -1) {
721 String prunePackExpireStr = repo.getConfig().getString(
722 ConfigConstants.CONFIG_GC_SECTION, null,
723 ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
724 if (prunePackExpireStr == null)
725 prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
726 packExpire = GitDateParser.parse(prunePackExpireStr, null,
727 SystemReader.getInstance().getLocale());
728 packExpireAgeMillis = -1;
729 }
730 if (packExpire != null)
731 packExpireDate = packExpire.getTime();
732 if (packExpireAgeMillis != -1)
733 packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
734 return packExpireDate;
735 }
736
737
738
739
740
741
742
743
744
745
746
747 private void removeReferenced(Map<ObjectId, File> id2File,
748 ObjectWalk w) throws MissingObjectException,
749 IncorrectObjectTypeException, IOException {
750 RevObject ro = w.next();
751 while (ro != null) {
752 checkCancelled();
753 if (id2File.remove(ro.getId()) != null)
754 if (id2File.isEmpty())
755 return;
756 ro = w.next();
757 }
758 ro = w.nextObject();
759 while (ro != null) {
760 checkCancelled();
761 if (id2File.remove(ro.getId()) != null)
762 if (id2File.isEmpty())
763 return;
764 ro = w.nextObject();
765 }
766 }
767
768 private static boolean equals(Ref r1, Ref r2) {
769 if (r1 == null || r2 == null)
770 return false;
771 if (r1.isSymbolic()) {
772 if (!r2.isSymbolic())
773 return false;
774 return r1.getTarget().getName().equals(r2.getTarget().getName());
775 } else {
776 if (r2.isSymbolic()) {
777 return false;
778 }
779 return Objects.equals(r1.getObjectId(), r2.getObjectId());
780 }
781 }
782
783
784
785
786
787
788 public void packRefs() throws IOException {
789 Collection<Ref> refs = repo.getRefDatabase().getRefs(Constants.R_REFS).values();
790 List<String> refsToBePacked = new ArrayList<>(refs.size());
791 pm.beginTask(JGitText.get().packRefs, refs.size());
792 try {
793 for (Ref ref : refs) {
794 checkCancelled();
795 if (!ref.isSymbolic() && ref.getStorage().isLoose())
796 refsToBePacked.add(ref.getName());
797 pm.update(1);
798 }
799 ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
800 } finally {
801 pm.endTask();
802 }
803 }
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819 public Collection<PackFile> repack() throws IOException {
820 Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
821
822 long time = System.currentTimeMillis();
823 Collection<Ref> refsBefore = getAllRefs();
824
825 Set<ObjectId> allHeadsAndTags = new HashSet<>();
826 Set<ObjectId> allHeads = new HashSet<>();
827 Set<ObjectId> allTags = new HashSet<>();
828 Set<ObjectId> nonHeads = new HashSet<>();
829 Set<ObjectId> txnHeads = new HashSet<>();
830 Set<ObjectId> tagTargets = new HashSet<>();
831 Set<ObjectId> indexObjects = listNonHEADIndexObjects();
832 RefDatabase refdb = repo.getRefDatabase();
833
834 for (Ref ref : refsBefore) {
835 checkCancelled();
836 nonHeads.addAll(listRefLogObjects(ref, 0));
837 if (ref.isSymbolic() || ref.getObjectId() == null) {
838 continue;
839 }
840 if (isHead(ref)) {
841 allHeads.add(ref.getObjectId());
842 } else if (isTag(ref)) {
843 allTags.add(ref.getObjectId());
844 } else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
845 txnHeads.add(ref.getObjectId());
846 } else {
847 nonHeads.add(ref.getObjectId());
848 }
849 if (ref.getPeeledObjectId() != null) {
850 tagTargets.add(ref.getPeeledObjectId());
851 }
852 }
853
854 List<ObjectIdSet> excluded = new LinkedList<>();
855 for (final PackFile f : repo.getObjectDatabase().getPacks()) {
856 checkCancelled();
857 if (f.shouldBeKept())
858 excluded.add(f.getIndex());
859 }
860
861
862 allTags.removeAll(allHeads);
863 allHeadsAndTags.addAll(allHeads);
864 allHeadsAndTags.addAll(allTags);
865
866
867 tagTargets.addAll(allHeadsAndTags);
868 nonHeads.addAll(indexObjects);
869
870
871 if (pconfig != null && pconfig.getSinglePack()) {
872 allHeadsAndTags.addAll(nonHeads);
873 nonHeads.clear();
874 }
875
876 List<PackFile> ret = new ArrayList<>(2);
877 PackFile heads = null;
878 if (!allHeadsAndTags.isEmpty()) {
879 heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
880 tagTargets, excluded);
881 if (heads != null) {
882 ret.add(heads);
883 excluded.add(0, heads.getIndex());
884 }
885 }
886 if (!nonHeads.isEmpty()) {
887 PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
888 tagTargets, excluded);
889 if (rest != null)
890 ret.add(rest);
891 }
892 if (!txnHeads.isEmpty()) {
893 PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
894 null, excluded);
895 if (txn != null)
896 ret.add(txn);
897 }
898 try {
899 deleteOldPacks(toBeDeleted, ret);
900 } catch (ParseException e) {
901
902
903
904 throw new IOException(e);
905 }
906 prunePacked();
907 deleteOrphans();
908 deleteTempPacksIdx();
909
910 lastPackedRefs = refsBefore;
911 lastRepackTime = time;
912 return ret;
913 }
914
915 private static boolean isHead(Ref ref) {
916 return ref.getName().startsWith(Constants.R_HEADS);
917 }
918
919 private static boolean isTag(Ref ref) {
920 return ref.getName().startsWith(Constants.R_TAGS);
921 }
922
923
924
925
926
927
928
929
930 private void deleteOrphans() {
931 Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
932 "pack");
933 List<String> fileNames = null;
934 try (Stream<Path> files = Files.list(packDir)) {
935 fileNames = files.map(path -> path.getFileName().toString())
936 .filter(name -> {
937 return (name.endsWith(PACK_EXT)
938 || name.endsWith(BITMAP_EXT)
939 || name.endsWith(INDEX_EXT));
940 }).sorted(Collections.reverseOrder())
941 .collect(Collectors.toList());
942 } catch (IOException e1) {
943
944 }
945 if (fileNames == null) {
946 return;
947 }
948
949 String base = null;
950 for (String n : fileNames) {
951 if (n.endsWith(PACK_EXT)) {
952 base = n.substring(0, n.lastIndexOf('.'));
953 } else {
954 if (base == null || !n.startsWith(base)) {
955 try {
956 Files.delete(new File(packDir.toFile(), n).toPath());
957 } catch (IOException e) {
958 LOG.error(e.getMessage(), e);
959 }
960 }
961 }
962 }
963 }
964
965 private void deleteTempPacksIdx() {
966 Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
967 "pack");
968 Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
969 try {
970 Files.newDirectoryStream(packDir, "gc_*_tmp")
971 .forEach(t -> {
972 try {
973 Instant lastModified = Files.getLastModifiedTime(t)
974 .toInstant();
975 if (lastModified.isBefore(threshold)) {
976 Files.deleteIfExists(t);
977 }
978 } catch (IOException e) {
979 LOG.error(e.getMessage(), e);
980 }
981 });
982 } catch (IOException e) {
983 LOG.error(e.getMessage(), e);
984 }
985 }
986
987
988
989
990
991
992
993
994 private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
995 ReflogReader reflogReader = repo.getReflogReader(ref.getName());
996 if (reflogReader == null) {
997 return Collections.emptySet();
998 }
999 List<ReflogEntry> rlEntries = reflogReader
1000 .getReverseEntries();
1001 if (rlEntries == null || rlEntries.isEmpty())
1002 return Collections.<ObjectId> emptySet();
1003 Set<ObjectId> ret = new HashSet<>();
1004 for (ReflogEntry e : rlEntries) {
1005 if (e.getWho().getWhen().getTime() < minTime)
1006 break;
1007 ObjectId newId = e.getNewId();
1008 if (newId != null && !ObjectId.zeroId().equals(newId))
1009 ret.add(newId);
1010 ObjectId oldId = e.getOldId();
1011 if (oldId != null && !ObjectId.zeroId().equals(oldId))
1012 ret.add(oldId);
1013 }
1014 return ret;
1015 }
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028 private Collection<Ref> getAllRefs() throws IOException {
1029 RefDatabase refdb = repo.getRefDatabase();
1030 Collection<Ref> refs = refdb.getRefs(RefDatabase.ALL).values();
1031 List<Ref> addl = refdb.getAdditionalRefs();
1032 if (!addl.isEmpty()) {
1033 List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1034 all.addAll(refs);
1035
1036 for (Ref r : addl) {
1037 checkCancelled();
1038 if (r.getName().startsWith(Constants.R_REFS)) {
1039 all.add(r);
1040 }
1041 }
1042 return all;
1043 }
1044 return refs;
1045 }
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056 private Set<ObjectId> listNonHEADIndexObjects()
1057 throws CorruptObjectException, IOException {
1058 if (repo.isBare()) {
1059 return Collections.emptySet();
1060 }
1061 try (TreeWalk treeWalk = new TreeWalk(repo)) {
1062 treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1063 ObjectId headID = repo.resolve(Constants.HEAD);
1064 if (headID != null) {
1065 try (RevWalk revWalk = new RevWalk(repo)) {
1066 treeWalk.addTree(revWalk.parseTree(headID));
1067 }
1068 }
1069
1070 treeWalk.setFilter(TreeFilter.ANY_DIFF);
1071 treeWalk.setRecursive(true);
1072 Set<ObjectId> ret = new HashSet<>();
1073
1074 while (treeWalk.next()) {
1075 checkCancelled();
1076 ObjectId objectId = treeWalk.getObjectId(0);
1077 switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1078 case FileMode.TYPE_MISSING:
1079 case FileMode.TYPE_GITLINK:
1080 continue;
1081 case FileMode.TYPE_TREE:
1082 case FileMode.TYPE_FILE:
1083 case FileMode.TYPE_SYMLINK:
1084 ret.add(objectId);
1085 continue;
1086 default:
1087 throw new IOException(MessageFormat.format(
1088 JGitText.get().corruptObjectInvalidMode3,
1089 String.format("%o",
1090 Integer.valueOf(treeWalk.getRawMode(0))),
1091 (objectId == null) ? "null" : objectId.name(),
1092 treeWalk.getPathString(),
1093 repo.getIndexFile()));
1094 }
1095 }
1096 return ret;
1097 }
1098 }
1099
1100 private PackFile writePack(@NonNull Set<? extends ObjectId> want,
1101 @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1102 Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1103 throws IOException {
1104 checkCancelled();
1105 File tmpPack = null;
1106 Map<PackExt, File> tmpExts = new TreeMap<>(
1107 new Comparator<PackExt>() {
1108 @Override
1109 public int compare(PackExt o1, PackExt o2) {
1110
1111
1112
1113 if (o1 == o2)
1114 return 0;
1115 if (o1 == PackExt.INDEX)
1116 return 1;
1117 if (o2 == PackExt.INDEX)
1118 return -1;
1119 return Integer.signum(o1.hashCode() - o2.hashCode());
1120 }
1121
1122 });
1123 try (PackWriter pw = new PackWriter(
1124 (pconfig == null) ? new PackConfig(repo) : pconfig,
1125 repo.newObjectReader())) {
1126
1127 pw.setDeltaBaseAsOffset(true);
1128 pw.setReuseDeltaCommits(false);
1129 if (tagTargets != null) {
1130 pw.setTagTargets(tagTargets);
1131 }
1132 if (excludeObjects != null)
1133 for (ObjectIdSet idx : excludeObjects)
1134 pw.excludeObjects(idx);
1135 pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1136 if (pw.getObjectCount() == 0)
1137 return null;
1138 checkCancelled();
1139
1140
1141 String id = pw.computeName().getName();
1142 File packdir = new File(repo.getObjectsDirectory(), "pack");
1143 tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir);
1144 final String tmpBase = tmpPack.getName()
1145 .substring(0, tmpPack.getName().lastIndexOf('.'));
1146 File tmpIdx = new File(packdir, tmpBase + ".idx_tmp");
1147 tmpExts.put(INDEX, tmpIdx);
1148
1149 if (!tmpIdx.createNewFile())
1150 throw new IOException(MessageFormat.format(
1151 JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1152
1153
1154 FileOutputStream fos = new FileOutputStream(tmpPack);
1155 FileChannel channel = fos.getChannel();
1156 OutputStream channelStream = Channels.newOutputStream(channel);
1157 try {
1158 pw.writePack(pm, pm, channelStream);
1159 } finally {
1160 channel.force(true);
1161 channelStream.close();
1162 fos.close();
1163 }
1164
1165
1166 fos = new FileOutputStream(tmpIdx);
1167 FileChannel idxChannel = fos.getChannel();
1168 OutputStream idxStream = Channels.newOutputStream(idxChannel);
1169 try {
1170 pw.writeIndex(idxStream);
1171 } finally {
1172 idxChannel.force(true);
1173 idxStream.close();
1174 fos.close();
1175 }
1176
1177 if (pw.prepareBitmapIndex(pm)) {
1178 File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp");
1179 tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1180
1181 if (!tmpBitmapIdx.createNewFile())
1182 throw new IOException(MessageFormat.format(
1183 JGitText.get().cannotCreateIndexfile,
1184 tmpBitmapIdx.getPath()));
1185
1186 fos = new FileOutputStream(tmpBitmapIdx);
1187 idxChannel = fos.getChannel();
1188 idxStream = Channels.newOutputStream(idxChannel);
1189 try {
1190 pw.writeBitmapIndex(idxStream);
1191 } finally {
1192 idxChannel.force(true);
1193 idxStream.close();
1194 fos.close();
1195 }
1196 }
1197
1198
1199 File realPack = nameFor(id, ".pack");
1200
1201 repo.getObjectDatabase().closeAllPackHandles(realPack);
1202 tmpPack.setReadOnly();
1203
1204 FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1205 for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1206 File tmpExt = tmpEntry.getValue();
1207 tmpExt.setReadOnly();
1208
1209 File realExt = nameFor(id,
1210 "." + tmpEntry.getKey().getExtension());
1211 try {
1212 FileUtils.rename(tmpExt, realExt,
1213 StandardCopyOption.ATOMIC_MOVE);
1214 } catch (IOException e) {
1215 File newExt = new File(realExt.getParentFile(),
1216 realExt.getName() + ".new");
1217 try {
1218 FileUtils.rename(tmpExt, newExt,
1219 StandardCopyOption.ATOMIC_MOVE);
1220 } catch (IOException e2) {
1221 newExt = tmpExt;
1222 e = e2;
1223 }
1224 throw new IOException(MessageFormat.format(
1225 JGitText.get().panicCantRenameIndexFile, newExt,
1226 realExt), e);
1227 }
1228 }
1229
1230 return repo.getObjectDatabase().openPack(realPack);
1231 } finally {
1232 if (tmpPack != null && tmpPack.exists())
1233 tmpPack.delete();
1234 for (File tmpExt : tmpExts.values()) {
1235 if (tmpExt.exists())
1236 tmpExt.delete();
1237 }
1238 }
1239 }
1240
1241 private File nameFor(String name, String ext) {
1242 File packdir = new File(repo.getObjectsDirectory(), "pack");
1243 return new File(packdir, "pack-" + name + ext);
1244 }
1245
1246 private void checkCancelled() throws CancelledException {
1247 if (pm.isCancelled()) {
1248 throw new CancelledException(JGitText.get().operationCanceled);
1249 }
1250 }
1251
1252
1253
1254
1255
1256 public static class RepoStatistics {
1257
1258
1259
1260
1261
1262 public long numberOfPackedObjects;
1263
1264
1265
1266
1267 public long numberOfPackFiles;
1268
1269
1270
1271
1272 public long numberOfLooseObjects;
1273
1274
1275
1276
1277 public long sizeOfLooseObjects;
1278
1279
1280
1281
1282 public long sizeOfPackedObjects;
1283
1284
1285
1286
1287 public long numberOfLooseRefs;
1288
1289
1290
1291
1292 public long numberOfPackedRefs;
1293
1294
1295
1296
1297 public long numberOfBitmaps;
1298
1299 @Override
1300 public String toString() {
1301 final StringBuilder b = new StringBuilder();
1302 b.append("numberOfPackedObjects=").append(numberOfPackedObjects);
1303 b.append(", numberOfPackFiles=").append(numberOfPackFiles);
1304 b.append(", numberOfLooseObjects=").append(numberOfLooseObjects);
1305 b.append(", numberOfLooseRefs=").append(numberOfLooseRefs);
1306 b.append(", numberOfPackedRefs=").append(numberOfPackedRefs);
1307 b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects);
1308 b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects);
1309 b.append(", numberOfBitmaps=").append(numberOfBitmaps);
1310 return b.toString();
1311 }
1312 }
1313
1314
1315
1316
1317
1318
1319
1320 public RepoStatistics getStatistics() throws IOException {
1321 RepoStatistics ret = new RepoStatistics();
1322 Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
1323 for (PackFile f : packs) {
1324 ret.numberOfPackedObjects += f.getIndex().getObjectCount();
1325 ret.numberOfPackFiles++;
1326 ret.sizeOfPackedObjects += f.getPackFile().length();
1327 if (f.getBitmapIndex() != null)
1328 ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
1329 }
1330 File objDir = repo.getObjectsDirectory();
1331 String[] fanout = objDir.list();
1332 if (fanout != null && fanout.length > 0) {
1333 for (String d : fanout) {
1334 if (d.length() != 2)
1335 continue;
1336 File[] entries = new File(objDir, d).listFiles();
1337 if (entries == null)
1338 continue;
1339 for (File f : entries) {
1340 if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1341 continue;
1342 ret.numberOfLooseObjects++;
1343 ret.sizeOfLooseObjects += f.length();
1344 }
1345 }
1346 }
1347
1348 RefDatabase refDb = repo.getRefDatabase();
1349 for (Ref r : refDb.getRefs(RefDatabase.ALL).values()) {
1350 Storage storage = r.getStorage();
1351 if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1352 ret.numberOfLooseRefs++;
1353 if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1354 ret.numberOfPackedRefs++;
1355 }
1356
1357 return ret;
1358 }
1359
1360
1361
1362
1363
1364
1365
1366 public GC setProgressMonitor(ProgressMonitor pm) {
1367 this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1368 return this;
1369 }
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380 public void setExpireAgeMillis(long expireAgeMillis) {
1381 this.expireAgeMillis = expireAgeMillis;
1382 expire = null;
1383 }
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394 public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1395 this.packExpireAgeMillis = packExpireAgeMillis;
1396 expire = null;
1397 }
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407 public void setPackConfig(PackConfig pconfig) {
1408 this.pconfig = pconfig;
1409 }
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423 public void setExpire(Date expire) {
1424 this.expire = expire;
1425 expireAgeMillis = -1;
1426 }
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437 public void setPackExpire(Date packExpire) {
1438 this.packExpire = packExpire;
1439 packExpireAgeMillis = -1;
1440 }
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477 public void setAuto(boolean auto) {
1478 this.automatic = auto;
1479 }
1480
1481
1482
1483
1484
1485 void setBackground(boolean background) {
1486 this.background = background;
1487 }
1488
1489 private boolean needGc() {
1490 if (tooManyPacks()) {
1491 addRepackAllOption();
1492 } else if (!tooManyLooseObjects()) {
1493 return false;
1494 }
1495
1496 return true;
1497 }
1498
1499 private void addRepackAllOption() {
1500
1501
1502 }
1503
1504
1505
1506
1507 boolean tooManyPacks() {
1508 int autopacklimit = repo.getConfig().getInt(
1509 ConfigConstants.CONFIG_GC_SECTION,
1510 ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1511 DEFAULT_AUTOPACKLIMIT);
1512 if (autopacklimit <= 0) {
1513 return false;
1514 }
1515
1516
1517 return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1518 }
1519
1520
1521
1522
1523
1524
1525
1526 boolean tooManyLooseObjects() {
1527 int auto = getLooseObjectLimit();
1528 if (auto <= 0) {
1529 return false;
1530 }
1531 int n = 0;
1532 int threshold = (auto + 255) / 256;
1533 Path dir = repo.getObjectsDirectory().toPath().resolve("17");
1534 if (!Files.exists(dir)) {
1535 return false;
1536 }
1537 try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir,
1538 new DirectoryStream.Filter<Path>() {
1539
1540 @Override
1541 public boolean accept(Path file) throws IOException {
1542 Path fileName = file.getFileName();
1543 return Files.isRegularFile(file) && fileName != null
1544 && PATTERN_LOOSE_OBJECT
1545 .matcher(fileName.toString()).matches();
1546 }
1547 })) {
1548 for (Iterator<Path> iter = stream.iterator(); iter.hasNext();
1549 iter.next()) {
1550 if (++n > threshold) {
1551 return true;
1552 }
1553 }
1554 } catch (IOException e) {
1555 LOG.error(e.getMessage(), e);
1556 }
1557 return false;
1558 }
1559
1560 private int getLooseObjectLimit() {
1561 return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1562 ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1563 }
1564 }