View Javadoc
1   /*
2    * Copyright (C) 2012, Christian Halstrick <christian.halstrick@sap.com>
3    * Copyright (C) 2011, Shawn O. Pearce <spearce@spearce.org>
4    * and other copyright owners as documented in the project's IP log.
5    *
6    * This program and the accompanying materials are made available
7    * under the terms of the Eclipse Distribution License v1.0 which
8    * accompanies this distribution, is reproduced below, and is
9    * available at http://www.eclipse.org/org/documents/edl-v10.php
10   *
11   * All rights reserved.
12   *
13   * Redistribution and use in source and binary forms, with or
14   * without modification, are permitted provided that the following
15   * conditions are met:
16   *
17   * - Redistributions of source code must retain the above copyright
18   *   notice, this list of conditions and the following disclaimer.
19   *
20   * - Redistributions in binary form must reproduce the above
21   *   copyright notice, this list of conditions and the following
22   *   disclaimer in the documentation and/or other materials provided
23   *   with the distribution.
24   *
25   * - Neither the name of the Eclipse Foundation, Inc. nor the
26   *   names of its contributors may be used to endorse or promote
27   *   products derived from this software without specific prior
28   *   written permission.
29   *
30   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
31   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
32   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
33   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
35   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
38   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
39   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
40   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
41   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
42   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43   */
44  package org.eclipse.jgit.internal.storage.file;
45  
46  import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
47  import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
48  
49  import java.io.File;
50  import java.io.FileOutputStream;
51  import java.io.IOException;
52  import java.io.OutputStream;
53  import java.io.PrintWriter;
54  import java.io.StringWriter;
55  import java.nio.channels.Channels;
56  import java.nio.channels.FileChannel;
57  import java.nio.file.DirectoryStream;
58  import java.nio.file.Files;
59  import java.nio.file.Path;
60  import java.nio.file.Paths;
61  import java.nio.file.StandardCopyOption;
62  import java.text.MessageFormat;
63  import java.text.ParseException;
64  import java.time.Instant;
65  import java.time.temporal.ChronoUnit;
66  import java.util.ArrayList;
67  import java.util.Collection;
68  import java.util.Collections;
69  import java.util.Comparator;
70  import java.util.Date;
71  import java.util.HashMap;
72  import java.util.HashSet;
73  import java.util.Iterator;
74  import java.util.LinkedList;
75  import java.util.List;
76  import java.util.Map;
77  import java.util.Objects;
78  import java.util.Set;
79  import java.util.TreeMap;
80  import java.util.concurrent.Callable;
81  import java.util.concurrent.ExecutionException;
82  import java.util.concurrent.ExecutorService;
83  import java.util.concurrent.Future;
84  import java.util.regex.Pattern;
85  import java.util.stream.Collectors;
86  import java.util.stream.Stream;
87  
88  import org.eclipse.jgit.annotations.NonNull;
89  import org.eclipse.jgit.api.errors.JGitInternalException;
90  import org.eclipse.jgit.dircache.DirCacheIterator;
91  import org.eclipse.jgit.errors.CancelledException;
92  import org.eclipse.jgit.errors.CorruptObjectException;
93  import org.eclipse.jgit.errors.IncorrectObjectTypeException;
94  import org.eclipse.jgit.errors.MissingObjectException;
95  import org.eclipse.jgit.errors.NoWorkTreeException;
96  import org.eclipse.jgit.internal.JGitText;
97  import org.eclipse.jgit.internal.storage.pack.PackExt;
98  import org.eclipse.jgit.internal.storage.pack.PackWriter;
99  import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
100 import org.eclipse.jgit.lib.ConfigConstants;
101 import org.eclipse.jgit.lib.Constants;
102 import org.eclipse.jgit.lib.FileMode;
103 import org.eclipse.jgit.lib.NullProgressMonitor;
104 import org.eclipse.jgit.lib.ObjectId;
105 import org.eclipse.jgit.lib.ObjectIdSet;
106 import org.eclipse.jgit.lib.ObjectLoader;
107 import org.eclipse.jgit.lib.ObjectReader;
108 import org.eclipse.jgit.lib.ProgressMonitor;
109 import org.eclipse.jgit.lib.Ref;
110 import org.eclipse.jgit.lib.Ref.Storage;
111 import org.eclipse.jgit.lib.RefDatabase;
112 import org.eclipse.jgit.lib.ReflogEntry;
113 import org.eclipse.jgit.lib.ReflogReader;
114 import org.eclipse.jgit.lib.internal.WorkQueue;
115 import org.eclipse.jgit.revwalk.ObjectWalk;
116 import org.eclipse.jgit.revwalk.RevObject;
117 import org.eclipse.jgit.revwalk.RevWalk;
118 import org.eclipse.jgit.storage.pack.PackConfig;
119 import org.eclipse.jgit.treewalk.TreeWalk;
120 import org.eclipse.jgit.treewalk.filter.TreeFilter;
121 import org.eclipse.jgit.util.FileUtils;
122 import org.eclipse.jgit.util.GitDateParser;
123 import org.eclipse.jgit.util.SystemReader;
124 import org.slf4j.Logger;
125 import org.slf4j.LoggerFactory;
126 
127 /**
128  * A garbage collector for git {@link FileRepository}. Instances of this class
129  * are not thread-safe. Don't use the same instance from multiple threads.
130  *
131  * This class started as a copy of DfsGarbageCollector from Shawn O. Pearce
132  * adapted to FileRepositories.
133  */
134 public class GC {
135 	private final static Logger LOG = LoggerFactory
136 			.getLogger(GC.class);
137 
138 	private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago"; //$NON-NLS-1$
139 
140 	private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago"; //$NON-NLS-1$
141 
142 	private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
143 			.compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$
144 
145 	private static final String PACK_EXT = "." + PackExt.PACK.getExtension();//$NON-NLS-1$
146 
147 	private static final String BITMAP_EXT = "." //$NON-NLS-1$
148 			+ PackExt.BITMAP_INDEX.getExtension();
149 
150 	private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension(); //$NON-NLS-1$
151 
152 	private static final int DEFAULT_AUTOPACKLIMIT = 50;
153 
154 	private static final int DEFAULT_AUTOLIMIT = 6700;
155 
156 	private static volatile ExecutorService executor;
157 
158 	/**
159 	 * Set the executor for running auto-gc in the background. If no executor is
160 	 * set JGit's own WorkQueue will be used.
161 	 *
162 	 * @param e
163 	 *            the executor to be used for running auto-gc
164 	 * @since 4.8
165 	 */
166 	public static void setExecutor(ExecutorService e) {
167 		executor = e;
168 	}
169 
170 	private final FileRepository repo;
171 
172 	private ProgressMonitor pm;
173 
174 	private long expireAgeMillis = -1;
175 
176 	private Date expire;
177 
178 	private long packExpireAgeMillis = -1;
179 
180 	private Date packExpire;
181 
182 	private PackConfig pconfig = null;
183 
184 	/**
185 	 * the refs which existed during the last call to {@link #repack()}. This is
186 	 * needed during {@link #prune(Set)} where we can optimize by looking at the
187 	 * difference between the current refs and the refs which existed during
188 	 * last {@link #repack()}.
189 	 */
190 	private Collection<Ref> lastPackedRefs;
191 
192 	/**
193 	 * Holds the starting time of the last repack() execution. This is needed in
194 	 * prune() to inspect only those reflog entries which have been added since
195 	 * last repack().
196 	 */
197 	private long lastRepackTime;
198 
199 	/**
200 	 * Whether gc should do automatic housekeeping
201 	 */
202 	private boolean automatic;
203 
204 	/**
205 	 * Whether to run gc in a background thread
206 	 */
207 	private boolean background;
208 
209 	/**
210 	 * Creates a new garbage collector with default values. An expirationTime of
211 	 * two weeks and <code>null</code> as progress monitor will be used.
212 	 *
213 	 * @param repo
214 	 *            the repo to work on
215 	 */
216 	public GC(FileRepository repo) {
217 		this.repo = repo;
218 		this.pm = NullProgressMonitor.INSTANCE;
219 	}
220 
221 	/**
222 	 * Runs a garbage collector on a {@link FileRepository}. It will
223 	 * <ul>
224 	 * <li>pack loose references into packed-refs</li>
225 	 * <li>repack all reachable objects into new pack files and delete the old
226 	 * pack files</li>
227 	 * <li>prune all loose objects which are now reachable by packs</li>
228 	 * </ul>
229 	 *
230 	 * If {@link #setAuto(boolean)} was set to {@code true} {@code gc} will
231 	 * first check whether any housekeeping is required; if not, it exits
232 	 * without performing any work.
233 	 *
234 	 * If {@link #setBackground(boolean)} was set to {@code true}
235 	 * {@code collectGarbage} will start the gc in the background, and then
236 	 * return immediately. In this case, errors will not be reported except in
237 	 * gc.log.
238 	 *
239 	 * @return the collection of {@link PackFile}'s which are newly created
240 	 * @throws IOException
241 	 * @throws ParseException
242 	 *             If the configuration parameter "gc.pruneexpire" couldn't be
243 	 *             parsed
244 	 */
245 	// TODO(ms): in 5.0 change signature and return Future<Collection<PackFile>>
246 	public Collection<PackFile> gc() throws IOException, ParseException {
247 		final GcLog gcLog = background ? new GcLog(repo) : null;
248 		if (gcLog != null && !gcLog.lock(background)) {
249 			// there is already a background gc running
250 			return Collections.emptyList();
251 		}
252 
253 		Callable<Collection<PackFile>> gcTask = () -> {
254 			try {
255 				Collection<PackFile> newPacks = doGc();
256 				if (automatic && tooManyLooseObjects() && gcLog != null) {
257 					String message = JGitText.get().gcTooManyUnpruned;
258 					gcLog.write(message);
259 					gcLog.commit();
260 				}
261 				return newPacks;
262 			} catch (IOException | ParseException e) {
263 				if (background) {
264 					if (gcLog == null) {
265 						// Lacking a log, there's no way to report this.
266 						return Collections.emptyList();
267 					}
268 					try {
269 						gcLog.write(e.getMessage());
270 						StringWriter sw = new StringWriter();
271 						e.printStackTrace(new PrintWriter(sw));
272 						gcLog.write(sw.toString());
273 						gcLog.commit();
274 					} catch (IOException e2) {
275 						e2.addSuppressed(e);
276 						LOG.error(e2.getMessage(), e2);
277 					}
278 				} else {
279 					throw new JGitInternalException(e.getMessage(), e);
280 				}
281 			} finally {
282 				if (gcLog != null) {
283 					gcLog.unlock();
284 				}
285 			}
286 			return Collections.emptyList();
287 		};
288 		Future<Collection<PackFile>> result = executor().submit(gcTask);
289 		if (background) {
290 			// TODO(ms): in 5.0 change signature and return the Future
291 			return Collections.emptyList();
292 		}
293 		try {
294 			return result.get();
295 		} catch (InterruptedException | ExecutionException e) {
296 			throw new IOException(e);
297 		}
298 	}
299 
300 	private ExecutorService executor() {
301 		return (executor != null) ? executor : WorkQueue.getExecutor();
302 	}
303 
304 	private Collection<PackFile> doGc() throws IOException, ParseException {
305 		if (automatic && !needGc()) {
306 			return Collections.emptyList();
307 		}
308 		pm.start(6 /* tasks */);
309 		packRefs();
310 		// TODO: implement reflog_expire(pm, repo);
311 		Collection<PackFile> newPacks = repack();
312 		prune(Collections.<ObjectId> emptySet());
313 		// TODO: implement rerere_gc(pm);
314 		return newPacks;
315 	}
316 
317 	/**
318 	 * Loosen objects in a pack file which are not also in the newly-created
319 	 * pack files.
320 	 *
321 	 * @param inserter
322 	 * @param reader
323 	 * @param pack
324 	 * @param existing
325 	 * @throws IOException
326 	 */
327 	private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
328 			throws IOException {
329 		for (PackIndex.MutableEntry entry : pack) {
330 			ObjectId oid = entry.toObjectId();
331 			if (existing.contains(oid)) {
332 				continue;
333 			}
334 			existing.add(oid);
335 			ObjectLoader loader = reader.open(oid);
336 			inserter.insert(loader.getType(),
337 					loader.getSize(),
338 					loader.openStream(),
339 					true /* create this object even though it's a duplicate */);
340 		}
341 	}
342 
343 	/**
344 	 * Delete old pack files. What is 'old' is defined by specifying a set of
345 	 * old pack files and a set of new pack files. Each pack file contained in
346 	 * old pack files but not contained in new pack files will be deleted. If
347 	 * preserveOldPacks is set, keep a copy of the pack file in the preserve
348 	 * directory. If an expirationDate is set then pack files which are younger
349 	 * than the expirationDate will not be deleted nor preserved.
350 	 * <p>
351 	 * If we're not immediately expiring loose objects, loosen any objects
352 	 * in the old pack files which aren't in the new pack files.
353 	 *
354 	 * @param oldPacks
355 	 * @param newPacks
356 	 * @throws ParseException
357 	 * @throws IOException
358 	 */
359 	private void deleteOldPacks(Collection<PackFile> oldPacks,
360 			Collection<PackFile> newPacks) throws ParseException, IOException {
361 		HashSet<ObjectId> ids = new HashSet<>();
362 		for (PackFile pack : newPacks) {
363 			for (PackIndex.MutableEntry entry : pack) {
364 				ids.add(entry.toObjectId());
365 			}
366 		}
367 		ObjectReader reader = repo.newObjectReader();
368 		ObjectDirectory dir = repo.getObjectDatabase();
369 		ObjectDirectoryInserter inserter = dir.newInserter();
370 		boolean shouldLoosen = !"now".equals(getPruneExpireStr()) && //$NON-NLS-1$
371 			getExpireDate() < Long.MAX_VALUE;
372 
373 		prunePreserved();
374 		long packExpireDate = getPackExpireDate();
375 		oldPackLoop: for (PackFile oldPack : oldPacks) {
376 			checkCancelled();
377 			String oldName = oldPack.getPackName();
378 			// check whether an old pack file is also among the list of new
379 			// pack files. Then we must not delete it.
380 			for (PackFile newPack : newPacks)
381 				if (oldName.equals(newPack.getPackName()))
382 					continue oldPackLoop;
383 
384 			if (!oldPack.shouldBeKept()
385 					&& repo.getFS().lastModified(
386 							oldPack.getPackFile()) < packExpireDate) {
387 				oldPack.close();
388 				if (shouldLoosen) {
389 					loosen(inserter, reader, oldPack, ids);
390 				}
391 				prunePack(oldName);
392 			}
393 		}
394 
395 		// close the complete object database. That's my only chance to force
396 		// rescanning and to detect that certain pack files are now deleted.
397 		repo.getObjectDatabase().close();
398 	}
399 
400 	/**
401 	 * Deletes old pack file, unless 'preserve-oldpacks' is set, in which case it
402 	 * moves the pack file to the preserved directory
403 	 *
404 	 * @param packFile
405 	 * @param packName
406 	 * @param ext
407 	 * @param deleteOptions
408 	 * @throws IOException
409 	 */
410 	private void removeOldPack(File packFile, String packName, PackExt ext,
411 			int deleteOptions) throws IOException {
412 		if (pconfig != null && pconfig.isPreserveOldPacks()) {
413 			File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
414 			FileUtils.mkdir(oldPackDir, true);
415 
416 			String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();  //$NON-NLS-1$ //$NON-NLS-2$
417 			File oldPackFile = new File(oldPackDir, oldPackName);
418 			FileUtils.rename(packFile, oldPackFile);
419 		} else {
420 			FileUtils.delete(packFile, deleteOptions);
421 		}
422 	}
423 
424 	/**
425 	 * Delete the preserved directory including all pack files within
426 	 */
427 	private void prunePreserved() {
428 		if (pconfig != null && pconfig.isPrunePreserved()) {
429 			try {
430 				FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
431 						FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
432 			} catch (IOException e) {
433 				// Deletion of the preserved pack files failed. Silently return.
434 			}
435 		}
436 	}
437 
438 	/**
439 	 * Delete files associated with a single pack file. First try to delete the
440 	 * ".pack" file because on some platforms the ".pack" file may be locked and
441 	 * can't be deleted. In such a case it is better to detect this early and
442 	 * give up on deleting files for this packfile. Otherwise we may delete the
443 	 * ".index" file and when failing to delete the ".pack" file we are left
444 	 * with a ".pack" file without a ".index" file.
445 	 *
446 	 * @param packName
447 	 */
448 	private void prunePack(String packName) {
449 		PackExt[] extensions = PackExt.values();
450 		try {
451 			// Delete the .pack file first and if this fails give up on deleting
452 			// the other files
453 			int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
454 			for (PackExt ext : extensions)
455 				if (PackExt.PACK.equals(ext)) {
456 					File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$
457 					removeOldPack(f, packName, ext, deleteOptions);
458 					break;
459 				}
460 			// The .pack file has been deleted. Delete as many as the other
461 			// files as you can.
462 			deleteOptions |= FileUtils.IGNORE_ERRORS;
463 			for (PackExt ext : extensions) {
464 				if (!PackExt.PACK.equals(ext)) {
465 					File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$
466 					removeOldPack(f, packName, ext, deleteOptions);
467 				}
468 			}
469 		} catch (IOException e) {
470 			// Deletion of the .pack file failed. Silently return.
471 		}
472 	}
473 
474 	/**
475 	 * Like "git prune-packed" this method tries to prune all loose objects
476 	 * which can be found in packs. If certain objects can't be pruned (e.g.
477 	 * because the filesystem delete operation fails) this is silently ignored.
478 	 *
479 	 * @throws IOException
480 	 */
481 	public void prunePacked() throws IOException {
482 		ObjectDirectory objdb = repo.getObjectDatabase();
483 		Collection<PackFile> packs = objdb.getPacks();
484 		File objects = repo.getObjectsDirectory();
485 		String[] fanout = objects.list();
486 
487 		if (fanout != null && fanout.length > 0) {
488 			pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
489 			try {
490 				for (String d : fanout) {
491 					checkCancelled();
492 					pm.update(1);
493 					if (d.length() != 2)
494 						continue;
495 					String[] entries = new File(objects, d).list();
496 					if (entries == null)
497 						continue;
498 					for (String e : entries) {
499 						checkCancelled();
500 						if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
501 							continue;
502 						ObjectId id;
503 						try {
504 							id = ObjectId.fromString(d + e);
505 						} catch (IllegalArgumentException notAnObject) {
506 							// ignoring the file that does not represent loose
507 							// object
508 							continue;
509 						}
510 						boolean found = false;
511 						for (PackFile p : packs) {
512 							checkCancelled();
513 							if (p.hasObject(id)) {
514 								found = true;
515 								break;
516 							}
517 						}
518 						if (found)
519 							FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
520 									| FileUtils.SKIP_MISSING
521 									| FileUtils.IGNORE_ERRORS);
522 					}
523 				}
524 			} finally {
525 				pm.endTask();
526 			}
527 		}
528 	}
529 
530 	/**
531 	 * Like "git prune" this method tries to prune all loose objects which are
532 	 * unreferenced. If certain objects can't be pruned (e.g. because the
533 	 * filesystem delete operation fails) this is silently ignored.
534 	 *
535 	 * @param objectsToKeep
536 	 *            a set of objects which should explicitly not be pruned
537 	 *
538 	 * @throws IOException
539 	 * @throws ParseException
540 	 *             If the configuration parameter "gc.pruneexpire" couldn't be
541 	 *             parsed
542 	 */
543 	public void prune(Set<ObjectId> objectsToKeep) throws IOException,
544 			ParseException {
545 		long expireDate = getExpireDate();
546 
547 		// Collect all loose objects which are old enough, not referenced from
548 		// the index and not in objectsToKeep
549 		Map<ObjectId, File> deletionCandidates = new HashMap<>();
550 		Set<ObjectId> indexObjects = null;
551 		File objects = repo.getObjectsDirectory();
552 		String[] fanout = objects.list();
553 		if (fanout == null || fanout.length == 0) {
554 			return;
555 		}
556 		pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
557 				fanout.length);
558 		try {
559 			for (String d : fanout) {
560 				checkCancelled();
561 				pm.update(1);
562 				if (d.length() != 2)
563 					continue;
564 				File[] entries = new File(objects, d).listFiles();
565 				if (entries == null)
566 					continue;
567 				for (File f : entries) {
568 					checkCancelled();
569 					String fName = f.getName();
570 					if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
571 						continue;
572 					if (repo.getFS().lastModified(f) >= expireDate)
573 						continue;
574 					try {
575 						ObjectId id = ObjectId.fromString(d + fName);
576 						if (objectsToKeep.contains(id))
577 							continue;
578 						if (indexObjects == null)
579 							indexObjects = listNonHEADIndexObjects();
580 						if (indexObjects.contains(id))
581 							continue;
582 						deletionCandidates.put(id, f);
583 					} catch (IllegalArgumentException notAnObject) {
584 						// ignoring the file that does not represent loose
585 						// object
586 						continue;
587 					}
588 				}
589 			}
590 		} finally {
591 			pm.endTask();
592 		}
593 
594 		if (deletionCandidates.isEmpty()) {
595 			return;
596 		}
597 
598 		checkCancelled();
599 
600 		// From the set of current refs remove all those which have been handled
601 		// during last repack(). Only those refs will survive which have been
602 		// added or modified since the last repack. Only these can save existing
603 		// loose refs from being pruned.
604 		Collection<Ref> newRefs;
605 		if (lastPackedRefs == null || lastPackedRefs.isEmpty())
606 			newRefs = getAllRefs();
607 		else {
608 			Map<String, Ref> last = new HashMap<>();
609 			for (Ref r : lastPackedRefs) {
610 				last.put(r.getName(), r);
611 			}
612 			newRefs = new ArrayList<>();
613 			for (Ref r : getAllRefs()) {
614 				Ref old = last.get(r.getName());
615 				if (!equals(r, old)) {
616 					newRefs.add(r);
617 				}
618 			}
619 		}
620 
621 		if (!newRefs.isEmpty()) {
622 			// There are new/modified refs! Check which loose objects are now
623 			// referenced by these modified refs (or their reflogentries).
624 			// Remove these loose objects
625 			// from the deletionCandidates. When the last candidate is removed
626 			// leave this method.
627 			ObjectWalk w = new ObjectWalk(repo);
628 			try {
629 				for (Ref cr : newRefs) {
630 					checkCancelled();
631 					w.markStart(w.parseAny(cr.getObjectId()));
632 				}
633 				if (lastPackedRefs != null)
634 					for (Ref lpr : lastPackedRefs) {
635 						w.markUninteresting(w.parseAny(lpr.getObjectId()));
636 					}
637 				removeReferenced(deletionCandidates, w);
638 			} finally {
639 				w.dispose();
640 			}
641 		}
642 
643 		if (deletionCandidates.isEmpty())
644 			return;
645 
646 		// Since we have not left the method yet there are still
647 		// deletionCandidates. Last chance for these objects not to be pruned is
648 		// that they are referenced by reflog entries. Even refs which currently
649 		// point to the same object as during last repack() may have
650 		// additional reflog entries not handled during last repack()
651 		ObjectWalk w = new ObjectWalk(repo);
652 		try {
653 			for (Ref ar : getAllRefs())
654 				for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
655 					checkCancelled();
656 					w.markStart(w.parseAny(id));
657 				}
658 			if (lastPackedRefs != null)
659 				for (Ref lpr : lastPackedRefs) {
660 					checkCancelled();
661 					w.markUninteresting(w.parseAny(lpr.getObjectId()));
662 				}
663 			removeReferenced(deletionCandidates, w);
664 		} finally {
665 			w.dispose();
666 		}
667 
668 		if (deletionCandidates.isEmpty())
669 			return;
670 
671 		checkCancelled();
672 
673 		// delete all candidates which have survived: these are unreferenced
674 		// loose objects. Make a last check, though, to avoid deleting objects
675 		// that could have been referenced while the candidates list was being
676 		// built (by an incoming push, for example).
677 		Set<File> touchedFanout = new HashSet<>();
678 		for (File f : deletionCandidates.values()) {
679 			if (f.lastModified() < expireDate) {
680 				f.delete();
681 				touchedFanout.add(f.getParentFile());
682 			}
683 		}
684 
685 		for (File f : touchedFanout) {
686 			FileUtils.delete(f,
687 					FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
688 		}
689 
690 		repo.getObjectDatabase().close();
691 	}
692 
693 	private long getExpireDate() throws ParseException {
694 		long expireDate = Long.MAX_VALUE;
695 
696 		if (expire == null && expireAgeMillis == -1) {
697 			String pruneExpireStr = getPruneExpireStr();
698 			if (pruneExpireStr == null)
699 				pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
700 			expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
701 					.getInstance().getLocale());
702 			expireAgeMillis = -1;
703 		}
704 		if (expire != null)
705 			expireDate = expire.getTime();
706 		if (expireAgeMillis != -1)
707 			expireDate = System.currentTimeMillis() - expireAgeMillis;
708 		return expireDate;
709 	}
710 
711 	private String getPruneExpireStr() {
712 		return repo.getConfig().getString(
713                         ConfigConstants.CONFIG_GC_SECTION, null,
714                         ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
715 	}
716 
717 	private long getPackExpireDate() throws ParseException {
718 		long packExpireDate = Long.MAX_VALUE;
719 
720 		if (packExpire == null && packExpireAgeMillis == -1) {
721 			String prunePackExpireStr = repo.getConfig().getString(
722 					ConfigConstants.CONFIG_GC_SECTION, null,
723 					ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
724 			if (prunePackExpireStr == null)
725 				prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
726 			packExpire = GitDateParser.parse(prunePackExpireStr, null,
727 					SystemReader.getInstance().getLocale());
728 			packExpireAgeMillis = -1;
729 		}
730 		if (packExpire != null)
731 			packExpireDate = packExpire.getTime();
732 		if (packExpireAgeMillis != -1)
733 			packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
734 		return packExpireDate;
735 	}
736 
737 	/**
738 	 * Remove all entries from a map which key is the id of an object referenced
739 	 * by the given ObjectWalk
740 	 *
741 	 * @param id2File
742 	 * @param w
743 	 * @throws MissingObjectException
744 	 * @throws IncorrectObjectTypeException
745 	 * @throws IOException
746 	 */
747 	private void removeReferenced(Map<ObjectId, File> id2File,
748 			ObjectWalk w) throws MissingObjectException,
749 			IncorrectObjectTypeException, IOException {
750 		RevObject ro = w.next();
751 		while (ro != null) {
752 			checkCancelled();
753 			if (id2File.remove(ro.getId()) != null)
754 				if (id2File.isEmpty())
755 					return;
756 			ro = w.next();
757 		}
758 		ro = w.nextObject();
759 		while (ro != null) {
760 			checkCancelled();
761 			if (id2File.remove(ro.getId()) != null)
762 				if (id2File.isEmpty())
763 					return;
764 			ro = w.nextObject();
765 		}
766 	}
767 
768 	private static boolean equals(Ref r1, Ref r2) {
769 		if (r1 == null || r2 == null)
770 			return false;
771 		if (r1.isSymbolic()) {
772 			if (!r2.isSymbolic())
773 				return false;
774 			return r1.getTarget().getName().equals(r2.getTarget().getName());
775 		} else {
776 			if (r2.isSymbolic()) {
777 				return false;
778 			}
779 			return Objects.equals(r1.getObjectId(), r2.getObjectId());
780 		}
781 	}
782 
783 	/**
784 	 * Packs all non-symbolic, loose refs into packed-refs.
785 	 *
786 	 * @throws IOException
787 	 */
788 	public void packRefs() throws IOException {
789 		Collection<Ref> refs = repo.getRefDatabase().getRefs(Constants.R_REFS).values();
790 		List<String> refsToBePacked = new ArrayList<>(refs.size());
791 		pm.beginTask(JGitText.get().packRefs, refs.size());
792 		try {
793 			for (Ref ref : refs) {
794 				checkCancelled();
795 				if (!ref.isSymbolic() && ref.getStorage().isLoose())
796 					refsToBePacked.add(ref.getName());
797 				pm.update(1);
798 			}
799 			((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
800 		} finally {
801 			pm.endTask();
802 		}
803 	}
804 
805 	/**
806 	 * Packs all objects which reachable from any of the heads into one pack
807 	 * file. Additionally all objects which are not reachable from any head but
808 	 * which are reachable from any of the other refs (e.g. tags), special refs
809 	 * (e.g. FETCH_HEAD) or index are packed into a separate pack file. Objects
810 	 * included in pack files which have a .keep file associated are never
811 	 * repacked. All old pack files which existed before are deleted.
812 	 *
813 	 * @return a collection of the newly created pack files
814 	 * @throws IOException
815 	 *             when during reading of refs, index, packfiles, objects,
816 	 *             reflog-entries or during writing to the packfiles
817 	 *             {@link IOException} occurs
818 	 */
819 	public Collection<PackFile> repack() throws IOException {
820 		Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
821 
822 		long time = System.currentTimeMillis();
823 		Collection<Ref> refsBefore = getAllRefs();
824 
825 		Set<ObjectId> allHeadsAndTags = new HashSet<>();
826 		Set<ObjectId> allHeads = new HashSet<>();
827 		Set<ObjectId> allTags = new HashSet<>();
828 		Set<ObjectId> nonHeads = new HashSet<>();
829 		Set<ObjectId> txnHeads = new HashSet<>();
830 		Set<ObjectId> tagTargets = new HashSet<>();
831 		Set<ObjectId> indexObjects = listNonHEADIndexObjects();
832 		RefDatabase refdb = repo.getRefDatabase();
833 
834 		for (Ref ref : refsBefore) {
835 			checkCancelled();
836 			nonHeads.addAll(listRefLogObjects(ref, 0));
837 			if (ref.isSymbolic() || ref.getObjectId() == null) {
838 				continue;
839 			}
840 			if (isHead(ref)) {
841 				allHeads.add(ref.getObjectId());
842 			} else if (isTag(ref)) {
843 				allTags.add(ref.getObjectId());
844 			} else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
845 				txnHeads.add(ref.getObjectId());
846 			} else {
847 				nonHeads.add(ref.getObjectId());
848 			}
849 			if (ref.getPeeledObjectId() != null) {
850 				tagTargets.add(ref.getPeeledObjectId());
851 			}
852 		}
853 
854 		List<ObjectIdSet> excluded = new LinkedList<>();
855 		for (final PackFile f : repo.getObjectDatabase().getPacks()) {
856 			checkCancelled();
857 			if (f.shouldBeKept())
858 				excluded.add(f.getIndex());
859 		}
860 
861 		// Don't exclude tags that are also branch tips
862 		allTags.removeAll(allHeads);
863 		allHeadsAndTags.addAll(allHeads);
864 		allHeadsAndTags.addAll(allTags);
865 
866 		// Hoist all branch tips and tags earlier in the pack file
867 		tagTargets.addAll(allHeadsAndTags);
868 		nonHeads.addAll(indexObjects);
869 
870 		// Combine the GC_REST objects into the GC pack if requested
871 		if (pconfig != null && pconfig.getSinglePack()) {
872 			allHeadsAndTags.addAll(nonHeads);
873 			nonHeads.clear();
874 		}
875 
876 		List<PackFile> ret = new ArrayList<>(2);
877 		PackFile heads = null;
878 		if (!allHeadsAndTags.isEmpty()) {
879 			heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
880 					tagTargets, excluded);
881 			if (heads != null) {
882 				ret.add(heads);
883 				excluded.add(0, heads.getIndex());
884 			}
885 		}
886 		if (!nonHeads.isEmpty()) {
887 			PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
888 					tagTargets, excluded);
889 			if (rest != null)
890 				ret.add(rest);
891 		}
892 		if (!txnHeads.isEmpty()) {
893 			PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
894 					null, excluded);
895 			if (txn != null)
896 				ret.add(txn);
897 		}
898 		try {
899 			deleteOldPacks(toBeDeleted, ret);
900 		} catch (ParseException e) {
901 			// TODO: the exception has to be wrapped into an IOException because
902 			// throwing the ParseException directly would break the API, instead
903 			// we should throw a ConfigInvalidException
904 			throw new IOException(e);
905 		}
906 		prunePacked();
907 		deleteOrphans();
908 		deleteTempPacksIdx();
909 
910 		lastPackedRefs = refsBefore;
911 		lastRepackTime = time;
912 		return ret;
913 	}
914 
915 	private static boolean isHead(Ref ref) {
916 		return ref.getName().startsWith(Constants.R_HEADS);
917 	}
918 
919 	private static boolean isTag(Ref ref) {
920 		return ref.getName().startsWith(Constants.R_TAGS);
921 	}
922 
923 	/**
924 	 * Deletes orphans
925 	 * <p>
926 	 * A file is considered an orphan if it is either a "bitmap" or an index
927 	 * file, and its corresponding pack file is missing in the list.
928 	 * </p>
929 	 */
930 	private void deleteOrphans() {
931 		Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
932 				"pack"); //$NON-NLS-1$
933 		List<String> fileNames = null;
934 		try (Stream<Path> files = Files.list(packDir)) {
935 			fileNames = files.map(path -> path.getFileName().toString())
936 					.filter(name -> {
937 						return (name.endsWith(PACK_EXT)
938 								|| name.endsWith(BITMAP_EXT)
939 								|| name.endsWith(INDEX_EXT));
940 					}).sorted(Collections.reverseOrder())
941 					.collect(Collectors.toList());
942 		} catch (IOException e1) {
943 			// ignore
944 		}
945 		if (fileNames == null) {
946 			return;
947 		}
948 
949 		String base = null;
950 		for (String n : fileNames) {
951 			if (n.endsWith(PACK_EXT)) {
952 				base = n.substring(0, n.lastIndexOf('.'));
953 			} else {
954 				if (base == null || !n.startsWith(base)) {
955 					try {
956 						Files.delete(new File(packDir.toFile(), n).toPath());
957 					} catch (IOException e) {
958 						LOG.error(e.getMessage(), e);
959 					}
960 				}
961 			}
962 		}
963 	}
964 
965 	private void deleteTempPacksIdx() {
966 		Path packDir = Paths.get(repo.getObjectsDirectory().getAbsolutePath(),
967 				"pack"); //$NON-NLS-1$
968 		Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
969 		try {
970 			Files.newDirectoryStream(packDir, "gc_*_tmp") //$NON-NLS-1$
971 					.forEach(t -> {
972 						try {
973 							Instant lastModified = Files.getLastModifiedTime(t)
974 									.toInstant();
975 							if (lastModified.isBefore(threshold)) {
976 								Files.deleteIfExists(t);
977 							}
978 						} catch (IOException e) {
979 							LOG.error(e.getMessage(), e);
980 						}
981 					});
982 		} catch (IOException e) {
983 			LOG.error(e.getMessage(), e);
984 		}
985 	}
986 
987 	/**
988 	 * @param ref
989 	 *            the ref which log should be inspected
990 	 * @param minTime only reflog entries not older then this time are processed
991 	 * @return the {@link ObjectId}s contained in the reflog
992 	 * @throws IOException
993 	 */
994 	private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
995 		ReflogReader reflogReader = repo.getReflogReader(ref.getName());
996 		if (reflogReader == null) {
997 			return Collections.emptySet();
998 		}
999 		List<ReflogEntry> rlEntries = reflogReader
1000 				.getReverseEntries();
1001 		if (rlEntries == null || rlEntries.isEmpty())
1002 			return Collections.<ObjectId> emptySet();
1003 		Set<ObjectId> ret = new HashSet<>();
1004 		for (ReflogEntry e : rlEntries) {
1005 			if (e.getWho().getWhen().getTime() < minTime)
1006 				break;
1007 			ObjectId newId = e.getNewId();
1008 			if (newId != null && !ObjectId.zeroId().equals(newId))
1009 				ret.add(newId);
1010 			ObjectId oldId = e.getOldId();
1011 			if (oldId != null && !ObjectId.zeroId().equals(oldId))
1012 				ret.add(oldId);
1013 		}
1014 		return ret;
1015 	}
1016 
1017 	/**
1018 	 * Returns a collection of all refs and additional refs.
1019 	 *
1020 	 * Additional refs which don't start with "refs/" are not returned because
1021 	 * they should not save objects from being garbage collected. Examples for
1022 	 * such references are ORIG_HEAD, MERGE_HEAD, FETCH_HEAD and
1023 	 * CHERRY_PICK_HEAD.
1024 	 *
1025 	 * @return a collection of refs pointing to live objects.
1026 	 * @throws IOException
1027 	 */
1028 	private Collection<Ref> getAllRefs() throws IOException {
1029 		RefDatabase refdb = repo.getRefDatabase();
1030 		Collection<Ref> refs = refdb.getRefs(RefDatabase.ALL).values();
1031 		List<Ref> addl = refdb.getAdditionalRefs();
1032 		if (!addl.isEmpty()) {
1033 			List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1034 			all.addAll(refs);
1035 			// add additional refs which start with refs/
1036 			for (Ref r : addl) {
1037 				checkCancelled();
1038 				if (r.getName().startsWith(Constants.R_REFS)) {
1039 					all.add(r);
1040 				}
1041 			}
1042 			return all;
1043 		}
1044 		return refs;
1045 	}
1046 
1047 	/**
1048 	 * Return a list of those objects in the index which differ from whats in
1049 	 * HEAD
1050 	 *
1051 	 * @return a set of ObjectIds of changed objects in the index
1052 	 * @throws IOException
1053 	 * @throws CorruptObjectException
1054 	 * @throws NoWorkTreeException
1055 	 */
1056 	private Set<ObjectId> listNonHEADIndexObjects()
1057 			throws CorruptObjectException, IOException {
1058 		if (repo.isBare()) {
1059 			return Collections.emptySet();
1060 		}
1061 		try (TreeWalk treeWalk = new TreeWalk(repo)) {
1062 			treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1063 			ObjectId headID = repo.resolve(Constants.HEAD);
1064 			if (headID != null) {
1065 				try (RevWalk revWalk = new RevWalk(repo)) {
1066 					treeWalk.addTree(revWalk.parseTree(headID));
1067 				}
1068 			}
1069 
1070 			treeWalk.setFilter(TreeFilter.ANY_DIFF);
1071 			treeWalk.setRecursive(true);
1072 			Set<ObjectId> ret = new HashSet<>();
1073 
1074 			while (treeWalk.next()) {
1075 				checkCancelled();
1076 				ObjectId objectId = treeWalk.getObjectId(0);
1077 				switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1078 				case FileMode.TYPE_MISSING:
1079 				case FileMode.TYPE_GITLINK:
1080 					continue;
1081 				case FileMode.TYPE_TREE:
1082 				case FileMode.TYPE_FILE:
1083 				case FileMode.TYPE_SYMLINK:
1084 					ret.add(objectId);
1085 					continue;
1086 				default:
1087 					throw new IOException(MessageFormat.format(
1088 							JGitText.get().corruptObjectInvalidMode3,
1089 							String.format("%o", //$NON-NLS-1$
1090 									Integer.valueOf(treeWalk.getRawMode(0))),
1091 							(objectId == null) ? "null" : objectId.name(), //$NON-NLS-1$
1092 							treeWalk.getPathString(), //
1093 							repo.getIndexFile()));
1094 				}
1095 			}
1096 			return ret;
1097 		}
1098 	}
1099 
1100 	private PackFile writePack(@NonNull Set<? extends ObjectId> want,
1101 			@NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1102 			Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1103 			throws IOException {
1104 		checkCancelled();
1105 		File tmpPack = null;
1106 		Map<PackExt, File> tmpExts = new TreeMap<>(
1107 				new Comparator<PackExt>() {
1108 					@Override
1109 					public int compare(PackExt o1, PackExt o2) {
1110 						// INDEX entries must be returned last, so the pack
1111 						// scanner does pick up the new pack until all the
1112 						// PackExt entries have been written.
1113 						if (o1 == o2)
1114 							return 0;
1115 						if (o1 == PackExt.INDEX)
1116 							return 1;
1117 						if (o2 == PackExt.INDEX)
1118 							return -1;
1119 						return Integer.signum(o1.hashCode() - o2.hashCode());
1120 					}
1121 
1122 				});
1123 		try (PackWriter pw = new PackWriter(
1124 				(pconfig == null) ? new PackConfig(repo) : pconfig,
1125 				repo.newObjectReader())) {
1126 			// prepare the PackWriter
1127 			pw.setDeltaBaseAsOffset(true);
1128 			pw.setReuseDeltaCommits(false);
1129 			if (tagTargets != null) {
1130 				pw.setTagTargets(tagTargets);
1131 			}
1132 			if (excludeObjects != null)
1133 				for (ObjectIdSet idx : excludeObjects)
1134 					pw.excludeObjects(idx);
1135 			pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1136 			if (pw.getObjectCount() == 0)
1137 				return null;
1138 			checkCancelled();
1139 
1140 			// create temporary files
1141 			String id = pw.computeName().getName();
1142 			File packdir = new File(repo.getObjectsDirectory(), "pack"); //$NON-NLS-1$
1143 			tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$
1144 			final String tmpBase = tmpPack.getName()
1145 					.substring(0, tmpPack.getName().lastIndexOf('.'));
1146 			File tmpIdx = new File(packdir, tmpBase + ".idx_tmp"); //$NON-NLS-1$
1147 			tmpExts.put(INDEX, tmpIdx);
1148 
1149 			if (!tmpIdx.createNewFile())
1150 				throw new IOException(MessageFormat.format(
1151 						JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1152 
1153 			// write the packfile
1154 			FileOutputStream fos = new FileOutputStream(tmpPack);
1155 			FileChannel channel = fos.getChannel();
1156 			OutputStream channelStream = Channels.newOutputStream(channel);
1157 			try {
1158 				pw.writePack(pm, pm, channelStream);
1159 			} finally {
1160 				channel.force(true);
1161 				channelStream.close();
1162 				fos.close();
1163 			}
1164 
1165 			// write the packindex
1166 			fos = new FileOutputStream(tmpIdx);
1167 			FileChannel idxChannel = fos.getChannel();
1168 			OutputStream idxStream = Channels.newOutputStream(idxChannel);
1169 			try {
1170 				pw.writeIndex(idxStream);
1171 			} finally {
1172 				idxChannel.force(true);
1173 				idxStream.close();
1174 				fos.close();
1175 			}
1176 
1177 			if (pw.prepareBitmapIndex(pm)) {
1178 				File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp"); //$NON-NLS-1$
1179 				tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1180 
1181 				if (!tmpBitmapIdx.createNewFile())
1182 					throw new IOException(MessageFormat.format(
1183 							JGitText.get().cannotCreateIndexfile,
1184 							tmpBitmapIdx.getPath()));
1185 
1186 				fos = new FileOutputStream(tmpBitmapIdx);
1187 				idxChannel = fos.getChannel();
1188 				idxStream = Channels.newOutputStream(idxChannel);
1189 				try {
1190 					pw.writeBitmapIndex(idxStream);
1191 				} finally {
1192 					idxChannel.force(true);
1193 					idxStream.close();
1194 					fos.close();
1195 				}
1196 			}
1197 
1198 			// rename the temporary files to real files
1199 			File realPack = nameFor(id, ".pack"); //$NON-NLS-1$
1200 
1201 			repo.getObjectDatabase().closeAllPackHandles(realPack);
1202 			tmpPack.setReadOnly();
1203 
1204 			FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1205 			for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1206 				File tmpExt = tmpEntry.getValue();
1207 				tmpExt.setReadOnly();
1208 
1209 				File realExt = nameFor(id,
1210 						"." + tmpEntry.getKey().getExtension()); //$NON-NLS-1$
1211 				try {
1212 					FileUtils.rename(tmpExt, realExt,
1213 							StandardCopyOption.ATOMIC_MOVE);
1214 				} catch (IOException e) {
1215 					File newExt = new File(realExt.getParentFile(),
1216 							realExt.getName() + ".new"); //$NON-NLS-1$
1217 					try {
1218 						FileUtils.rename(tmpExt, newExt,
1219 								StandardCopyOption.ATOMIC_MOVE);
1220 					} catch (IOException e2) {
1221 						newExt = tmpExt;
1222 						e = e2;
1223 					}
1224 					throw new IOException(MessageFormat.format(
1225 							JGitText.get().panicCantRenameIndexFile, newExt,
1226 							realExt), e);
1227 				}
1228 			}
1229 
1230 			return repo.getObjectDatabase().openPack(realPack);
1231 		} finally {
1232 			if (tmpPack != null && tmpPack.exists())
1233 				tmpPack.delete();
1234 			for (File tmpExt : tmpExts.values()) {
1235 				if (tmpExt.exists())
1236 					tmpExt.delete();
1237 			}
1238 		}
1239 	}
1240 
1241 	private File nameFor(String name, String ext) {
1242 		File packdir = new File(repo.getObjectsDirectory(), "pack"); //$NON-NLS-1$
1243 		return new File(packdir, "pack-" + name + ext); //$NON-NLS-1$
1244 	}
1245 
1246 	private void checkCancelled() throws CancelledException {
1247 		if (pm.isCancelled()) {
1248 			throw new CancelledException(JGitText.get().operationCanceled);
1249 		}
1250 	}
1251 
1252 	/**
1253 	 * A class holding statistical data for a FileRepository regarding how many
1254 	 * objects are stored as loose or packed objects
1255 	 */
1256 	public static class RepoStatistics {
1257 		/**
1258 		 * The number of objects stored in pack files. If the same object is
1259 		 * stored in multiple pack files then it is counted as often as it
1260 		 * occurs in pack files.
1261 		 */
1262 		public long numberOfPackedObjects;
1263 
1264 		/**
1265 		 * The number of pack files
1266 		 */
1267 		public long numberOfPackFiles;
1268 
1269 		/**
1270 		 * The number of objects stored as loose objects.
1271 		 */
1272 		public long numberOfLooseObjects;
1273 
1274 		/**
1275 		 * The sum of the sizes of all files used to persist loose objects.
1276 		 */
1277 		public long sizeOfLooseObjects;
1278 
1279 		/**
1280 		 * The sum of the sizes of all pack files.
1281 		 */
1282 		public long sizeOfPackedObjects;
1283 
1284 		/**
1285 		 * The number of loose refs.
1286 		 */
1287 		public long numberOfLooseRefs;
1288 
1289 		/**
1290 		 * The number of refs stored in pack files.
1291 		 */
1292 		public long numberOfPackedRefs;
1293 
1294 		/**
1295 		 * The number of bitmaps in the bitmap indices.
1296 		 */
1297 		public long numberOfBitmaps;
1298 
1299 		@Override
1300 		public String toString() {
1301 			final StringBuilder b = new StringBuilder();
1302 			b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$
1303 			b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$
1304 			b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$
1305 			b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$
1306 			b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$
1307 			b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$
1308 			b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$
1309 			b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$
1310 			return b.toString();
1311 		}
1312 	}
1313 
1314 	/**
1315 	 * Returns information about objects and pack files for a FileRepository.
1316 	 *
1317 	 * @return information about objects and pack files for a FileRepository
1318 	 * @throws IOException
1319 	 */
1320 	public RepoStatistics getStatistics() throws IOException {
1321 		RepoStatistics ret = new RepoStatistics();
1322 		Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
1323 		for (PackFile f : packs) {
1324 			ret.numberOfPackedObjects += f.getIndex().getObjectCount();
1325 			ret.numberOfPackFiles++;
1326 			ret.sizeOfPackedObjects += f.getPackFile().length();
1327 			if (f.getBitmapIndex() != null)
1328 				ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
1329 		}
1330 		File objDir = repo.getObjectsDirectory();
1331 		String[] fanout = objDir.list();
1332 		if (fanout != null && fanout.length > 0) {
1333 			for (String d : fanout) {
1334 				if (d.length() != 2)
1335 					continue;
1336 				File[] entries = new File(objDir, d).listFiles();
1337 				if (entries == null)
1338 					continue;
1339 				for (File f : entries) {
1340 					if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1341 						continue;
1342 					ret.numberOfLooseObjects++;
1343 					ret.sizeOfLooseObjects += f.length();
1344 				}
1345 			}
1346 		}
1347 
1348 		RefDatabase refDb = repo.getRefDatabase();
1349 		for (Ref r : refDb.getRefs(RefDatabase.ALL).values()) {
1350 			Storage storage = r.getStorage();
1351 			if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1352 				ret.numberOfLooseRefs++;
1353 			if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1354 				ret.numberOfPackedRefs++;
1355 		}
1356 
1357 		return ret;
1358 	}
1359 
1360 	/**
1361 	 * Set the progress monitor used for garbage collection methods.
1362 	 *
1363 	 * @param pm
1364 	 * @return this
1365 	 */
1366 	public GC setProgressMonitor(ProgressMonitor pm) {
1367 		this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1368 		return this;
1369 	}
1370 
1371 	/**
1372 	 * During gc() or prune() each unreferenced, loose object which has been
1373 	 * created or modified in the last <code>expireAgeMillis</code> milliseconds
1374 	 * will not be pruned. Only older objects may be pruned. If set to 0 then
1375 	 * every object is a candidate for pruning.
1376 	 *
1377 	 * @param expireAgeMillis
1378 	 *            minimal age of objects to be pruned in milliseconds.
1379 	 */
1380 	public void setExpireAgeMillis(long expireAgeMillis) {
1381 		this.expireAgeMillis = expireAgeMillis;
1382 		expire = null;
1383 	}
1384 
1385 	/**
1386 	 * During gc() or prune() packfiles which are created or modified in the
1387 	 * last <code>packExpireAgeMillis</code> milliseconds will not be deleted.
1388 	 * Only older packfiles may be deleted. If set to 0 then every packfile is a
1389 	 * candidate for deletion.
1390 	 *
1391 	 * @param packExpireAgeMillis
1392 	 *            minimal age of packfiles to be deleted in milliseconds.
1393 	 */
1394 	public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1395 		this.packExpireAgeMillis = packExpireAgeMillis;
1396 		expire = null;
1397 	}
1398 
1399 	/**
1400 	 * Set the PackConfig used when (re-)writing packfiles. This allows to
1401 	 * influence how packs are written and to implement something similar to
1402 	 * "git gc --aggressive"
1403 	 *
1404 	 * @param pconfig
1405 	 *            the {@link PackConfig} used when writing packs
1406 	 */
1407 	public void setPackConfig(PackConfig pconfig) {
1408 		this.pconfig = pconfig;
1409 	}
1410 
1411 	/**
1412 	 * During gc() or prune() each unreferenced, loose object which has been
1413 	 * created or modified after or at <code>expire</code> will not be pruned.
1414 	 * Only older objects may be pruned. If set to null then every object is a
1415 	 * candidate for pruning.
1416 	 *
1417 	 * @param expire
1418 	 *            instant in time which defines object expiration
1419 	 *            objects with modification time before this instant are expired
1420 	 *            objects with modification time newer or equal to this instant
1421 	 *            are not expired
1422 	 */
1423 	public void setExpire(Date expire) {
1424 		this.expire = expire;
1425 		expireAgeMillis = -1;
1426 	}
1427 
1428 	/**
1429 	 * During gc() or prune() packfiles which are created or modified after or
1430 	 * at <code>packExpire</code> will not be deleted. Only older packfiles may
1431 	 * be deleted. If set to null then every packfile is a candidate for
1432 	 * deletion.
1433 	 *
1434 	 * @param packExpire
1435 	 *            instant in time which defines packfile expiration
1436 	 */
1437 	public void setPackExpire(Date packExpire) {
1438 		this.packExpire = packExpire;
1439 		packExpireAgeMillis = -1;
1440 	}
1441 
1442 	/**
1443 	 * Set the {@code gc --auto} option.
1444 	 *
1445 	 * With this option, gc checks whether any housekeeping is required; if not,
1446 	 * it exits without performing any work. Some JGit commands run
1447 	 * {@code gc --auto} after performing operations that could create many
1448 	 * loose objects.
1449 	 * <p/>
1450 	 * Housekeeping is required if there are too many loose objects or too many
1451 	 * packs in the repository. If the number of loose objects exceeds the value
1452 	 * of the gc.auto option JGit GC consolidates all existing packs into a
1453 	 * single pack (equivalent to {@code -A} option), whereas git-core would
1454 	 * combine all loose objects into a single pack using {@code repack -d -l}.
1455 	 * Setting the value of {@code gc.auto} to 0 disables automatic packing of
1456 	 * loose objects.
1457 	 * <p/>
1458 	 * If the number of packs exceeds the value of {@code gc.autoPackLimit},
1459 	 * then existing packs (except those marked with a .keep file) are
1460 	 * consolidated into a single pack by using the {@code -A} option of repack.
1461 	 * Setting {@code gc.autoPackLimit} to 0 disables automatic consolidation of
1462 	 * packs.
1463 	 * <p/>
1464 	 * Like git the following jgit commands run auto gc:
1465 	 * <ul>
1466 	 * <li>fetch</li>
1467 	 * <li>merge</li>
1468 	 * <li>rebase</li>
1469 	 * <li>receive-pack</li>
1470 	 * </ul>
1471 	 * The auto gc for receive-pack can be suppressed by setting the config
1472 	 * option {@code receive.autogc = false}
1473 	 *
1474 	 * @param auto
1475 	 *            defines whether gc should do automatic housekeeping
1476 	 */
1477 	public void setAuto(boolean auto) {
1478 		this.automatic = auto;
1479 	}
1480 
1481 	/**
1482 	 * @param background
1483 	 *            whether to run the gc in a background thread.
1484 	 */
1485 	void setBackground(boolean background) {
1486 		this.background = background;
1487 	}
1488 
1489 	private boolean needGc() {
1490 		if (tooManyPacks()) {
1491 			addRepackAllOption();
1492 		} else if (!tooManyLooseObjects()) {
1493 			return false;
1494 		}
1495 		// TODO run pre-auto-gc hook, if it fails return false
1496 		return true;
1497 	}
1498 
1499 	private void addRepackAllOption() {
1500 		// TODO: if JGit GC is enhanced to support repack's option -l this
1501 		// method needs to be implemented
1502 	}
1503 
1504 	/**
1505 	 * @return {@code true} if number of packs > gc.autopacklimit (default 50)
1506 	 */
1507 	boolean tooManyPacks() {
1508 		int autopacklimit = repo.getConfig().getInt(
1509 				ConfigConstants.CONFIG_GC_SECTION,
1510 				ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1511 				DEFAULT_AUTOPACKLIMIT);
1512 		if (autopacklimit <= 0) {
1513 			return false;
1514 		}
1515 		// JGit always creates two packfiles, one for the objects reachable from
1516 		// branches, and another one for the rest
1517 		return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1518 	}
1519 
1520 	/**
1521 	 * Quickly estimate number of loose objects, SHA1 is distributed evenly so
1522 	 * counting objects in one directory (bucket 17) is sufficient
1523 	 *
1524 	 * @return {@code true} if number of loose objects > gc.auto (default 6700)
1525 	 */
1526 	boolean tooManyLooseObjects() {
1527 		int auto = getLooseObjectLimit();
1528 		if (auto <= 0) {
1529 			return false;
1530 		}
1531 		int n = 0;
1532 		int threshold = (auto + 255) / 256;
1533 		Path dir = repo.getObjectsDirectory().toPath().resolve("17"); //$NON-NLS-1$
1534 		if (!Files.exists(dir)) {
1535 			return false;
1536 		}
1537 		try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir,
1538 				new DirectoryStream.Filter<Path>() {
1539 
1540 					@Override
1541 					public boolean accept(Path file) throws IOException {
1542 						Path fileName = file.getFileName();
1543 						return Files.isRegularFile(file) && fileName != null
1544 								&& PATTERN_LOOSE_OBJECT
1545 										.matcher(fileName.toString()).matches();
1546 					}
1547 				})) {
1548 			for (Iterator<Path> iter = stream.iterator(); iter.hasNext();
1549 					iter.next()) {
1550 				if (++n > threshold) {
1551 					return true;
1552 				}
1553 			}
1554 		} catch (IOException e) {
1555 			LOG.error(e.getMessage(), e);
1556 		}
1557 		return false;
1558 	}
1559 
1560 	private int getLooseObjectLimit() {
1561 		return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1562 				ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1563 	}
1564 }