PackDirectory.java

  1. /*
  2.  * Copyright (C) 2009, Google Inc. and others
  3.  *
  4.  * This program and the accompanying materials are made available under the
  5.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  6.  * https://www.eclipse.org/org/documents/edl-v10.php.
  7.  *
  8.  * SPDX-License-Identifier: BSD-3-Clause
  9.  */

  11. package org.eclipse.jgit.internal.storage.file;

  13. import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
  14. import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
  15. import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;

  17. import java.io.File;
  18. import java.io.FileNotFoundException;
  19. import java.io.IOException;
  20. import java.text.MessageFormat;
  21. import java.util.ArrayList;
  22. import java.util.Arrays;
  23. import java.util.Collection;
  24. import java.util.Collections;
  25. import java.util.EnumMap;
  26. import java.util.HashMap;
  27. import java.util.List;
  28. import java.util.Map;
  29. import java.util.Set;
  30. import java.util.concurrent.atomic.AtomicReference;

  32. import org.eclipse.jgit.annotations.Nullable;
  33. import org.eclipse.jgit.errors.CorruptObjectException;
  34. import org.eclipse.jgit.errors.PackInvalidException;
  35. import org.eclipse.jgit.errors.PackMismatchException;
  36. import org.eclipse.jgit.internal.JGitText;
  37. import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
  38. import org.eclipse.jgit.internal.storage.pack.PackExt;
  39. import org.eclipse.jgit.internal.storage.pack.PackWriter;
  40. import org.eclipse.jgit.lib.AbbreviatedObjectId;
  41. import org.eclipse.jgit.lib.AnyObjectId;
  42. import org.eclipse.jgit.lib.Config;
  43. import org.eclipse.jgit.lib.ConfigConstants;
  44. import org.eclipse.jgit.lib.ObjectId;
  45. import org.eclipse.jgit.lib.ObjectLoader;
  46. import org.eclipse.jgit.util.FileUtils;
  47. import org.slf4j.Logger;
  48. import org.slf4j.LoggerFactory;

  50. /**
  51.  * Traditional file system packed objects directory handler.
  52.  * <p>
  53.  * This is the {@link org.eclipse.jgit.internal.storage.file.Pack}s object
  54.  * representation for a Git object database, where objects are stored in
  55.  * compressed containers known as
  56.  * {@link org.eclipse.jgit.internal.storage.file.Pack}s.
  57.  */
  58. class PackDirectory {
  59.     private final static Logger LOG = LoggerFactory
  60.             .getLogger(PackDirectory.class);

  62.     private static final PackList NO_PACKS = new PackList(FileSnapshot.DIRTY,
  63.             new Pack[0]);

  65.     private final Config config;

  67.     private final File directory;

  69.     private final AtomicReference<PackList> packList;

  71.     /**
  72.      * Initialize a reference to an on-disk 'pack' directory.
  73.      *
  74.      * @param config
  75.      *            configuration this directory consults for write settings.
  76.      * @param directory
  77.      *            the location of the {@code pack} directory.
  78.      */
  79.     PackDirectory(Config config, File directory) {
  80.         this.config = config;
  81.         this.directory = directory;
  82.         packList = new AtomicReference<>(NO_PACKS);
  83.     }

  85.     /**
  86.      * Getter for the field {@code directory}.
  87.      *
  88.      * @return the location of the {@code pack} directory.
  89.      */
  90.     File getDirectory() {
  91.         return directory;
  92.     }

  94.     void create() throws IOException {
  95.         FileUtils.mkdir(directory);
  96.     }

  98.     void close() {
  99.         PackList packs = packList.get();
  100.         if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) {
  101.             for (Pack p : packs.packs) {
  102.                 p.close();
  103.             }
  104.         }
  105.     }

  107.     Collection<Pack> getPacks() {
  108.         PackList list = packList.get();
  109.         if (list == NO_PACKS) {
  110.             list = scanPacks(list);
  111.         }
  112.         Pack[] packs = list.packs;
  113.         return Collections.unmodifiableCollection(Arrays.asList(packs));
  114.     }

  116.     /** {@inheritDoc} */
  117.     @Override
  118.     public String toString() {
  119.         return "PackDirectory[" + getDirectory() + "]"; //$NON-NLS-1$ //$NON-NLS-2$
  120.     }

  122.     /**
  123.      * Does the requested object exist in this PackDirectory?
  124.      *
  125.      * @param objectId
  126.      *            identity of the object to test for existence of.
  127.      * @return {@code true} if the specified object is stored in this PackDirectory.
  128.      */
  129.     boolean has(AnyObjectId objectId) {
  130.         return getPack(objectId) != null;
  131.     }

  133.     /**
  134.      * Get the {@link org.eclipse.jgit.internal.storage.file.Pack} for the
  135.      * specified object if it is stored in this PackDirectory.
  136.      *
  137.      * @param objectId
  138.      *            identity of the object to find the Pack for.
  139.      * @return {@link org.eclipse.jgit.internal.storage.file.Pack} which
  140.      *         contains the specified object or {@code null} if it is not stored
  141.      *         in this PackDirectory.
  142.      */
  143.     @Nullable
  144.     Pack getPack(AnyObjectId objectId) {
  145.         PackList pList;
  146.         do {
  147.             pList = packList.get();
  148.             for (Pack p : pList.packs) {
  149.                 try {
  150.                     if (p.hasObject(objectId)) {
  151.                         return p;
  152.                     }
  153.                 } catch (IOException e) {
  154.                     // The hasObject call should have only touched the index, so
  155.                     // any failure here indicates the index is unreadable by
  156.                     // this process, and the pack is likewise not readable.
  157.                     LOG.warn(MessageFormat.format(
  158.                             JGitText.get().unableToReadPackfile,
  159.                             p.getPackFile().getAbsolutePath()), e);
  160.                     remove(p);
  161.                 }
  162.             }
  163.         } while (searchPacksAgain(pList));
  164.         return null;
  165.     }

  167.     /**
  168.      * Find objects matching the prefix abbreviation.
  169.      *
  170.      * @param matches
  171.      *            set to add any located ObjectIds to. This is an output
  172.      *            parameter.
  173.      * @param id
  174.      *            prefix to search for.
  175.      * @param matchLimit
  176.      *            maximum number of results to return. At most this many
  177.      *            ObjectIds should be added to matches before returning.
  178.      * @return {@code true} if the matches were exhausted before reaching
  179.      *         {@code maxLimit}.
  180.      */
  181.     boolean resolve(Set<ObjectId> matches, AbbreviatedObjectId id,
  182.             int matchLimit) {
  183.         // Go through the packs once. If we didn't find any resolutions
  184.         // scan for new packs and check once more.
  185.         int oldSize = matches.size();
  186.         PackList pList;
  187.         do {
  188.             pList = packList.get();
  189.             for (Pack p : pList.packs) {
  190.                 try {
  191.                     p.resolve(matches, id, matchLimit);
  192.                     p.resetTransientErrorCount();
  193.                 } catch (IOException e) {
  194.                     handlePackError(e, p);
  195.                 }
  196.                 if (matches.size() > matchLimit) {
  197.                     return false;
  198.                 }
  199.             }
  200.         } while (matches.size() == oldSize && searchPacksAgain(pList));
  201.         return true;
  202.     }

  204.     ObjectLoader open(WindowCursor curs, AnyObjectId objectId) {
  205.         PackList pList;
  206.         do {
  207.             SEARCH: for (;;) {
  208.                 pList = packList.get();
  209.                 for (Pack p : pList.packs) {
  210.                     try {
  211.                         ObjectLoader ldr = p.get(curs, objectId);
  212.                         p.resetTransientErrorCount();
  213.                         if (ldr != null)
  214.                             return ldr;
  215.                     } catch (PackMismatchException e) {
  216.                         // Pack was modified; refresh the entire pack list.
  217.                         if (searchPacksAgain(pList)) {
  218.                             continue SEARCH;
  219.                         }
  220.                     } catch (IOException e) {
  221.                         handlePackError(e, p);
  222.                     }
  223.                 }
  224.                 break SEARCH;
  225.             }
  226.         } while (searchPacksAgain(pList));
  227.         return null;
  228.     }

  230.     long getSize(WindowCursor curs, AnyObjectId id) {
  231.         PackList pList;
  232.         do {
  233.             SEARCH: for (;;) {
  234.                 pList = packList.get();
  235.                 for (Pack p : pList.packs) {
  236.                     try {
  237.                         long len = p.getObjectSize(curs, id);
  238.                         p.resetTransientErrorCount();
  239.                         if (0 <= len) {
  240.                             return len;
  241.                         }
  242.                     } catch (PackMismatchException e) {
  243.                         // Pack was modified; refresh the entire pack list.
  244.                         if (searchPacksAgain(pList)) {
  245.                             continue SEARCH;
  246.                         }
  247.                     } catch (IOException e) {
  248.                         handlePackError(e, p);
  249.                     }
  250.                 }
  251.                 break SEARCH;
  252.             }
  253.         } while (searchPacksAgain(pList));
  254.         return -1;
  255.     }

  257.     void selectRepresentation(PackWriter packer, ObjectToPack otp,
  258.             WindowCursor curs) {
  259.         PackList pList = packList.get();
  260.         SEARCH: for (;;) {
  261.             for (Pack p : pList.packs) {
  262.                 try {
  263.                     LocalObjectRepresentation rep = p.representation(curs, otp);
  264.                     p.resetTransientErrorCount();
  265.                     if (rep != null) {
  266.                         packer.select(otp, rep);
  267.                     }
  268.                 } catch (PackMismatchException e) {
  269.                     // Pack was modified; refresh the entire pack list.
  270.                     //
  271.                     pList = scanPacks(pList);
  272.                     continue SEARCH;
  273.                 } catch (IOException e) {
  274.                     handlePackError(e, p);
  275.                 }
  276.             }
  277.             break SEARCH;
  278.         }
  279.     }

  281.     private void handlePackError(IOException e, Pack p) {
  282.         String warnTmpl = null;
  283.         int transientErrorCount = 0;
  284.         String errTmpl = JGitText.get().exceptionWhileReadingPack;
  285.         if ((e instanceof CorruptObjectException)
  286.                 || (e instanceof PackInvalidException)) {
  287.             warnTmpl = JGitText.get().corruptPack;
  288.             LOG.warn(MessageFormat.format(warnTmpl,
  289.                     p.getPackFile().getAbsolutePath()), e);
  290.             // Assume the pack is corrupted, and remove it from the list.
  291.             remove(p);
  292.         } else if (e instanceof FileNotFoundException) {
  293.             if (p.getPackFile().exists()) {
  294.                 errTmpl = JGitText.get().packInaccessible;
  295.                 transientErrorCount = p.incrementTransientErrorCount();
  296.             } else {
  297.                 warnTmpl = JGitText.get().packWasDeleted;
  298.                 remove(p);
  299.             }
  300.         } else if (FileUtils.isStaleFileHandleInCausalChain(e)) {
  301.             warnTmpl = JGitText.get().packHandleIsStale;
  302.             remove(p);
  303.         } else {
  304.             transientErrorCount = p.incrementTransientErrorCount();
  305.         }
  306.         if (warnTmpl != null) {
  307.             LOG.warn(MessageFormat.format(warnTmpl,
  308.                     p.getPackFile().getAbsolutePath()), e);
  309.         } else {
  310.             if (doLogExponentialBackoff(transientErrorCount)) {
  311.                 // Don't remove the pack from the list, as the error may be
  312.                 // transient.
  313.                 LOG.error(MessageFormat.format(errTmpl,
  314.                         p.getPackFile().getAbsolutePath(),
  315.                         Integer.valueOf(transientErrorCount)), e);
  316.             }
  317.         }
  318.     }

  320.     /**
  321.      * @param n
  322.      *            count of consecutive failures
  323.      * @return @{code true} if i is a power of 2
  324.      */
  325.     private boolean doLogExponentialBackoff(int n) {
  326.         return (n & (n - 1)) == 0;
  327.     }

  329.     boolean searchPacksAgain(PackList old) {
  330.         // Whether to trust the pack folder's modification time. If set
  331.         // to false we will always scan the .git/objects/pack folder to
  332.         // check for new pack files. If set to true (default) we use the
  333.         // lastmodified attribute of the folder and assume that no new
  334.         // pack files can be in this folder if his modification time has
  335.         // not changed.
  336.         boolean trustFolderStat = config.getBoolean(
  337.                 ConfigConstants.CONFIG_CORE_SECTION,
  338.                 ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true);

  340.         return ((!trustFolderStat) || old.snapshot.isModified(directory))
  341.                 && old != scanPacks(old);
  342.     }

  344.     void insert(Pack pack) {
  345.         PackList o, n;
  346.         do {
  347.             o = packList.get();

  349.             // If the pack in question is already present in the list
  350.             // (picked up by a concurrent thread that did a scan?) we
  351.             // do not want to insert it a second time.
  352.             //
  353.             final Pack[] oldList = o.packs;
  354.             final String name = pack.getPackFile().getName();
  355.             for (Pack p : oldList) {
  356.                 if (name.equals(p.getPackFile().getName())) {
  357.                     return;
  358.                 }
  359.             }

  361.             final Pack[] newList = new Pack[1 + oldList.length];
  362.             newList[0] = pack;
  363.             System.arraycopy(oldList, 0, newList, 1, oldList.length);
  364.             n = new PackList(o.snapshot, newList);
  365.         } while (!packList.compareAndSet(o, n));
  366.     }

  368.     private void remove(Pack deadPack) {
  369.         PackList o, n;
  370.         do {
  371.             o = packList.get();

  373.             final Pack[] oldList = o.packs;
  374.             final int j = indexOf(oldList, deadPack);
  375.             if (j < 0) {
  376.                 break;
  377.             }

  379.             final Pack[] newList = new Pack[oldList.length - 1];
  380.             System.arraycopy(oldList, 0, newList, 0, j);
  381.             System.arraycopy(oldList, j + 1, newList, j, newList.length - j);
  382.             n = new PackList(o.snapshot, newList);
  383.         } while (!packList.compareAndSet(o, n));
  384.         deadPack.close();
  385.     }

  387.     private static int indexOf(Pack[] list, Pack pack) {
  388.         for (int i = 0; i < list.length; i++) {
  389.             if (list[i] == pack) {
  390.                 return i;
  391.             }
  392.         }
  393.         return -1;
  394.     }

  396.     private PackList scanPacks(PackList original) {
  397.         synchronized (packList) {
  398.             PackList o, n;
  399.             do {
  400.                 o = packList.get();
  401.                 if (o != original) {
  402.                     // Another thread did the scan for us, while we
  403.                     // were blocked on the monitor above.
  404.                     //
  405.                     return o;
  406.                 }
  407.                 n = scanPacksImpl(o);
  408.                 if (n == o) {
  409.                     return n;
  410.                 }
  411.             } while (!packList.compareAndSet(o, n));
  412.             return n;
  413.         }
  414.     }

  416.     private PackList scanPacksImpl(PackList old) {
  417.         final Map<String, Pack> forReuse = reuseMap(old);
  418.         final FileSnapshot snapshot = FileSnapshot.save(directory);
  419.         Map<String, Map<PackExt, PackFile>> packFilesByExtById = getPackFilesByExtById();
  420.         List<Pack> list = new ArrayList<>(packFilesByExtById.size());
  421.         boolean foundNew = false;
  422.         for (Map<PackExt, PackFile> packFilesByExt : packFilesByExtById
  423.                 .values()) {
  424.             PackFile packFile = packFilesByExt.get(PACK);
  425.             if (packFile == null || !packFilesByExt.containsKey(INDEX)) {
  426.                 // Sometimes C Git's HTTP fetch transport leaves a
  427.                 // .idx file behind and does not download the .pack.
  428.                 // We have to skip over such useless indexes.
  429.                 // Also skip if we don't have any index for this id
  430.                 continue;
  431.             }

  433.             Pack oldPack = forReuse.get(packFile.getName());
  434.             if (oldPack != null
  435.                     && !oldPack.getFileSnapshot().isModified(packFile)) {
  436.                 forReuse.remove(packFile.getName());
  437.                 list.add(oldPack);
  438.                 continue;
  439.             }

  441.             list.add(new Pack(packFile, packFilesByExt.get(BITMAP_INDEX)));
  442.             foundNew = true;
  443.         }

  445.         // If we did not discover any new files, the modification time was not
  446.         // changed, and we did not remove any files, then the set of files is
  447.         // the same as the set we were given. Instead of building a new object
  448.         // return the same collection.
  449.         //
  450.         if (!foundNew && forReuse.isEmpty() && snapshot.equals(old.snapshot)) {
  451.             old.snapshot.setClean(snapshot);
  452.             return old;
  453.         }

  455.         for (Pack p : forReuse.values()) {
  456.             p.close();
  457.         }

  459.         if (list.isEmpty()) {
  460.             return new PackList(snapshot, NO_PACKS.packs);
  461.         }

  463.         final Pack[] r = list.toArray(new Pack[0]);
  464.         Arrays.sort(r, Pack.SORT);
  465.         return new PackList(snapshot, r);
  466.     }

  468.     private static Map<String, Pack> reuseMap(PackList old) {
  469.         final Map<String, Pack> forReuse = new HashMap<>();
  470.         for (Pack p : old.packs) {
  471.             if (p.invalid()) {
  472.                 // The pack instance is corrupted, and cannot be safely used
  473.                 // again. Do not include it in our reuse map.
  474.                 //
  475.                 p.close();
  476.                 continue;
  477.             }

  479.             final Pack prior = forReuse.put(p.getPackFile().getName(), p);
  480.             if (prior != null) {
  481.                 // This should never occur. It should be impossible for us
  482.                 // to have two pack files with the same name, as all of them
  483.                 // came out of the same directory. If it does, we promised to
  484.                 // close any PackFiles we did not reuse, so close the second,
  485.                 // readers are likely to be actively using the first.
  486.                 //
  487.                 forReuse.put(prior.getPackFile().getName(), prior);
  488.                 p.close();
  489.             }
  490.         }
  491.         return forReuse;
  492.     }

  494.     /**
  495.      * Scans the pack directory for
  496.      * {@link org.eclipse.jgit.internal.storage.file.PackFile}s and returns them
  497.      * organized by their extensions and their pack ids
  498.      *
  499.      * Skips files in the directory that we cannot create a
  500.      * {@link org.eclipse.jgit.internal.storage.file.PackFile} for.
  501.      *
  502.      * @return a map of {@link org.eclipse.jgit.internal.storage.file.PackFile}s
  503.      *         and {@link org.eclipse.jgit.internal.storage.pack.PackExt}s keyed
  504.      *         by pack ids
  505.      */
  506.     private Map<String, Map<PackExt, PackFile>> getPackFilesByExtById() {
  507.         final String[] nameList = directory.list();
  508.         if (nameList == null) {
  509.             return Collections.emptyMap();
  510.         }
  511.         Map<String, Map<PackExt, PackFile>> packFilesByExtById = new HashMap<>(
  512.                 nameList.length / 2); // assume roughly 2 files per id
  513.         for (String name : nameList) {
  514.             try {
  515.                 PackFile pack = new PackFile(directory, name);
  516.                 if (pack.getPackExt() != null) {
  517.                     Map<PackExt, PackFile> packByExt = packFilesByExtById
  518.                             .get(pack.getId());
  519.                     if (packByExt == null) {
  520.                         packByExt = new EnumMap<>(PackExt.class);
  521.                         packFilesByExtById.put(pack.getId(), packByExt);
  522.                     }
  523.                     packByExt.put(pack.getPackExt(), pack);
  524.                 }
  525.             } catch (IllegalArgumentException e) {
  526.                 continue;
  527.             }
  528.         }
  529.         return packFilesByExtById;
  530.     }

  532.     static final class PackList {
  533.         /** State just before reading the pack directory. */
  534.         final FileSnapshot snapshot;

  536.         /** All known packs, sorted by {@link Pack#SORT}. */
  537.         final Pack[] packs;

  539.         PackList(FileSnapshot monitor, Pack[] packs) {
  540.             this.snapshot = monitor;
  541.             this.packs = packs;
  542.         }
  543.     }
  544. }
Created with JaCoCo 0.8.6.202009150832