ReftableBatchRefUpdate.java

  1. /*
  2.  * Copyright (C) 2017, Google Inc.
  3.  * and other copyright owners as documented in the project's IP log.
  4.  *
  5.  * This program and the accompanying materials are made available
  6.  * under the terms of the Eclipse Distribution License v1.0 which
  7.  * accompanies this distribution, is reproduced below, and is
  8.  * available at http://www.eclipse.org/org/documents/edl-v10.php
  9.  *
  10.  * All rights reserved.
  11.  *
  12.  * Redistribution and use in source and binary forms, with or
  13.  * without modification, are permitted provided that the following
  14.  * conditions are met:
  15.  *
  16.  * - Redistributions of source code must retain the above copyright
  17.  *   notice, this list of conditions and the following disclaimer.
  18.  *
  19.  * - Redistributions in binary form must reproduce the above
  20.  *   copyright notice, this list of conditions and the following
  21.  *   disclaimer in the documentation and/or other materials provided
  22.  *   with the distribution.
  23.  *
  24.  * - Neither the name of the Eclipse Foundation, Inc. nor the
  25.  *   names of its contributors may be used to endorse or promote
  26.  *   products derived from this software without specific prior
  27.  *   written permission.
  28.  *
  29.  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  30.  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  31.  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  32.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33.  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  34.  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  35.  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  36.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  37.  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  38.  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  39.  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  40.  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  41.  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42.  */

  44. package org.eclipse.jgit.internal.storage.dfs;

  46. import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
  47. import static org.eclipse.jgit.lib.Ref.Storage.NEW;
  48. import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
  49. import static org.eclipse.jgit.transport.ReceiveCommand.Result.LOCK_FAILURE;
  50. import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED;
  51. import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK;
  52. import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_MISSING_OBJECT;
  53. import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD;
  54. import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE_NONFASTFORWARD;

  56. import java.io.ByteArrayOutputStream;
  57. import java.io.IOException;
  58. import java.io.OutputStream;
  59. import java.util.ArrayList;
  60. import java.util.Collections;
  61. import java.util.HashMap;
  62. import java.util.HashSet;
  63. import java.util.List;
  64. import java.util.Map;
  65. import java.util.Set;
  66. import java.util.concurrent.locks.ReentrantLock;

  68. import org.eclipse.jgit.annotations.Nullable;
  69. import org.eclipse.jgit.errors.MissingObjectException;
  70. import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
  71. import org.eclipse.jgit.internal.storage.io.BlockSource;
  72. import org.eclipse.jgit.internal.storage.pack.PackExt;
  73. import org.eclipse.jgit.internal.storage.reftable.Reftable;
  74. import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
  75. import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
  76. import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
  77. import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
  78. import org.eclipse.jgit.lib.AnyObjectId;
  79. import org.eclipse.jgit.lib.BatchRefUpdate;
  80. import org.eclipse.jgit.lib.ObjectId;
  81. import org.eclipse.jgit.lib.ObjectIdRef;
  82. import org.eclipse.jgit.lib.PersonIdent;
  83. import org.eclipse.jgit.lib.ProgressMonitor;
  84. import org.eclipse.jgit.lib.Ref;
  85. import org.eclipse.jgit.lib.ReflogEntry;
  86. import org.eclipse.jgit.lib.SymbolicRef;
  87. import org.eclipse.jgit.revwalk.RevObject;
  88. import org.eclipse.jgit.revwalk.RevTag;
  89. import org.eclipse.jgit.revwalk.RevWalk;
  90. import org.eclipse.jgit.transport.ReceiveCommand;

  92. /**
  93.  * {@link org.eclipse.jgit.lib.BatchRefUpdate} for
  94.  * {@link org.eclipse.jgit.internal.storage.dfs.DfsReftableDatabase}.
  95.  */
  96. public class ReftableBatchRefUpdate extends BatchRefUpdate {
  97.     private static final int AVG_BYTES = 36;

  99.     private final DfsReftableDatabase refdb;

  101.     private final DfsObjDatabase odb;

  103.     private final ReentrantLock lock;

  105.     private final ReftableConfig reftableConfig;

  107.     /**
  108.      * Initialize batch update.
  109.      *
  110.      * @param refdb
  111.      *            database the update will modify.
  112.      * @param odb
  113.      *            object database to store the reftable.
  114.      */
  115.     protected ReftableBatchRefUpdate(DfsReftableDatabase refdb,
  116.             DfsObjDatabase odb) {
  117.         super(refdb);
  118.         this.refdb = refdb;
  119.         this.odb = odb;
  120.         lock = refdb.getLock();
  121.         reftableConfig = refdb.getReftableConfig();
  122.     }

  124.     /** {@inheritDoc} */
  125.     @Override
  126.     public void execute(RevWalk rw, ProgressMonitor pm, List<String> options) {
  127.         List<ReceiveCommand> pending = getPending();
  128.         if (pending.isEmpty()) {
  129.             return;
  130.         }
  131.         if (options != null) {
  132.             setPushOptions(options);
  133.         }
  134.         try {
  135.             if (!checkObjectExistence(rw, pending)) {
  136.                 return;
  137.             }
  138.             if (!checkNonFastForwards(rw, pending)) {
  139.                 return;
  140.             }

  142.             lock.lock();
  143.             try {
  144.                 Reftable table = refdb.reader();
  145.                 if (!checkExpected(table, pending)) {
  146.                     return;
  147.                 }
  148.                 if (!checkConflicting(pending)) {
  149.                     return;
  150.                 }
  151.                 if (!blockUntilTimestamps(MAX_WAIT)) {
  152.                     return;
  153.                 }
  154.                 applyUpdates(rw, pending);
  155.                 for (ReceiveCommand cmd : pending) {
  156.                     cmd.setResult(OK);
  157.                 }
  158.             } finally {
  159.                 lock.unlock();
  160.             }
  161.         } catch (IOException e) {
  162.             pending.get(0).setResult(LOCK_FAILURE, "io error"); //$NON-NLS-1$
  163.             ReceiveCommand.abort(pending);
  164.         }
  165.     }

  167.     private List<ReceiveCommand> getPending() {
  168.         return ReceiveCommand.filter(getCommands(), NOT_ATTEMPTED);
  169.     }

  171.     private boolean checkObjectExistence(RevWalk rw,
  172.             List<ReceiveCommand> pending) throws IOException {
  173.         for (ReceiveCommand cmd : pending) {
  174.             try {
  175.                 if (!cmd.getNewId().equals(ObjectId.zeroId())) {
  176.                     rw.parseAny(cmd.getNewId());
  177.                 }
  178.             } catch (MissingObjectException e) {
  179.                 // ReceiveCommand#setResult(Result) converts REJECTED to
  180.                 // REJECTED_NONFASTFORWARD, even though that result is also
  181.                 // used for a missing object. Eagerly handle this case so we
  182.                 // can set the right result.
  183.                 cmd.setResult(REJECTED_MISSING_OBJECT);
  184.                 ReceiveCommand.abort(pending);
  185.                 return false;
  186.             }
  187.         }
  188.         return true;
  189.     }

  191.     private boolean checkNonFastForwards(RevWalk rw,
  192.             List<ReceiveCommand> pending) throws IOException {
  193.         if (isAllowNonFastForwards()) {
  194.             return true;
  195.         }
  196.         for (ReceiveCommand cmd : pending) {
  197.             cmd.updateType(rw);
  198.             if (cmd.getType() == UPDATE_NONFASTFORWARD) {
  199.                 cmd.setResult(REJECTED_NONFASTFORWARD);
  200.                 ReceiveCommand.abort(pending);
  201.                 return false;
  202.             }
  203.         }
  204.         return true;
  205.     }

  207.     private boolean checkConflicting(List<ReceiveCommand> pending)
  208.             throws IOException {
  209.         Set<String> names = new HashSet<>();
  210.         for (ReceiveCommand cmd : pending) {
  211.             names.add(cmd.getRefName());
  212.         }

  214.         boolean ok = true;
  215.         for (ReceiveCommand cmd : pending) {
  216.             String name = cmd.getRefName();
  217.             if (refdb.isNameConflicting(name)) {
  218.                 cmd.setResult(LOCK_FAILURE);
  219.                 ok = false;
  220.             } else {
  221.                 int s = name.lastIndexOf('/');
  222.                 while (0 < s) {
  223.                     if (names.contains(name.substring(0, s))) {
  224.                         cmd.setResult(LOCK_FAILURE);
  225.                         ok = false;
  226.                         break;
  227.                     }
  228.                     s = name.lastIndexOf('/', s - 1);
  229.                 }
  230.             }
  231.         }
  232.         if (!ok && isAtomic()) {
  233.             ReceiveCommand.abort(pending);
  234.             return false;
  235.         }
  236.         return ok;
  237.     }

  239.     private boolean checkExpected(Reftable table, List<ReceiveCommand> pending)
  240.             throws IOException {
  241.         for (ReceiveCommand cmd : pending) {
  242.             if (!matchOld(cmd, table.exactRef(cmd.getRefName()))) {
  243.                 cmd.setResult(LOCK_FAILURE);
  244.                 if (isAtomic()) {
  245.                     ReceiveCommand.abort(pending);
  246.                     return false;
  247.                 }
  248.             }
  249.         }
  250.         return true;
  251.     }

  253.     private static boolean matchOld(ReceiveCommand cmd, @Nullable Ref ref) {
  254.         if (ref == null) {
  255.             return AnyObjectId.isEqual(ObjectId.zeroId(), cmd.getOldId())
  256.                     && cmd.getOldSymref() == null;
  257.         } else if (ref.isSymbolic()) {
  258.             return ref.getTarget().getName().equals(cmd.getOldSymref());
  259.         }
  260.         ObjectId id = ref.getObjectId();
  261.         if (id == null) {
  262.             id = ObjectId.zeroId();
  263.         }
  264.         return cmd.getOldId().equals(id);
  265.     }

  267.     private void applyUpdates(RevWalk rw, List<ReceiveCommand> pending)
  268.             throws IOException {
  269.         List<Ref> newRefs = toNewRefs(rw, pending);
  270.         long updateIndex = nextUpdateIndex();
  271.         Set<DfsPackDescription> prune = Collections.emptySet();
  272.         DfsPackDescription pack = odb.newPack(PackSource.INSERT);
  273.         try (DfsOutputStream out = odb.writeFile(pack, REFTABLE)) {
  274.             ReftableConfig cfg = DfsPackCompactor
  275.                     .configureReftable(reftableConfig, out);

  277.             ReftableWriter.Stats stats;
  278.             if (refdb.compactDuringCommit()
  279.                     && newRefs.size() * AVG_BYTES <= cfg.getRefBlockSize()
  280.                     && canCompactTopOfStack(cfg)) {
  281.                 ByteArrayOutputStream tmp = new ByteArrayOutputStream();
  282.                 write(tmp, cfg, updateIndex, newRefs, pending);
  283.                 stats = compactTopOfStack(out, cfg, tmp.toByteArray());
  284.                 prune = toPruneTopOfStack();
  285.             } else {
  286.                 stats = write(out, cfg, updateIndex, newRefs, pending);
  287.             }
  288.             pack.addFileExt(REFTABLE);
  289.             pack.setReftableStats(stats);
  290.         }

  292.         odb.commitPack(Collections.singleton(pack), prune);
  293.         odb.addReftable(pack, prune);
  294.         refdb.clearCache();
  295.     }

  297.     private ReftableWriter.Stats write(OutputStream os, ReftableConfig cfg,
  298.             long updateIndex, List<Ref> newRefs, List<ReceiveCommand> pending)
  299.             throws IOException {
  300.         ReftableWriter writer = new ReftableWriter(cfg)
  301.                 .setMinUpdateIndex(updateIndex).setMaxUpdateIndex(updateIndex)
  302.                 .begin(os).sortAndWriteRefs(newRefs);
  303.         if (!isRefLogDisabled()) {
  304.             writeLog(writer, updateIndex, pending);
  305.         }
  306.         writer.finish();
  307.         return writer.getStats();
  308.     }

  310.     private void writeLog(ReftableWriter writer, long updateIndex,
  311.             List<ReceiveCommand> pending) throws IOException {
  312.         Map<String, ReceiveCommand> cmds = new HashMap<>();
  313.         List<String> byName = new ArrayList<>(pending.size());
  314.         for (ReceiveCommand cmd : pending) {
  315.             cmds.put(cmd.getRefName(), cmd);
  316.             byName.add(cmd.getRefName());
  317.         }
  318.         Collections.sort(byName);

  320.         PersonIdent ident = getRefLogIdent();
  321.         if (ident == null) {
  322.             ident = new PersonIdent(refdb.getRepository());
  323.         }
  324.         for (String name : byName) {
  325.             ReceiveCommand cmd = cmds.get(name);
  326.             if (isRefLogDisabled(cmd)) {
  327.                 continue;
  328.             }
  329.             String msg = getRefLogMessage(cmd);
  330.             if (isRefLogIncludingResult(cmd)) {
  331.                 String strResult = toResultString(cmd);
  332.                 if (strResult != null) {
  333.                     msg = msg.isEmpty() ? strResult : msg + ": " + strResult; //$NON-NLS-1$
  334.                 }
  335.             }
  336.             writer.writeLog(name, updateIndex, ident, cmd.getOldId(),
  337.                     cmd.getNewId(), msg);
  338.         }
  339.     }

  341.     private String toResultString(ReceiveCommand cmd) {
  342.         switch (cmd.getType()) {
  343.         case CREATE:
  344.             return ReflogEntry.PREFIX_CREATED;
  345.         case UPDATE:
  346.             // Match the behavior of a single RefUpdate. In that case, setting
  347.             // the force bit completely bypasses the potentially expensive
  348.             // isMergedInto check, by design, so the reflog message may be
  349.             // inaccurate.
  350.             //
  351.             // Similarly, this class bypasses the isMergedInto checks when the
  352.             // force bit is set, meaning we can't actually distinguish between
  353.             // UPDATE and UPDATE_NONFASTFORWARD when isAllowNonFastForwards()
  354.             // returns true.
  355.             return isAllowNonFastForwards() ? ReflogEntry.PREFIX_FORCED_UPDATE
  356.                     : ReflogEntry.PREFIX_FAST_FORWARD;
  357.         case UPDATE_NONFASTFORWARD:
  358.             return ReflogEntry.PREFIX_FORCED_UPDATE;
  359.         default:
  360.             return null;
  361.         }
  362.     }

  364.     private static List<Ref> toNewRefs(RevWalk rw, List<ReceiveCommand> pending)
  365.             throws IOException {
  366.         List<Ref> refs = new ArrayList<>(pending.size());
  367.         for (ReceiveCommand cmd : pending) {
  368.             String name = cmd.getRefName();
  369.             ObjectId newId = cmd.getNewId();
  370.             String newSymref = cmd.getNewSymref();
  371.             if (AnyObjectId.isEqual(ObjectId.zeroId(), newId)
  372.                     && newSymref == null) {
  373.                 refs.add(new ObjectIdRef.Unpeeled(NEW, name, null));
  374.                 continue;
  375.             } else if (newSymref != null) {
  376.                 refs.add(new SymbolicRef(name,
  377.                         new ObjectIdRef.Unpeeled(NEW, newSymref, null)));
  378.                 continue;
  379.             }

  381.             RevObject obj = rw.parseAny(newId);
  382.             RevObject peel = null;
  383.             if (obj instanceof RevTag) {
  384.                 peel = rw.peel(obj);
  385.             }
  386.             if (peel != null) {
  387.                 refs.add(new ObjectIdRef.PeeledTag(PACKED, name, newId,
  388.                         peel.copy()));
  389.             } else {
  390.                 refs.add(new ObjectIdRef.PeeledNonTag(PACKED, name, newId));
  391.             }
  392.         }
  393.         return refs;
  394.     }

  396.     private long nextUpdateIndex() throws IOException {
  397.         long updateIndex = 0;
  398.         for (Reftable r : refdb.stack().readers()) {
  399.             if (r instanceof ReftableReader) {
  400.                 updateIndex = Math.max(updateIndex,
  401.                         ((ReftableReader) r).maxUpdateIndex());
  402.             }
  403.         }
  404.         return updateIndex + 1;
  405.     }

  407.     private boolean canCompactTopOfStack(ReftableConfig cfg)
  408.             throws IOException {
  409.         ReftableStack stack = refdb.stack();
  410.         List<Reftable> readers = stack.readers();
  411.         if (readers.isEmpty()) {
  412.             return false;
  413.         }

  415.         int lastIdx = readers.size() - 1;
  416.         DfsReftable last = stack.files().get(lastIdx);
  417.         DfsPackDescription desc = last.getPackDescription();
  418.         if (desc.getPackSource() != PackSource.INSERT
  419.                 || !packOnlyContainsReftable(desc)) {
  420.             return false;
  421.         }

  423.         Reftable table = readers.get(lastIdx);
  424.         int bs = cfg.getRefBlockSize();
  425.         return table instanceof ReftableReader
  426.                 && ((ReftableReader) table).size() <= 3 * bs;
  427.     }

  429.     private ReftableWriter.Stats compactTopOfStack(OutputStream out,
  430.             ReftableConfig cfg, byte[] newTable) throws IOException {
  431.         List<Reftable> stack = refdb.stack().readers();
  432.         Reftable last = stack.get(stack.size() - 1);

  434.         List<Reftable> tables = new ArrayList<>(2);
  435.         tables.add(last);
  436.         tables.add(new ReftableReader(BlockSource.from(newTable)));

  438.         ReftableCompactor compactor = new ReftableCompactor();
  439.         compactor.setConfig(cfg);
  440.         compactor.setIncludeDeletes(true);
  441.         compactor.addAll(tables);
  442.         compactor.compact(out);
  443.         return compactor.getStats();
  444.     }

  446.     private Set<DfsPackDescription> toPruneTopOfStack() throws IOException {
  447.         List<DfsReftable> stack = refdb.stack().files();
  448.         DfsReftable last = stack.get(stack.size() - 1);
  449.         return Collections.singleton(last.getPackDescription());
  450.     }

  452.     private boolean packOnlyContainsReftable(DfsPackDescription desc) {
  453.         for (PackExt ext : PackExt.values()) {
  454.             if (ext != REFTABLE && desc.hasFileExt(ext)) {
  455.                 return false;
  456.             }
  457.         }
  458.         return true;
  459.     }
  460. }
Created with JaCoCo 0.8.4.201905082037