ObjectDirectoryPackParser.java

  1. /*
  2.  * Copyright (C) 2008-2011, Google Inc.
  3.  * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com>
  4.  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
  5.  *
  6.  * This program and the accompanying materials are made available under the
  7.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  8.  * https://www.eclipse.org/org/documents/edl-v10.php.
  9.  *
  10.  * SPDX-License-Identifier: BSD-3-Clause
  11.  */

  13. package org.eclipse.jgit.internal.storage.file;

  15. import java.io.File;
  16. import java.io.FileOutputStream;
  17. import java.io.IOException;
  18. import java.io.InputStream;
  19. import java.io.RandomAccessFile;
  20. import java.nio.file.StandardCopyOption;
  21. import java.security.MessageDigest;
  22. import java.text.MessageFormat;
  23. import java.util.Arrays;
  24. import java.util.List;
  25. import java.util.zip.CRC32;
  26. import java.util.zip.Deflater;

  28. import org.eclipse.jgit.errors.LockFailedException;
  29. import org.eclipse.jgit.internal.JGitText;
  30. import org.eclipse.jgit.internal.storage.pack.PackExt;
  31. import org.eclipse.jgit.lib.AnyObjectId;
  32. import org.eclipse.jgit.lib.Constants;
  33. import org.eclipse.jgit.lib.CoreConfig;
  34. import org.eclipse.jgit.lib.ObjectId;
  35. import org.eclipse.jgit.lib.ProgressMonitor;
  36. import org.eclipse.jgit.storage.pack.PackConfig;
  37. import org.eclipse.jgit.transport.PackParser;
  38. import org.eclipse.jgit.transport.PackedObjectInfo;
  39. import org.eclipse.jgit.util.FileUtils;
  40. import org.eclipse.jgit.util.NB;

  42. /**
  43.  * Consumes a pack stream and stores as a pack file in
  44.  * {@link org.eclipse.jgit.internal.storage.file.ObjectDirectory}.
  45.  * <p>
  46.  * To obtain an instance of a parser, applications should use
  47.  * {@link org.eclipse.jgit.lib.ObjectInserter#newPackParser(InputStream)}.
  48.  */
  49. public class ObjectDirectoryPackParser extends PackParser {
  50.     private final FileObjectDatabase db;

  52.     /** CRC-32 computation for objects that are appended onto the pack. */
  53.     private final CRC32 crc;

  55.     /** Running SHA-1 of any base objects appended after {@link #origEnd}. */
  56.     private final MessageDigest tailDigest;

  58.     /** Preferred format version of the pack-*.idx file to generate. */
  59.     private int indexVersion;

  61.     /** If true, pack with 0 objects will be stored. Usually these are deleted. */
  62.     private boolean keepEmpty;

  64.     /** Path of the temporary file holding the pack data. */
  65.     private File tmpPack;

  67.     /**
  68.      * Path of the index created for the pack, to find objects quickly at read
  69.      * time.
  70.      */
  71.     private File tmpIdx;

  73.     /** Read/write handle to {@link #tmpPack} while it is being parsed. */
  74.     private RandomAccessFile out;

  76.     /** Length of the original pack stream, before missing bases were appended. */
  77.     private long origEnd;

  79.     /** The original checksum of data up to {@link #origEnd}. */
  80.     private byte[] origHash;

  82.     /** Current end of the pack file. */
  83.     private long packEnd;

  85.     /** Checksum of the entire pack file. */
  86.     private byte[] packHash;

  88.     /** Compresses delta bases when completing a thin pack. */
  89.     private Deflater def;

  91.     /** The pack that was created, if parsing was successful. */
  92.     private Pack newPack;

  94.     private PackConfig pconfig;

  96.     ObjectDirectoryPackParser(FileObjectDatabase odb, InputStream src) {
  97.         super(odb, src);
  98.         this.db = odb;
  99.         this.pconfig = new PackConfig(odb.getConfig());
  100.         this.crc = new CRC32();
  101.         this.tailDigest = Constants.newMessageDigest();

  103.         indexVersion = db.getConfig().get(CoreConfig.KEY).getPackIndexVersion();
  104.     }

  106.     /**
  107.      * Set the pack index file format version this instance will create.
  108.      *
  109.      * @param version
  110.      *            the version to write. The special version 0 designates the
  111.      *            oldest (most compatible) format available for the objects.
  112.      * @see PackIndexWriter
  113.      */
  114.     public void setIndexVersion(int version) {
  115.         indexVersion = version;
  116.     }

  118.     /**
  119.      * Configure this index pack instance to keep an empty pack.
  120.      * <p>
  121.      * By default an empty pack (a pack with no objects) is not kept, as doi so
  122.      * is completely pointless. With no objects in the pack there is no d stored
  123.      * by it, so the pack is unnecessary.
  124.      *
  125.      * @param empty
  126.      *            true to enable keeping an empty pack.
  127.      */
  128.     public void setKeepEmpty(boolean empty) {
  129.         keepEmpty = empty;
  130.     }

  132.     /**
  133.      * Get the imported {@link org.eclipse.jgit.internal.storage.file.Pack}.
  134.      * <p>
  135.      * This method is supplied only to support testing; applications shouldn't
  136.      * be using it directly to access the imported data.
  137.      *
  138.      * @return the imported PackFile, if parsing was successful.
  139.      */
  140.     public Pack getPack() {
  141.         return newPack;
  142.     }

  144.     /** {@inheritDoc} */
  145.     @Override
  146.     public long getPackSize() {
  147.         if (newPack == null)
  148.             return super.getPackSize();

  150.         File pack = newPack.getPackFile();
  151.         long size = pack.length();
  152.         String p = pack.getAbsolutePath();
  153.         String i = p.substring(0, p.length() - ".pack".length()) + ".idx"; //$NON-NLS-1$ //$NON-NLS-2$
  154.         File idx = new File(i);
  155.         if (idx.exists() && idx.isFile())
  156.             size += idx.length();
  157.         return size;
  158.     }

  160.     /** {@inheritDoc} */
  161.     @Override
  162.     public PackLock parse(ProgressMonitor receiving, ProgressMonitor resolving)
  163.             throws IOException {
  164.         tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$
  165.         tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx"); //$NON-NLS-1$
  166.         try {
  167.             out = new RandomAccessFile(tmpPack, "rw"); //$NON-NLS-1$

  169.             super.parse(receiving, resolving);

  171.             out.seek(packEnd);
  172.             out.write(packHash);
  173.             out.getChannel().force(true);
  174.             out.close();

  176.             writeIdx();

  178.             tmpPack.setReadOnly();
  179.             tmpIdx.setReadOnly();

  181.             return renameAndOpenPack(getLockMessage());
  182.         } finally {
  183.             if (def != null)
  184.                 def.end();
  185.             try {
  186.                 if (out != null && out.getChannel().isOpen())
  187.                     out.close();
  188.             } catch (IOException closeError) {
  189.                 // Ignored. We want to delete the file.
  190.             }
  191.             cleanupTemporaryFiles();
  192.         }
  193.     }

  195.     /** {@inheritDoc} */
  196.     @Override
  197.     protected void onPackHeader(long objectCount) throws IOException {
  198.         // Ignored, the count is not required.
  199.     }

  201.     /** {@inheritDoc} */
  202.     @Override
  203.     protected void onBeginWholeObject(long streamPosition, int type,
  204.             long inflatedSize) throws IOException {
  205.         crc.reset();
  206.     }

  208.     /** {@inheritDoc} */
  209.     @Override
  210.     protected void onEndWholeObject(PackedObjectInfo info) throws IOException {
  211.         info.setCRC((int) crc.getValue());
  212.     }

  214.     /** {@inheritDoc} */
  215.     @Override
  216.     protected void onBeginOfsDelta(long streamPosition,
  217.             long baseStreamPosition, long inflatedSize) throws IOException {
  218.         crc.reset();
  219.     }

  221.     /** {@inheritDoc} */
  222.     @Override
  223.     protected void onBeginRefDelta(long streamPosition, AnyObjectId baseId,
  224.             long inflatedSize) throws IOException {
  225.         crc.reset();
  226.     }

  228.     /** {@inheritDoc} */
  229.     @Override
  230.     protected UnresolvedDelta onEndDelta() throws IOException {
  231.         UnresolvedDelta delta = new UnresolvedDelta();
  232.         delta.setCRC((int) crc.getValue());
  233.         return delta;
  234.     }

  236.     /** {@inheritDoc} */
  237.     @Override
  238.     protected void onInflatedObjectData(PackedObjectInfo obj, int typeCode,
  239.             byte[] data) throws IOException {
  240.         // ObjectDirectory ignores this event.
  241.     }

  243.     /** {@inheritDoc} */
  244.     @Override
  245.     protected void onObjectHeader(Source src, byte[] raw, int pos, int len)
  246.             throws IOException {
  247.         crc.update(raw, pos, len);
  248.     }

  250.     /** {@inheritDoc} */
  251.     @Override
  252.     protected void onObjectData(Source src, byte[] raw, int pos, int len)
  253.             throws IOException {
  254.         crc.update(raw, pos, len);
  255.     }

  257.     /** {@inheritDoc} */
  258.     @Override
  259.     protected void onStoreStream(byte[] raw, int pos, int len)
  260.             throws IOException {
  261.         out.write(raw, pos, len);
  262.     }

  264.     /** {@inheritDoc} */
  265.     @Override
  266.     protected void onPackFooter(byte[] hash) throws IOException {
  267.         packEnd = out.getFilePointer();
  268.         origEnd = packEnd;
  269.         origHash = hash;
  270.         packHash = hash;
  271.     }

  273.     /** {@inheritDoc} */
  274.     @Override
  275.     protected ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
  276.             ObjectTypeAndSize info) throws IOException {
  277.         out.seek(delta.getOffset());
  278.         crc.reset();
  279.         return readObjectHeader(info);
  280.     }

  282.     /** {@inheritDoc} */
  283.     @Override
  284.     protected ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
  285.             ObjectTypeAndSize info) throws IOException {
  286.         out.seek(obj.getOffset());
  287.         crc.reset();
  288.         return readObjectHeader(info);
  289.     }

  291.     /** {@inheritDoc} */
  292.     @Override
  293.     protected int readDatabase(byte[] dst, int pos, int cnt) throws IOException {
  294.         return out.read(dst, pos, cnt);
  295.     }

  297.     /** {@inheritDoc} */
  298.     @Override
  299.     protected boolean checkCRC(int oldCRC) {
  300.         return oldCRC == (int) crc.getValue();
  301.     }

  303.     private static String baseName(File tmpPack) {
  304.         String name = tmpPack.getName();
  305.         return name.substring(0, name.lastIndexOf('.'));
  306.     }

  308.     private void cleanupTemporaryFiles() {
  309.         if (tmpIdx != null && !tmpIdx.delete() && tmpIdx.exists())
  310.             tmpIdx.deleteOnExit();
  311.         if (tmpPack != null && !tmpPack.delete() && tmpPack.exists())
  312.             tmpPack.deleteOnExit();
  313.     }

  315.     /** {@inheritDoc} */
  316.     @Override
  317.     protected boolean onAppendBase(final int typeCode, final byte[] data,
  318.             final PackedObjectInfo info) throws IOException {
  319.         info.setOffset(packEnd);

  321.         final byte[] buf = buffer();
  322.         int sz = data.length;
  323.         int len = 0;
  324.         buf[len++] = (byte) ((typeCode << 4) | (sz & 15));
  325.         sz >>>= 4;
  326.         while (sz > 0) {
  327.             buf[len - 1] |= (byte) 0x80;
  328.             buf[len++] = (byte) (sz & 0x7f);
  329.             sz >>>= 7;
  330.         }

  332.         tailDigest.update(buf, 0, len);
  333.         crc.reset();
  334.         crc.update(buf, 0, len);
  335.         out.seek(packEnd);
  336.         out.write(buf, 0, len);
  337.         packEnd += len;

  339.         if (def == null)
  340.             def = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
  341.         else
  342.             def.reset();
  343.         def.setInput(data);
  344.         def.finish();

  346.         while (!def.finished()) {
  347.             len = def.deflate(buf);
  348.             tailDigest.update(buf, 0, len);
  349.             crc.update(buf, 0, len);
  350.             out.write(buf, 0, len);
  351.             packEnd += len;
  352.         }

  354.         info.setCRC((int) crc.getValue());
  355.         return true;
  356.     }

  358.     /** {@inheritDoc} */
  359.     @Override
  360.     protected void onEndThinPack() throws IOException {
  361.         final byte[] buf = buffer();

  363.         final MessageDigest origDigest = Constants.newMessageDigest();
  364.         final MessageDigest tailDigest2 = Constants.newMessageDigest();
  365.         final MessageDigest packDigest = Constants.newMessageDigest();

  367.         long origRemaining = origEnd;
  368.         out.seek(0);
  369.         out.readFully(buf, 0, 12);
  370.         origDigest.update(buf, 0, 12);
  371.         origRemaining -= 12;

  373.         NB.encodeInt32(buf, 8, getObjectCount());
  374.         out.seek(0);
  375.         out.write(buf, 0, 12);
  376.         packDigest.update(buf, 0, 12);

  378.         for (;;) {
  379.             final int n = out.read(buf);
  380.             if (n < 0)
  381.                 break;
  382.             if (origRemaining != 0) {
  383.                 final int origCnt = (int) Math.min(n, origRemaining);
  384.                 origDigest.update(buf, 0, origCnt);
  385.                 origRemaining -= origCnt;
  386.                 if (origRemaining == 0)
  387.                     tailDigest2.update(buf, origCnt, n - origCnt);
  388.             } else
  389.                 tailDigest2.update(buf, 0, n);

  391.             packDigest.update(buf, 0, n);
  392.         }

  394.         if (!Arrays.equals(origDigest.digest(), origHash) || !Arrays
  395.                 .equals(tailDigest2.digest(), this.tailDigest.digest()))
  396.             throw new IOException(
  397.                     JGitText.get().packCorruptedWhileWritingToFilesystem);

  399.         packHash = packDigest.digest();
  400.     }

  402.     private void writeIdx() throws IOException {
  403.         List<PackedObjectInfo> list = getSortedObjectList(null /* by ObjectId */);
  404.         try (FileOutputStream os = new FileOutputStream(tmpIdx)) {
  405.             final PackIndexWriter iw;
  406.             if (indexVersion <= 0)
  407.                 iw = PackIndexWriter.createOldestPossible(os, list);
  408.             else
  409.                 iw = PackIndexWriter.createVersion(os, indexVersion);
  410.             iw.write(list, packHash);
  411.             os.getChannel().force(true);
  412.         }
  413.     }

  415.     private PackLock renameAndOpenPack(String lockMessage)
  416.             throws IOException {
  417.         if (!keepEmpty && getObjectCount() == 0) {
  418.             cleanupTemporaryFiles();
  419.             return null;
  420.         }

  422.         final MessageDigest d = Constants.newMessageDigest();
  423.         final byte[] oeBytes = new byte[Constants.OBJECT_ID_LENGTH];
  424.         for (int i = 0; i < getObjectCount(); i++) {
  425.             final PackedObjectInfo oe = getObject(i);
  426.             oe.copyRawTo(oeBytes, 0);
  427.             d.update(oeBytes);
  428.         }

  430.         ObjectId id = ObjectId.fromRaw(d.digest());
  431.         File packDir = new File(db.getDirectory(), "pack"); //$NON-NLS-1$
  432.         PackFile finalPack = new PackFile(packDir, id, PackExt.PACK);
  433.         PackFile finalIdx = finalPack.create(PackExt.INDEX);
  434.         final PackLock keep = new PackLock(finalPack, db.getFS());

  436.         if (!packDir.exists() && !packDir.mkdir() && !packDir.exists()) {
  437.             // The objects/pack directory isn't present, and we are unable
  438.             // to create it. There is no way to move this pack in.
  439.             //
  440.             cleanupTemporaryFiles();
  441.             throw new IOException(MessageFormat.format(
  442.                     JGitText.get().cannotCreateDirectory, packDir
  443.                             .getAbsolutePath()));
  444.         }

  446.         if (finalPack.exists()) {
  447.             // If the pack is already present we should never replace it.
  448.             //
  449.             cleanupTemporaryFiles();
  450.             return null;
  451.         }

  453.         if (lockMessage != null) {
  454.             // If we have a reason to create a keep file for this pack, do
  455.             // so, or fail fast and don't put the pack in place.
  456.             //
  457.             try {
  458.                 if (!keep.lock(lockMessage))
  459.                     throw new LockFailedException(finalPack,
  460.                             MessageFormat.format(
  461.                                     JGitText.get().cannotLockPackIn, finalPack));
  462.             } catch (IOException e) {
  463.                 cleanupTemporaryFiles();
  464.                 throw e;
  465.             }
  466.         }

  468.         try {
  469.             FileUtils.rename(tmpPack, finalPack,
  470.                     StandardCopyOption.ATOMIC_MOVE);
  471.         } catch (IOException e) {
  472.             cleanupTemporaryFiles();
  473.             keep.unlock();
  474.             throw new IOException(MessageFormat.format(
  475.                     JGitText.get().cannotMovePackTo, finalPack), e);
  476.         }

  478.         try {
  479.             FileUtils.rename(tmpIdx, finalIdx, StandardCopyOption.ATOMIC_MOVE);
  480.         } catch (IOException e) {
  481.             cleanupTemporaryFiles();
  482.             keep.unlock();
  483.             if (!finalPack.delete())
  484.                 finalPack.deleteOnExit();
  485.             throw new IOException(MessageFormat.format(
  486.                     JGitText.get().cannotMoveIndexTo, finalIdx), e);
  487.         }

  489.         boolean interrupted = false;
  490.         try {
  491.             FileSnapshot snapshot = FileSnapshot.save(finalPack);
  492.             if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
  493.                 snapshot.waitUntilNotRacy();
  494.             }
  495.         } catch (InterruptedException e) {
  496.             interrupted = true;
  497.         }
  498.         try {
  499.             newPack = db.openPack(finalPack);
  500.         } catch (IOException err) {
  501.             keep.unlock();
  502.             if (finalPack.exists())
  503.                 FileUtils.delete(finalPack);
  504.             if (finalIdx.exists())
  505.                 FileUtils.delete(finalIdx);
  506.             throw err;
  507.         } finally {
  508.             if (interrupted) {
  509.                 // Re-set interrupted flag
  510.                 Thread.currentThread().interrupt();
  511.             }
  512.         }

  514.         return lockMessage != null ? keep : null;
  515.     }
  516. }
Created with JaCoCo 0.8.6.202009150832