ReftableOutputStream.java

  1. /*
  2.  * Copyright (C) 2017, Google Inc.
  3.  * and other copyright owners as documented in the project's IP log.
  4.  *
  5.  * This program and the accompanying materials are made available
  6.  * under the terms of the Eclipse Distribution License v1.0 which
  7.  * accompanies this distribution, is reproduced below, and is
  8.  * available at http://www.eclipse.org/org/documents/edl-v10.php
  9.  *
  10.  * All rights reserved.
  11.  *
  12.  * Redistribution and use in source and binary forms, with or
  13.  * without modification, are permitted provided that the following
  14.  * conditions are met:
  15.  *
  16.  * - Redistributions of source code must retain the above copyright
  17.  *   notice, this list of conditions and the following disclaimer.
  18.  *
  19.  * - Redistributions in binary form must reproduce the above
  20.  *   copyright notice, this list of conditions and the following
  21.  *   disclaimer in the documentation and/or other materials provided
  22.  *   with the distribution.
  23.  *
  24.  * - Neither the name of the Eclipse Foundation, Inc. nor the
  25.  *   names of its contributors may be used to endorse or promote
  26.  *   products derived from this software without specific prior
  27.  *   written permission.
  28.  *
  29.  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  30.  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  31.  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  32.  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  33.  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  34.  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  35.  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  36.  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  37.  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  38.  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  39.  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  40.  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  41.  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42.  */

  43. package org.eclipse.jgit.internal.storage.reftable;

  44. import static java.nio.charset.StandardCharsets.UTF_8;
  45. import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN;
  46. import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE;
  47. import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.LOG_BLOCK_TYPE;
  48. import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH;

  49. import java.io.IOException;
  50. import java.io.OutputStream;
  51. import java.util.Arrays;
  52. import java.util.zip.Deflater;
  53. import java.util.zip.DeflaterOutputStream;

  54. import org.eclipse.jgit.internal.JGitText;
  55. import org.eclipse.jgit.lib.ObjectId;
  56. import org.eclipse.jgit.util.NB;
  57. import org.eclipse.jgit.util.io.CountingOutputStream;

  58. /**
  59.  * Wrapper to assist formatting a reftable to an {@link OutputStream}.
  60.  * <p>
  61.  * Internally buffers at block size boundaries, flushing only complete blocks to
  62.  * the {@code OutputStream}.
  63.  */
  64. class ReftableOutputStream extends OutputStream {
  65.     private final byte[] tmp = new byte[10];
  66.     private final CountingOutputStream out;
  67.     private final boolean alignBlocks;

  68.     private Deflater deflater;
  69.     private DeflaterOutputStream compressor;

  70.     private int blockType;
  71.     private int blockSize;
  72.     private int blockStart;
  73.     private byte[] blockBuf;
  74.     private int cur;
  75.     private long paddingUsed;

  76.     ReftableOutputStream(OutputStream os, int bs, boolean align) {
  77.         blockSize = bs;
  78.         blockBuf = new byte[bs];
  79.         alignBlocks = align;
  80.         out = new CountingOutputStream(os);
  81.     }

  82.     void setBlockSize(int bs) {
  83.         blockSize = bs;
  84.     }

  85.     /** {@inheritDoc} */
  86.     @Override
  87.     public void write(int b) {
  88.         ensureBytesAvailableInBlockBuf(1);
  89.         blockBuf[cur++] = (byte) b;
  90.     }

  91.     /** {@inheritDoc} */
  92.     @Override
  93.     public void write(byte[] b, int off, int cnt) {
  94.         ensureBytesAvailableInBlockBuf(cnt);
  95.         System.arraycopy(b, off, blockBuf, cur, cnt);
  96.         cur += cnt;
  97.     }

  98.     int bytesWrittenInBlock() {
  99.         return cur;
  100.     }

  101.     int bytesAvailableInBlock() {
  102.         return blockSize - cur;
  103.     }

  104.     long paddingUsed() {
  105.         return paddingUsed;
  106.     }

  107.     /** @return bytes flushed; excludes {@link #bytesWrittenInBlock()}. */
  108.     long size() {
  109.         return out.getCount();
  110.     }

  111.     static int computeVarintSize(long val) {
  112.         int n = 1;
  113.         for (; (val >>>= 7) != 0; n++) {
  114.             val--;
  115.         }
  116.         return n;
  117.     }

  118.     void writeVarint(long val) {
  119.         int n = tmp.length;
  120.         tmp[--n] = (byte) (val & 0x7f);
  121.         while ((val >>>= 7) != 0) {
  122.             tmp[--n] = (byte) (0x80 | (--val & 0x7F));
  123.         }
  124.         write(tmp, n, tmp.length - n);
  125.     }

  126.     void writeInt16(int val) {
  127.         ensureBytesAvailableInBlockBuf(2);
  128.         NB.encodeInt16(blockBuf, cur, val);
  129.         cur += 2;
  130.     }

  131.     void writeInt24(int val) {
  132.         ensureBytesAvailableInBlockBuf(3);
  133.         NB.encodeInt24(blockBuf, cur, val);
  134.         cur += 3;
  135.     }

  136.     void writeId(ObjectId id) {
  137.         ensureBytesAvailableInBlockBuf(OBJECT_ID_LENGTH);
  138.         id.copyRawTo(blockBuf, cur);
  139.         cur += OBJECT_ID_LENGTH;
  140.     }

  141.     void writeVarintString(String s) {
  142.         writeVarintString(s.getBytes(UTF_8));
  143.     }

  144.     void writeVarintString(byte[] msg) {
  145.         writeVarint(msg.length);
  146.         write(msg, 0, msg.length);
  147.     }

  148.     private void ensureBytesAvailableInBlockBuf(int cnt) {
  149.         if (cur + cnt > blockBuf.length) {
  150.             int n = Math.max(cur + cnt, blockBuf.length * 2);
  151.             blockBuf = Arrays.copyOf(blockBuf, n);
  152.         }
  153.     }

  154.     void flushFileHeader() throws IOException {
  155.         if (cur == FILE_HEADER_LEN && out.getCount() == 0) {
  156.             out.write(blockBuf, 0, cur);
  157.             cur = 0;
  158.         }
  159.     }

  160.     void beginBlock(byte type) {
  161.         blockType = type;
  162.         blockStart = cur;
  163.         cur += 4; // reserve space for 4-byte block header.
  164.     }

  165.     void flushBlock() throws IOException {
  166.         if (cur > blockSize && blockType != INDEX_BLOCK_TYPE) {
  167.             throw new IOException(JGitText.get().overflowedReftableBlock);
  168.         }
  169.         NB.encodeInt32(blockBuf, blockStart, (blockType << 24) | cur);

  170.         if (blockType == LOG_BLOCK_TYPE) {
  171.             // Log blocks are deflated after the block header.
  172.             out.write(blockBuf, 0, 4);
  173.             if (deflater != null) {
  174.                 deflater.reset();
  175.             } else {
  176.                 deflater = new Deflater(Deflater.BEST_COMPRESSION);
  177.                 compressor = new DeflaterOutputStream(out, deflater);
  178.             }
  179.             compressor.write(blockBuf, 4, cur - 4);
  180.             compressor.finish();
  181.         } else {
  182.             // Other blocks are uncompressed.
  183.             out.write(blockBuf, 0, cur);
  184.         }

  185.         cur = 0;
  186.         blockType = 0;
  187.         blockStart = 0;
  188.     }

  189.     void padBetweenBlocksToNextBlock() throws IOException {
  190.         if (alignBlocks) {
  191.             long m = size() % blockSize;
  192.             if (m > 0) {
  193.                 int pad = blockSize - (int) m;
  194.                 ensureBytesAvailableInBlockBuf(pad);
  195.                 Arrays.fill(blockBuf, 0, pad, (byte) 0);
  196.                 out.write(blockBuf, 0, pad);
  197.                 paddingUsed += pad;
  198.             }
  199.         }
  200.     }

  201.     int estimatePadBetweenBlocks(int currentBlockSize) {
  202.         if (alignBlocks) {
  203.             long m = (size() + currentBlockSize) % blockSize;
  204.             return m > 0 ? blockSize - (int) m : 0;
  205.         }
  206.         return 0;
  207.     }

  208.     void finishFile() throws IOException {
  209.         // File footer doesn't need patching for the block start.
  210.         // Just flush what has been buffered.
  211.         out.write(blockBuf, 0, cur);
  212.         cur = 0;

  213.         if (deflater != null) {
  214.             deflater.end();
  215.         }
  216.     }
  217. }