/*
    * Copyright (C) 2017, Google Inc.
    * and other copyright owners as documented in the project's IP log.
    *
    * This program and the accompanying materials are made available
    * under the terms of the Eclipse Distribution License v1.0 which
    * accompanies this distribution, is reproduced below, and is
    * available at http://www.eclipse.org/org/documents/edl-v10.php
    *
   * All rights reserved.
   *
   * Redistribution and use in source and binary forms, with or
   * without modification, are permitted provided that the following
   * conditions are met:
   *
   * - Redistributions of source code must retain the above copyright
   *	 notice, this list of conditions and the following disclaimer.
   *
   * - Redistributions in binary form must reproduce the above
   *	 copyright notice, this list of conditions and the following
   *	 disclaimer in the documentation and/or other materials provided
   *	 with the distribution.
   *
   * - Neither the name of the Eclipse Foundation, Inc. nor the
   *	 names of its contributors may be used to endorse or promote
   *	 products derived from this software without specific prior
   *	 written permission.
   *
   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   */
  
  package org.eclipse.jgit.internal.storage.file;
  
  import static java.util.Comparator.comparing;
  import static java.util.stream.Collectors.toList;
  import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
  import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
  import static org.hamcrest.MatcherAssert.assertThat;
  import static org.hamcrest.Matchers.greaterThan;
  import static org.hamcrest.Matchers.lessThan;
  import static org.junit.Assert.assertArrayEquals;
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertNotEquals;
  import static org.junit.Assert.fail;
  
  import java.io.ByteArrayInputStream;
  import java.io.File;
  import java.io.IOException;
  import java.nio.file.FileVisitResult;
  import java.nio.file.Files;
  import java.nio.file.Path;
  import java.nio.file.SimpleFileVisitor;
  import java.nio.file.attribute.BasicFileAttributes;
  import java.util.ArrayList;
  import java.util.Collection;
  import java.util.List;
  import java.util.Random;
  import java.util.function.Predicate;
  import java.util.regex.Matcher;
  import java.util.regex.Pattern;
  
  import org.eclipse.jgit.dircache.DirCache;
  import org.eclipse.jgit.dircache.DirCacheBuilder;
  import org.eclipse.jgit.dircache.DirCacheEntry;
  import org.eclipse.jgit.errors.MissingObjectException;
  import org.eclipse.jgit.junit.RepositoryTestCase;
  import org.eclipse.jgit.lib.CommitBuilder;
  import org.eclipse.jgit.lib.Constants;
  import org.eclipse.jgit.lib.FileMode;
  import org.eclipse.jgit.lib.ObjectId;
  import org.eclipse.jgit.lib.ObjectLoader;
  import org.eclipse.jgit.lib.ObjectReader;
  import org.eclipse.jgit.lib.ObjectStream;
  import org.eclipse.jgit.storage.file.WindowCacheConfig;
  import org.eclipse.jgit.treewalk.CanonicalTreeParser;
  import org.eclipse.jgit.util.IO;
  import org.junit.After;
  import org.junit.Before;
  import org.junit.Test;
  
  @SuppressWarnings("boxing")
  public class PackInserterTest extends RepositoryTestCase {
  	private WindowCacheConfig origWindowCacheConfig;
  
  	private static final Random random = new Random(0);
  
  	@Before
 	public void setWindowCacheConfig() {
 		origWindowCacheConfig = new WindowCacheConfig();
 		origWindowCacheConfig.install();
 	}
 
 	@After
 	public void resetWindowCacheConfig() {
 		origWindowCacheConfig.install();
 	}
 
 	@Before
 	public void emptyAtSetUp() throws Exception {
 		assertEquals(0, listPacks().size());
 		assertNoObjects();
 	}
 
 	@Test
 	public void noFlush() throws Exception {
 		try (PackInserter ins = newInserter()) {
 			ins.insert(OBJ_BLOB, Constants.encode("foo contents"));
 			// No flush.
 		}
 		assertNoObjects();
 	}
 
 	@Test
 	public void flushEmptyPack() throws Exception {
 		try (PackInserter ins = newInserter()) {
 			ins.flush();
 		}
 		assertNoObjects();
 	}
 
 	@Test
 	public void singlePack() throws Exception {
 		ObjectId blobId;
 		byte[] blob = Constants.encode("foo contents");
 		ObjectId treeId;
 		ObjectId commitId;
 		byte[] commit;
 		try (PackInserter ins = newInserter()) {
 			blobId = ins.insert(OBJ_BLOB, blob);
 
 			DirCache dc = DirCache.newInCore();
 			DirCacheBuilder b = dc.builder();
 			DirCacheEntry dce = new DirCacheEntry("foo");
 			dce.setFileMode(FileMode.REGULAR_FILE);
 			dce.setObjectId(blobId);
 			b.add(dce);
 			b.finish();
 			treeId = dc.writeTree(ins);
 
 			CommitBuilder cb = new CommitBuilder();
 			cb.setTreeId(treeId);
 			cb.setAuthor(author);
 			cb.setCommitter(committer);
 			cb.setMessage("Commit message");
 			commit = cb.toByteArray();
 			commitId = ins.insert(cb);
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		List<PackFile> packs = listPacks();
 		assertEquals(1, packs.size());
 		assertEquals(3, packs.get(0).getObjectCount());
 
 		try (ObjectReader reader = db.newObjectReader()) {
 			assertBlob(reader, blobId, blob);
 
 			CanonicalTreeParser treeParser =
 					new CanonicalTreeParser(null, reader, treeId);
 			assertEquals("foo", treeParser.getEntryPathString());
 			assertEquals(blobId, treeParser.getEntryObjectId());
 
 			ObjectLoader commitLoader = reader.open(commitId);
 			assertEquals(OBJ_COMMIT, commitLoader.getType());
 			assertArrayEquals(commit, commitLoader.getBytes());
 		}
 	}
 
 	@Test
 	public void multiplePacks() throws Exception {
 		ObjectId blobId1;
 		ObjectId blobId2;
 		byte[] blob1 = Constants.encode("blob1");
 		byte[] blob2 = Constants.encode("blob2");
 
 		try (PackInserter ins = newInserter()) {
 			blobId1 = ins.insert(OBJ_BLOB, blob1);
 			ins.flush();
 			blobId2 = ins.insert(OBJ_BLOB, blob2);
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		List<PackFile> packs = listPacks();
 		assertEquals(2, packs.size());
 		assertEquals(1, packs.get(0).getObjectCount());
 		assertEquals(1, packs.get(1).getObjectCount());
 
 		try (ObjectReader reader = db.newObjectReader()) {
 			assertBlob(reader, blobId1, blob1);
 			assertBlob(reader, blobId2, blob2);
 		}
 	}
 
 	@Test
 	public void largeBlob() throws Exception {
 		ObjectId blobId;
 		byte[] blob = newLargeBlob();
 		try (PackInserter ins = newInserter()) {
 			assertThat(blob.length, greaterThan(ins.getBufferSize()));
 			blobId =
 					ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob));
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		Collection<PackFile> packs = listPacks();
 		assertEquals(1, packs.size());
 		PackFile p = packs.iterator().next();
 		assertEquals(1, p.getObjectCount());
 
 		try (ObjectReader reader = db.newObjectReader()) {
 			assertBlob(reader, blobId, blob);
 		}
 	}
 
 	@Test
 	public void overwriteExistingPack() throws Exception {
 		ObjectId blobId;
 		byte[] blob = Constants.encode("foo contents");
 
 		try (PackInserter ins = newInserter()) {
 			blobId = ins.insert(OBJ_BLOB, blob);
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		List<PackFile> packs = listPacks();
 		assertEquals(1, packs.size());
 		PackFile pack = packs.get(0);
 		assertEquals(1, pack.getObjectCount());
 
 		String inode = getInode(pack.getPackFile());
 
 		try (PackInserter ins = newInserter()) {
 			ins.checkExisting(false);
 			assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		packs = listPacks();
 		assertEquals(1, packs.size());
 		pack = packs.get(0);
 		assertEquals(1, pack.getObjectCount());
 
 		if (inode != null) {
 			// Old file was overwritten with new file, although objects were
 			// equivalent.
 			assertNotEquals(inode, getInode(pack.getPackFile()));
 		}
 	}
 
 	@Test
 	public void checkExisting() throws Exception {
 		ObjectId blobId;
 		byte[] blob = Constants.encode("foo contents");
 
 		try (PackInserter ins = newInserter()) {
 			blobId = ins.insert(OBJ_BLOB, blob);
 			ins.insert(OBJ_BLOB, Constants.encode("another blob"));
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		assertEquals(1, listPacks().size());
 
 		try (PackInserter ins = newInserter()) {
 			assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		assertEquals(1, listPacks().size());
 
 		try (PackInserter ins = newInserter()) {
 			ins.checkExisting(false);
 			assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		assertEquals(2, listPacks().size());
 
 		try (ObjectReader reader = db.newObjectReader()) {
 			assertBlob(reader, blobId, blob);
 		}
 	}
 
 	@Test
 	public void insertSmallInputStreamRespectsCheckExisting() throws Exception {
 		ObjectId blobId;
 		byte[] blob = Constants.encode("foo contents");
 		try (PackInserter ins = newInserter()) {
 			assertThat(blob.length, lessThan(ins.getBufferSize()));
 			blobId = ins.insert(OBJ_BLOB, blob);
 			ins.insert(OBJ_BLOB, Constants.encode("another blob"));
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		assertEquals(1, listPacks().size());
 
 		try (PackInserter ins = newInserter()) {
 			assertEquals(blobId,
 					ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)));
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		assertEquals(1, listPacks().size());
 	}
 
 	@Test
 	public void insertLargeInputStreamBypassesCheckExisting() throws Exception {
 		ObjectId blobId;
 		byte[] blob = newLargeBlob();
 
 		try (PackInserter ins = newInserter()) {
 			assertThat(blob.length, greaterThan(ins.getBufferSize()));
 			blobId = ins.insert(OBJ_BLOB, blob);
 			ins.insert(OBJ_BLOB, Constants.encode("another blob"));
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		assertEquals(1, listPacks().size());
 
 		try (PackInserter ins = newInserter()) {
 			assertEquals(blobId,
 					ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)));
 			ins.flush();
 		}
 
 		assertPacksOnly();
 		assertEquals(2, listPacks().size());
 	}
 
 	@Test
 	public void readBackSmallFiles() throws Exception {
 		ObjectId blobId1;
 		ObjectId blobId2;
 		ObjectId blobId3;
 		byte[] blob1 = Constants.encode("blob1");
 		byte[] blob2 = Constants.encode("blob2");
 		byte[] blob3 = Constants.encode("blob3");
 		try (PackInserter ins = newInserter()) {
 			assertThat(blob1.length, lessThan(ins.getBufferSize()));
 			blobId1 = ins.insert(OBJ_BLOB, blob1);
 
 			try (ObjectReader reader = ins.newReader()) {
 				assertBlob(reader, blobId1, blob1);
 			}
 
 			// Read-back should not mess up the file pointer.
 			blobId2 = ins.insert(OBJ_BLOB, blob2);
 			ins.flush();
 
 			blobId3 = ins.insert(OBJ_BLOB, blob3);
 		}
 
 		assertPacksOnly();
 		List<PackFile> packs = listPacks();
 		assertEquals(1, packs.size());
 		assertEquals(2, packs.get(0).getObjectCount());
 
 		try (ObjectReader reader = db.newObjectReader()) {
 			assertBlob(reader, blobId1, blob1);
 			assertBlob(reader, blobId2, blob2);
 
 			try {
 				reader.open(blobId3);
 				fail("Expected MissingObjectException");
 			} catch (MissingObjectException expected) {
 				// Expected.
 			}
 		}
 	}
 
 	@Test
 	public void readBackLargeFile() throws Exception {
 		ObjectId blobId;
 		byte[] blob = newLargeBlob();
 
 		WindowCacheConfig wcc = new WindowCacheConfig();
 		wcc.setStreamFileThreshold(1024);
 		wcc.install();
 		try (ObjectReader reader = db.newObjectReader()) {
 			assertThat(blob.length, greaterThan(reader.getStreamFileThreshold()));
 		}
 
 		try (PackInserter ins = newInserter()) {
 			blobId = ins.insert(OBJ_BLOB, blob);
 
 			try (ObjectReader reader = ins.newReader()) {
 				// Double-check threshold is propagated.
 				assertThat(blob.length, greaterThan(reader.getStreamFileThreshold()));
 				assertBlob(reader, blobId, blob);
 			}
 		}
 
 		assertPacksOnly();
 		// Pack was streamed out to disk and read back from the temp file, but
 		// ultimately rolled back and deleted.
 		assertEquals(0, listPacks().size());
 
 		try (ObjectReader reader = db.newObjectReader()) {
 			try {
 				reader.open(blobId);
 				fail("Expected MissingObjectException");
 			} catch (MissingObjectException expected) {
 				// Expected.
 			}
 		}
 	}
 
 	@Test
 	public void readBackFallsBackToRepo() throws Exception {
 		ObjectId blobId;
 		byte[] blob = Constants.encode("foo contents");
 		try (PackInserter ins = newInserter()) {
 			assertThat(blob.length, lessThan(ins.getBufferSize()));
 			blobId = ins.insert(OBJ_BLOB, blob);
 			ins.flush();
 		}
 
 		try (PackInserter ins = newInserter();
 				ObjectReader reader = ins.newReader()) {
 			assertBlob(reader, blobId, blob);
 		}
 	}
 
 	@Test
 	public void readBackSmallObjectBeforeLargeObject() throws Exception {
 		WindowCacheConfig wcc = new WindowCacheConfig();
 		wcc.setStreamFileThreshold(1024);
 		wcc.install();
 
 		ObjectId blobId1;
 		ObjectId blobId2;
 		ObjectId largeId;
 		byte[] blob1 = Constants.encode("blob1");
 		byte[] blob2 = Constants.encode("blob2");
 		byte[] largeBlob = newLargeBlob();
 		try (PackInserter ins = newInserter()) {
 			assertThat(blob1.length, lessThan(ins.getBufferSize()));
 			assertThat(largeBlob.length, greaterThan(ins.getBufferSize()));
 
 			blobId1 = ins.insert(OBJ_BLOB, blob1);
 			largeId = ins.insert(OBJ_BLOB, largeBlob);
 
 			try (ObjectReader reader = ins.newReader()) {
 				// A previous bug did not reset the file pointer to EOF after reading
 				// back. We need to seek to something further back than a full buffer,
 				// since the read-back code eagerly reads a full buffer's worth of data
 				// from the file to pass to the inflater. If we seeked back just a small
 				// amount, this step would consume the rest of the file, so the file
 				// pointer would coincidentally end up back at EOF, hiding the bug.
 				assertBlob(reader, blobId1, blob1);
 			}
 
 			blobId2 = ins.insert(OBJ_BLOB, blob2);
 
 			try (ObjectReader reader = ins.newReader()) {
 				assertBlob(reader, blobId1, blob1);
 				assertBlob(reader, blobId2, blob2);
 				assertBlob(reader, largeId, largeBlob);
 			}
 
 			ins.flush();
 		}
 
 		try (ObjectReader reader = db.newObjectReader()) {
 				assertBlob(reader, blobId1, blob1);
 				assertBlob(reader, blobId2, blob2);
 				assertBlob(reader, largeId, largeBlob);
 		}
 	}
 
 	private List<PackFile> listPacks() throws Exception {
 		List<PackFile> fromOpenDb = listPacks(db);
 		List<PackFile> reopened;
 		try (FileRepository db2 = new FileRepository(db.getDirectory())) {
 			reopened = listPacks(db2);
 		}
 		assertEquals(fromOpenDb.size(), reopened.size());
 		for (int i = 0 ; i < fromOpenDb.size(); i++) {
 			PackFile a = fromOpenDb.get(i);
 			PackFile b = reopened.get(i);
 			assertEquals(a.getPackName(), b.getPackName());
 			assertEquals(
 					a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath());
 			assertEquals(a.getObjectCount(), b.getObjectCount());
 			a.getObjectCount();
 		}
 		return fromOpenDb;
 	}
 
 	private static List<PackFile> listPacks(FileRepository db) throws Exception {
 		return db.getObjectDatabase().getPacks().stream()
 				.sorted(comparing(PackFile::getPackName)).collect(toList());
 	}
 
 	private PackInserter newInserter() {
 		return db.getObjectDatabase().newPackInserter();
 	}
 
 	private static byte[] newLargeBlob() {
 		byte[] blob = new byte[10240];
 		random.nextBytes(blob);
 		return blob;
 	}
 
 	private static String getInode(File f) throws Exception {
 		BasicFileAttributes attrs = Files.readAttributes(
 				f.toPath(), BasicFileAttributes.class);
 		Object k = attrs.fileKey();
 		if (k == null) {
 			return null;
 		}
 		Pattern p = Pattern.compile("^\\(dev=[^,]*,ino=(\\d+)\\)$");
 		Matcher m = p.matcher(k.toString());
 		return m.matches() ? m.group(1) : null;
 	}
 
 	private static void assertBlob(ObjectReader reader, ObjectId id,
 			byte[] expected) throws Exception {
 		ObjectLoader loader = reader.open(id);
 		assertEquals(OBJ_BLOB, loader.getType());
 		assertEquals(expected.length, loader.getSize());
 		try (ObjectStream s = loader.openStream()) {
 			int n = (int) s.getSize();
 			byte[] actual = new byte[n];
 			assertEquals(n, IO.readFully(s, actual, 0));
 			assertArrayEquals(expected, actual);
 		}
 	}
 
 	private void assertPacksOnly() throws Exception {
 		new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx"))
 				.assertNoBadFiles(db.getObjectDatabase().getDirectory());
 	}
 
 	private void assertNoObjects() throws Exception {
 		new BadFileCollector(f -> true)
 				.assertNoBadFiles(db.getObjectDatabase().getDirectory());
 	}
 
 	private static class BadFileCollector extends SimpleFileVisitor<Path> {
 		private final Predicate<String> badName;
 		private List<String> bad;
 
 		BadFileCollector(Predicate<String> badName) {
 			this.badName = badName;
 		}
 
 		void assertNoBadFiles(File f) throws IOException {
 			bad = new ArrayList<>();
 			Files.walkFileTree(f.toPath(), this);
 			if (!bad.isEmpty()) {
 				fail("unexpected files in object directory: " + bad);
 			}
 		}
 
 		@Override
 		public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
 			Path fileName = file.getFileName();
 			if (fileName != null) {
 				String name = fileName.toString();
 				if (!attrs.isDirectory() && badName.test(name)) {
 					bad.add(name);
 				}
 			}
 			return FileVisitResult.CONTINUE;
 		}
 	}
 }