View Javadoc
1   /*
2    * Copyright (C) 2009-2010, Google Inc.
3    * Copyright (C) 2009, Robin Rosenberg <robin.rosenberg@dewire.com>
4    * and other copyright owners as documented in the project's IP log.
5    *
6    * This program and the accompanying materials are made available
7    * under the terms of the Eclipse Distribution License v1.0 which
8    * accompanies this distribution, is reproduced below, and is
9    * available at http://www.eclipse.org/org/documents/edl-v10.php
10   *
11   * All rights reserved.
12   *
13   * Redistribution and use in source and binary forms, with or
14   * without modification, are permitted provided that the following
15   * conditions are met:
16   *
17   * - Redistributions of source code must retain the above copyright
18   *   notice, this list of conditions and the following disclaimer.
19   *
20   * - Redistributions in binary form must reproduce the above
21   *   copyright notice, this list of conditions and the following
22   *   disclaimer in the documentation and/or other materials provided
23   *   with the distribution.
24   *
25   * - Neither the name of the Eclipse Foundation, Inc. nor the
26   *   names of its contributors may be used to endorse or promote
27   *   products derived from this software without specific prior
28   *   written permission.
29   *
30   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
31   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
32   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
33   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
35   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
38   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
39   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
40   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
41   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
42   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43   */
44  
45  package org.eclipse.jgit.internal.storage.file;
46  
47  import static org.junit.Assert.assertArrayEquals;
48  import static org.junit.Assert.assertEquals;
49  import static org.junit.Assert.assertFalse;
50  import static org.junit.Assert.assertNotNull;
51  import static org.junit.Assert.assertNotSame;
52  import static org.junit.Assert.fail;
53  
54  import java.io.BufferedOutputStream;
55  import java.io.File;
56  import java.io.FileOutputStream;
57  import java.io.IOException;
58  import java.io.OutputStream;
59  import java.time.Instant;
60  
61  import org.eclipse.jgit.errors.IncorrectObjectTypeException;
62  import org.eclipse.jgit.errors.MissingObjectException;
63  import org.eclipse.jgit.internal.storage.pack.PackWriter;
64  import org.eclipse.jgit.junit.RepositoryTestCase;
65  import org.eclipse.jgit.lib.AnyObjectId;
66  import org.eclipse.jgit.lib.Constants;
67  import org.eclipse.jgit.lib.NullProgressMonitor;
68  import org.eclipse.jgit.lib.ObjectId;
69  import org.eclipse.jgit.lib.ObjectInserter;
70  import org.eclipse.jgit.lib.ObjectLoader;
71  import org.eclipse.jgit.lib.Repository;
72  import org.eclipse.jgit.revwalk.RevObject;
73  import org.eclipse.jgit.revwalk.RevWalk;
74  import org.eclipse.jgit.storage.file.WindowCacheConfig;
75  import org.eclipse.jgit.util.FS;
76  import org.eclipse.jgit.util.FileUtils;
77  import org.junit.After;
78  import org.junit.Before;
79  import org.junit.Test;
80  
81  public class ConcurrentRepackTest extends RepositoryTestCase {
82  	@Override
83  	@Before
84  	public void setUp() throws Exception {
85  		WindowCacheConfig windowCacheConfig = new WindowCacheConfig();
86  		windowCacheConfig.setPackedGitOpenFiles(1);
87  		windowCacheConfig.install();
88  		super.setUp();
89  	}
90  
91  	@Override
92  	@After
93  	public void tearDown() throws Exception {
94  		super.tearDown();
95  		new WindowCacheConfig().install();
96  	}
97  
98  	@Test
99  	public void testObjectInNewPack() throws IncorrectObjectTypeException,
100 			IOException {
101 		// Create a new object in a new pack, and test that it is present.
102 		//
103 		final Repository eden = createBareRepository();
104 		final RevObject o1 = writeBlob(eden, "o1");
105 		pack(eden, o1);
106 		assertEquals(o1.name(), parse(o1).name());
107 	}
108 
109 	@Test
110 	public void testObjectMovedToNewPack1()
111 			throws IncorrectObjectTypeException, IOException {
112 		// Create an object and pack it. Then remove that pack and put the
113 		// object into a different pack file, with some other object. We
114 		// still should be able to access the objects.
115 		//
116 		final Repository eden = createBareRepository();
117 		final RevObject o1 = writeBlob(eden, "o1");
118 		final File[] out1 = pack(eden, o1);
119 		assertEquals(o1.name(), parse(o1).name());
120 
121 		final RevObject o2 = writeBlob(eden, "o2");
122 		pack(eden, o2, o1);
123 
124 		// Force close, and then delete, the old pack.
125 		//
126 		whackCache();
127 		delete(out1);
128 
129 		// Now here is the interesting thing. Will git figure the new
130 		// object exists in the new pack, and not the old one.
131 		//
132 		assertEquals(o2.name(), parse(o2).name());
133 		assertEquals(o1.name(), parse(o1).name());
134 	}
135 
136 	@Test
137 	public void testObjectMovedWithinPack()
138 			throws IncorrectObjectTypeException, IOException {
139 		// Create an object and pack it.
140 		//
141 		final Repository eden = createBareRepository();
142 		final RevObject o1 = writeBlob(eden, "o1");
143 		final File[] out1 = pack(eden, o1);
144 		assertEquals(o1.name(), parse(o1).name());
145 
146 		// Force close the old pack.
147 		//
148 		whackCache();
149 
150 		// Now overwrite the old pack in place. This method of creating a
151 		// different pack under the same file name is partially broken. We
152 		// should also have a different file name because the list of objects
153 		// within the pack has been modified.
154 		//
155 		final RevObject o2 = writeBlob(eden, "o2");
156 		try (PackWriter pw = new PackWriter(eden)) {
157 			pw.addObject(o2);
158 			pw.addObject(o1);
159 			write(out1, pw);
160 		}
161 
162 		// Try the old name, then the new name. The old name should cause the
163 		// pack to reload when it opens and the index and pack mismatch.
164 		//
165 		assertEquals(o1.name(), parse(o1).name());
166 		assertEquals(o2.name(), parse(o2).name());
167 	}
168 
169 	@Test
170 	public void testObjectMovedToNewPack2()
171 			throws IncorrectObjectTypeException, IOException {
172 		// Create an object and pack it. Then remove that pack and put the
173 		// object into a different pack file, with some other object. We
174 		// still should be able to access the objects.
175 		//
176 		final Repository eden = createBareRepository();
177 		final RevObject o1 = writeBlob(eden, "o1");
178 		final File[] out1 = pack(eden, o1);
179 		assertEquals(o1.name(), parse(o1).name());
180 
181 		final ObjectLoader load1 = db.open(o1, Constants.OBJ_BLOB);
182 		assertNotNull(load1);
183 
184 		final RevObject o2 = writeBlob(eden, "o2");
185 		pack(eden, o2, o1);
186 
187 		// Force close, and then delete, the old pack.
188 		//
189 		whackCache();
190 		delete(out1);
191 
192 		// Now here is the interesting thing... can the loader we made
193 		// earlier still resolve the object, even though its underlying
194 		// pack is gone, but the object still exists.
195 		//
196 		final ObjectLoader load2 = db.open(o1, Constants.OBJ_BLOB);
197 		assertNotNull(load2);
198 		assertNotSame(load1, load2);
199 
200 		final byte[] data2 = load2.getCachedBytes();
201 		final byte[] data1 = load1.getCachedBytes();
202 		assertNotNull(data2);
203 		assertNotNull(data1);
204 		assertNotSame(data1, data2); // cache should be per-pack, not per object
205 		assertArrayEquals(data1, data2);
206 		assertEquals(load2.getType(), load1.getType());
207 	}
208 
209 	private static void whackCache() {
210 		final WindowCacheConfig config = new WindowCacheConfig();
211 		config.setPackedGitOpenFiles(1);
212 		config.install();
213 	}
214 
215 	private RevObject parse(AnyObjectId id)
216 			throws MissingObjectException, IOException {
217 		try (RevWalk rw = new RevWalk(db)) {
218 			return rw.parseAny(id);
219 		}
220 	}
221 
222 	private File[] pack(Repository src, RevObject... list)
223 			throws IOException {
224 		try (PackWriter pw = new PackWriter(src)) {
225 			for (RevObject o : list) {
226 				pw.addObject(o);
227 			}
228 
229 			final ObjectId name = pw.computeName();
230 			final File packFile = fullPackFileName(name, ".pack");
231 			final File idxFile = fullPackFileName(name, ".idx");
232 			final File[] files = new File[] { packFile, idxFile };
233 			write(files, pw);
234 			return files;
235 		}
236 	}
237 
238 	private static void write(File[] files, PackWriter pw)
239 			throws IOException {
240 		final Instant begin = FS.DETECTED
241 				.lastModifiedInstant(files[0].getParentFile());
242 		NullProgressMonitor m = NullProgressMonitor.INSTANCE;
243 
244 		try (OutputStream out = new BufferedOutputStream(
245 				new FileOutputStream(files[0]))) {
246 			pw.writePack(m, m, out);
247 		}
248 
249 		try (OutputStream out = new BufferedOutputStream(
250 				new FileOutputStream(files[1]))) {
251 			pw.writeIndex(out);
252 		}
253 
254 		touch(begin, files[0].getParentFile());
255 	}
256 
257 	private static void delete(File[] list) throws IOException {
258 		final Instant begin = FS.DETECTED
259 				.lastModifiedInstant(list[0].getParentFile());
260 		for (File f : list) {
261 			FileUtils.delete(f);
262 			assertFalse(f + " was removed", f.exists());
263 		}
264 		touch(begin, list[0].getParentFile());
265 	}
266 
267 	private static void touch(Instant begin, File dir) throws IOException {
268 		while (begin.compareTo(FS.DETECTED.lastModifiedInstant(dir)) >= 0) {
269 			try {
270 				Thread.sleep(25);
271 			} catch (InterruptedException ie) {
272 				//
273 			}
274 			FS.DETECTED.setLastModified(dir.toPath(), Instant.now());
275 		}
276 	}
277 
278 	private File fullPackFileName(ObjectId name, String suffix) {
279 		final File packdir = db.getObjectDatabase().getPackDirectory();
280 		return new File(packdir, "pack-" + name.name() + suffix);
281 	}
282 
283 	private RevObject writeBlob(Repository repo, String data)
284 			throws IOException {
285 		final byte[] bytes = Constants.encode(data);
286 		final ObjectId id;
287 		try (ObjectInserter inserter = repo.newObjectInserter()) {
288 			id = inserter.insert(Constants.OBJ_BLOB, bytes);
289 			inserter.flush();
290 		}
291 		try {
292 			parse(id);
293 			fail("Object " + id.name() + " should not exist in test repository");
294 		} catch (MissingObjectException e) {
295 			// Ok
296 		}
297 		try (RevWalk revWalk = new RevWalk(repo)) {
298 			return revWalk.lookupBlob(id);
299 		}
300 	}
301 }