View Javadoc
1   /*
2    * Copyright (C) 2009-2010, Google Inc.
3    * Copyright (C) 2009, Robin Rosenberg <robin.rosenberg@dewire.com>
4    * and other copyright owners as documented in the project's IP log.
5    *
6    * This program and the accompanying materials are made available
7    * under the terms of the Eclipse Distribution License v1.0 which
8    * accompanies this distribution, is reproduced below, and is
9    * available at http://www.eclipse.org/org/documents/edl-v10.php
10   *
11   * All rights reserved.
12   *
13   * Redistribution and use in source and binary forms, with or
14   * without modification, are permitted provided that the following
15   * conditions are met:
16   *
17   * - Redistributions of source code must retain the above copyright
18   *   notice, this list of conditions and the following disclaimer.
19   *
20   * - Redistributions in binary form must reproduce the above
21   *   copyright notice, this list of conditions and the following
22   *   disclaimer in the documentation and/or other materials provided
23   *   with the distribution.
24   *
25   * - Neither the name of the Eclipse Foundation, Inc. nor the
26   *   names of its contributors may be used to endorse or promote
27   *   products derived from this software without specific prior
28   *   written permission.
29   *
30   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
31   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
32   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
33   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
35   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
38   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
39   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
40   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
41   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
42   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43   */
44  
45  package org.eclipse.jgit.internal.storage.file;
46  
47  import static org.junit.Assert.assertArrayEquals;
48  import static org.junit.Assert.assertEquals;
49  import static org.junit.Assert.assertFalse;
50  import static org.junit.Assert.assertNotNull;
51  import static org.junit.Assert.assertNotSame;
52  import static org.junit.Assert.fail;
53  
54  import java.io.File;
55  import java.io.FileOutputStream;
56  import java.io.IOException;
57  import java.io.OutputStream;
58  
59  import org.eclipse.jgit.errors.IncorrectObjectTypeException;
60  import org.eclipse.jgit.errors.MissingObjectException;
61  import org.eclipse.jgit.internal.storage.pack.PackWriter;
62  import org.eclipse.jgit.junit.RepositoryTestCase;
63  import org.eclipse.jgit.lib.AnyObjectId;
64  import org.eclipse.jgit.lib.Constants;
65  import org.eclipse.jgit.lib.NullProgressMonitor;
66  import org.eclipse.jgit.lib.ObjectId;
67  import org.eclipse.jgit.lib.ObjectInserter;
68  import org.eclipse.jgit.lib.ObjectLoader;
69  import org.eclipse.jgit.lib.Repository;
70  import org.eclipse.jgit.revwalk.RevObject;
71  import org.eclipse.jgit.revwalk.RevWalk;
72  import org.eclipse.jgit.storage.file.WindowCacheConfig;
73  import org.eclipse.jgit.util.FileUtils;
74  import org.eclipse.jgit.util.io.SafeBufferedOutputStream;
75  import org.junit.After;
76  import org.junit.Before;
77  import org.junit.Test;
78  
79  public class ConcurrentRepackTest extends RepositoryTestCase {
80  	@Before
81  	public void setUp() throws Exception {
82  		WindowCacheConfig windowCacheConfig = new WindowCacheConfig();
83  		windowCacheConfig.setPackedGitOpenFiles(1);
84  		windowCacheConfig.install();
85  		super.setUp();
86  	}
87  
88  	@After
89  	public void tearDown() throws Exception {
90  		super.tearDown();
91  		new WindowCacheConfig().install();
92  	}
93  
94  	@Test
95  	public void testObjectInNewPack() throws IncorrectObjectTypeException,
96  			IOException {
97  		// Create a new object in a new pack, and test that it is present.
98  		//
99  		final Repository eden = createBareRepository();
100 		final RevObject o1 = writeBlob(eden, "o1");
101 		pack(eden, o1);
102 		assertEquals(o1.name(), parse(o1).name());
103 	}
104 
105 	@Test
106 	public void testObjectMovedToNewPack1()
107 			throws IncorrectObjectTypeException, IOException {
108 		// Create an object and pack it. Then remove that pack and put the
109 		// object into a different pack file, with some other object. We
110 		// still should be able to access the objects.
111 		//
112 		final Repository eden = createBareRepository();
113 		final RevObject o1 = writeBlob(eden, "o1");
114 		final File[] out1 = pack(eden, o1);
115 		assertEquals(o1.name(), parse(o1).name());
116 
117 		final RevObject o2 = writeBlob(eden, "o2");
118 		pack(eden, o2, o1);
119 
120 		// Force close, and then delete, the old pack.
121 		//
122 		whackCache();
123 		delete(out1);
124 
125 		// Now here is the interesting thing. Will git figure the new
126 		// object exists in the new pack, and not the old one.
127 		//
128 		assertEquals(o2.name(), parse(o2).name());
129 		assertEquals(o1.name(), parse(o1).name());
130 	}
131 
132 	@Test
133 	public void testObjectMovedWithinPack()
134 			throws IncorrectObjectTypeException, IOException {
135 		// Create an object and pack it.
136 		//
137 		final Repository eden = createBareRepository();
138 		final RevObject o1 = writeBlob(eden, "o1");
139 		final File[] out1 = pack(eden, o1);
140 		assertEquals(o1.name(), parse(o1).name());
141 
142 		// Force close the old pack.
143 		//
144 		whackCache();
145 
146 		// Now overwrite the old pack in place. This method of creating a
147 		// different pack under the same file name is partially broken. We
148 		// should also have a different file name because the list of objects
149 		// within the pack has been modified.
150 		//
151 		final RevObject o2 = writeBlob(eden, "o2");
152 		try (PackWriter pw = new PackWriter(eden)) {
153 			pw.addObject(o2);
154 			pw.addObject(o1);
155 			write(out1, pw);
156 		}
157 
158 		// Try the old name, then the new name. The old name should cause the
159 		// pack to reload when it opens and the index and pack mismatch.
160 		//
161 		assertEquals(o1.name(), parse(o1).name());
162 		assertEquals(o2.name(), parse(o2).name());
163 	}
164 
165 	@Test
166 	public void testObjectMovedToNewPack2()
167 			throws IncorrectObjectTypeException, IOException {
168 		// Create an object and pack it. Then remove that pack and put the
169 		// object into a different pack file, with some other object. We
170 		// still should be able to access the objects.
171 		//
172 		final Repository eden = createBareRepository();
173 		final RevObject o1 = writeBlob(eden, "o1");
174 		final File[] out1 = pack(eden, o1);
175 		assertEquals(o1.name(), parse(o1).name());
176 
177 		final ObjectLoader load1 = db.open(o1, Constants.OBJ_BLOB);
178 		assertNotNull(load1);
179 
180 		final RevObject o2 = writeBlob(eden, "o2");
181 		pack(eden, o2, o1);
182 
183 		// Force close, and then delete, the old pack.
184 		//
185 		whackCache();
186 		delete(out1);
187 
188 		// Now here is the interesting thing... can the loader we made
189 		// earlier still resolve the object, even though its underlying
190 		// pack is gone, but the object still exists.
191 		//
192 		final ObjectLoader load2 = db.open(o1, Constants.OBJ_BLOB);
193 		assertNotNull(load2);
194 		assertNotSame(load1, load2);
195 
196 		final byte[] data2 = load2.getCachedBytes();
197 		final byte[] data1 = load1.getCachedBytes();
198 		assertNotNull(data2);
199 		assertNotNull(data1);
200 		assertNotSame(data1, data2); // cache should be per-pack, not per object
201 		assertArrayEquals(data1, data2);
202 		assertEquals(load2.getType(), load1.getType());
203 	}
204 
205 	private static void whackCache() {
206 		final WindowCacheConfig config = new WindowCacheConfig();
207 		config.setPackedGitOpenFiles(1);
208 		config.install();
209 	}
210 
211 	private RevObject parse(final AnyObjectId id)
212 			throws MissingObjectException, IOException {
213 		try (RevWalk rw = new RevWalk(db)) {
214 			return rw.parseAny(id);
215 		}
216 	}
217 
218 	private File[] pack(final Repository src, final RevObject... list)
219 			throws IOException {
220 		try (PackWriter pw = new PackWriter(src)) {
221 			for (final RevObject o : list) {
222 				pw.addObject(o);
223 			}
224 
225 			final ObjectId name = pw.computeName();
226 			final File packFile = fullPackFileName(name, ".pack");
227 			final File idxFile = fullPackFileName(name, ".idx");
228 			final File[] files = new File[] { packFile, idxFile };
229 			write(files, pw);
230 			return files;
231 		}
232 	}
233 
234 	private static void write(final File[] files, final PackWriter pw)
235 			throws IOException {
236 		final long begin = files[0].getParentFile().lastModified();
237 		NullProgressMonitor m = NullProgressMonitor.INSTANCE;
238 		OutputStream out;
239 
240 		out = new SafeBufferedOutputStream(new FileOutputStream(files[0]));
241 		try {
242 			pw.writePack(m, m, out);
243 		} finally {
244 			out.close();
245 		}
246 
247 		out = new SafeBufferedOutputStream(new FileOutputStream(files[1]));
248 		try {
249 			pw.writeIndex(out);
250 		} finally {
251 			out.close();
252 		}
253 
254 		touch(begin, files[0].getParentFile());
255 	}
256 
257 	private static void delete(final File[] list) throws IOException {
258 		final long begin = list[0].getParentFile().lastModified();
259 		for (final File f : list) {
260 			FileUtils.delete(f);
261 			assertFalse(f + " was removed", f.exists());
262 		}
263 		touch(begin, list[0].getParentFile());
264 	}
265 
266 	private static void touch(final long begin, final File dir) {
267 		while (begin >= dir.lastModified()) {
268 			try {
269 				Thread.sleep(25);
270 			} catch (InterruptedException ie) {
271 				//
272 			}
273 			dir.setLastModified(System.currentTimeMillis());
274 		}
275 	}
276 
277 	private File fullPackFileName(final ObjectId name, final String suffix) {
278 		final File packdir = new File(db.getObjectDatabase().getDirectory(), "pack");
279 		return new File(packdir, "pack-" + name.name() + suffix);
280 	}
281 
282 	private RevObject writeBlob(final Repository repo, final String data)
283 			throws IOException {
284 		final byte[] bytes = Constants.encode(data);
285 		final ObjectId id;
286 		try (ObjectInserter inserter = repo.newObjectInserter()) {
287 			id = inserter.insert(Constants.OBJ_BLOB, bytes);
288 			inserter.flush();
289 		}
290 		try {
291 			parse(id);
292 			fail("Object " + id.name() + " should not exist in test repository");
293 		} catch (MissingObjectException e) {
294 			// Ok
295 		}
296 		try (RevWalk revWalk = new RevWalk(repo)) {
297 			return revWalk.lookupBlob(id);
298 		}
299 	}
300 }