View Javadoc
1   /*
2    * Copyright (C) 2009-2010, Google Inc.
3    * Copyright (C) 2009, Robin Rosenberg <robin.rosenberg@dewire.com> and others
4    *
5    * This program and the accompanying materials are made available under the
6    * terms of the Eclipse Distribution License v. 1.0 which is available at
7    * https://www.eclipse.org/org/documents/edl-v10.php.
8    *
9    * SPDX-License-Identifier: BSD-3-Clause
10   */
11  
12  package org.eclipse.jgit.internal.storage.file;
13  
14  import static org.junit.Assert.assertArrayEquals;
15  import static org.junit.Assert.assertEquals;
16  import static org.junit.Assert.assertFalse;
17  import static org.junit.Assert.assertNotNull;
18  import static org.junit.Assert.assertNotSame;
19  import static org.junit.Assert.fail;
20  
21  import java.io.BufferedOutputStream;
22  import java.io.File;
23  import java.io.FileOutputStream;
24  import java.io.IOException;
25  import java.io.OutputStream;
26  import java.time.Instant;
27  
28  import org.eclipse.jgit.errors.IncorrectObjectTypeException;
29  import org.eclipse.jgit.errors.MissingObjectException;
30  import org.eclipse.jgit.internal.storage.pack.PackWriter;
31  import org.eclipse.jgit.junit.RepositoryTestCase;
32  import org.eclipse.jgit.lib.AnyObjectId;
33  import org.eclipse.jgit.lib.Constants;
34  import org.eclipse.jgit.lib.NullProgressMonitor;
35  import org.eclipse.jgit.lib.ObjectId;
36  import org.eclipse.jgit.lib.ObjectInserter;
37  import org.eclipse.jgit.lib.ObjectLoader;
38  import org.eclipse.jgit.lib.Repository;
39  import org.eclipse.jgit.revwalk.RevObject;
40  import org.eclipse.jgit.revwalk.RevWalk;
41  import org.eclipse.jgit.storage.file.WindowCacheConfig;
42  import org.eclipse.jgit.util.FS;
43  import org.eclipse.jgit.util.FileUtils;
44  import org.junit.After;
45  import org.junit.Before;
46  import org.junit.Test;
47  
48  public class ConcurrentRepackTest extends RepositoryTestCase {
49  	@Override
50  	@Before
51  	public void setUp() throws Exception {
52  		WindowCacheConfig windowCacheConfig = new WindowCacheConfig();
53  		windowCacheConfig.setPackedGitOpenFiles(1);
54  		windowCacheConfig.install();
55  		super.setUp();
56  	}
57  
58  	@Override
59  	@After
60  	public void tearDown() throws Exception {
61  		super.tearDown();
62  		new WindowCacheConfig().install();
63  	}
64  
65  	@Test
66  	public void testObjectInNewPack() throws IncorrectObjectTypeException,
67  			IOException {
68  		// Create a new object in a new pack, and test that it is present.
69  		//
70  		final Repository eden = createBareRepository();
71  		final RevObject o1 = writeBlob(eden, "o1");
72  		pack(eden, o1);
73  		assertEquals(o1.name(), parse(o1).name());
74  	}
75  
76  	@Test
77  	public void testObjectMovedToNewPack1()
78  			throws IncorrectObjectTypeException, IOException {
79  		// Create an object and pack it. Then remove that pack and put the
80  		// object into a different pack file, with some other object. We
81  		// still should be able to access the objects.
82  		//
83  		final Repository eden = createBareRepository();
84  		final RevObject o1 = writeBlob(eden, "o1");
85  		final File[] out1 = pack(eden, o1);
86  		assertEquals(o1.name(), parse(o1).name());
87  
88  		final RevObject o2 = writeBlob(eden, "o2");
89  		pack(eden, o2, o1);
90  
91  		// Force close, and then delete, the old pack.
92  		//
93  		whackCache();
94  		delete(out1);
95  
96  		// Now here is the interesting thing. Will git figure the new
97  		// object exists in the new pack, and not the old one.
98  		//
99  		assertEquals(o2.name(), parse(o2).name());
100 		assertEquals(o1.name(), parse(o1).name());
101 	}
102 
103 	@Test
104 	public void testObjectMovedWithinPack()
105 			throws IncorrectObjectTypeException, IOException {
106 		// Create an object and pack it.
107 		//
108 		final Repository eden = createBareRepository();
109 		final RevObject o1 = writeBlob(eden, "o1");
110 		final File[] out1 = pack(eden, o1);
111 		assertEquals(o1.name(), parse(o1).name());
112 
113 		// Force close the old pack.
114 		//
115 		whackCache();
116 
117 		// Now overwrite the old pack in place. This method of creating a
118 		// different pack under the same file name is partially broken. We
119 		// should also have a different file name because the list of objects
120 		// within the pack has been modified.
121 		//
122 		final RevObject o2 = writeBlob(eden, "o2");
123 		try (PackWriter pw = new PackWriter(eden)) {
124 			pw.addObject(o2);
125 			pw.addObject(o1);
126 			write(out1, pw);
127 		}
128 
129 		// Try the old name, then the new name. The old name should cause the
130 		// pack to reload when it opens and the index and pack mismatch.
131 		//
132 		assertEquals(o1.name(), parse(o1).name());
133 		assertEquals(o2.name(), parse(o2).name());
134 	}
135 
136 	@Test
137 	public void testObjectMovedToNewPack2()
138 			throws IncorrectObjectTypeException, IOException {
139 		// Create an object and pack it. Then remove that pack and put the
140 		// object into a different pack file, with some other object. We
141 		// still should be able to access the objects.
142 		//
143 		final Repository eden = createBareRepository();
144 		final RevObject o1 = writeBlob(eden, "o1");
145 		final File[] out1 = pack(eden, o1);
146 		assertEquals(o1.name(), parse(o1).name());
147 
148 		final ObjectLoader load1 = db.open(o1, Constants.OBJ_BLOB);
149 		assertNotNull(load1);
150 
151 		final RevObject o2 = writeBlob(eden, "o2");
152 		pack(eden, o2, o1);
153 
154 		// Force close, and then delete, the old pack.
155 		//
156 		whackCache();
157 		delete(out1);
158 
159 		// Now here is the interesting thing... can the loader we made
160 		// earlier still resolve the object, even though its underlying
161 		// pack is gone, but the object still exists.
162 		//
163 		final ObjectLoader load2 = db.open(o1, Constants.OBJ_BLOB);
164 		assertNotNull(load2);
165 		assertNotSame(load1, load2);
166 
167 		final byte[] data2 = load2.getCachedBytes();
168 		final byte[] data1 = load1.getCachedBytes();
169 		assertNotNull(data2);
170 		assertNotNull(data1);
171 		assertNotSame(data1, data2); // cache should be per-pack, not per object
172 		assertArrayEquals(data1, data2);
173 		assertEquals(load2.getType(), load1.getType());
174 	}
175 
176 	private static void whackCache() {
177 		final WindowCacheConfig config = new WindowCacheConfig();
178 		config.setPackedGitOpenFiles(1);
179 		config.install();
180 	}
181 
182 	private RevObject parse(AnyObjectId id)
183 			throws MissingObjectException, IOException {
184 		try (RevWalk rw = new RevWalk(db)) {
185 			return rw.parseAny(id);
186 		}
187 	}
188 
189 	private File[] pack(Repository src, RevObject... list)
190 			throws IOException {
191 		try (PackWriter pw = new PackWriter(src)) {
192 			for (RevObject o : list) {
193 				pw.addObject(o);
194 			}
195 
196 			final ObjectId name = pw.computeName();
197 			final File packFile = fullPackFileName(name, ".pack");
198 			final File idxFile = fullPackFileName(name, ".idx");
199 			final File[] files = new File[] { packFile, idxFile };
200 			write(files, pw);
201 			return files;
202 		}
203 	}
204 
205 	private static void write(File[] files, PackWriter pw)
206 			throws IOException {
207 		final Instant begin = FS.DETECTED
208 				.lastModifiedInstant(files[0].getParentFile());
209 		NullProgressMonitor m = NullProgressMonitor.INSTANCE;
210 
211 		try (OutputStream out = new BufferedOutputStream(
212 				new FileOutputStream(files[0]))) {
213 			pw.writePack(m, m, out);
214 		}
215 
216 		try (OutputStream out = new BufferedOutputStream(
217 				new FileOutputStream(files[1]))) {
218 			pw.writeIndex(out);
219 		}
220 
221 		touch(begin, files[0].getParentFile());
222 	}
223 
224 	private static void delete(File[] list) throws IOException {
225 		final Instant begin = FS.DETECTED
226 				.lastModifiedInstant(list[0].getParentFile());
227 		for (File f : list) {
228 			FileUtils.delete(f);
229 			assertFalse(f + " was removed", f.exists());
230 		}
231 		touch(begin, list[0].getParentFile());
232 	}
233 
234 	private static void touch(Instant begin, File dir) throws IOException {
235 		while (begin.compareTo(FS.DETECTED.lastModifiedInstant(dir)) >= 0) {
236 			try {
237 				Thread.sleep(25);
238 			} catch (InterruptedException ie) {
239 				//
240 			}
241 			FS.DETECTED.setLastModified(dir.toPath(), Instant.now());
242 		}
243 	}
244 
245 	private File fullPackFileName(ObjectId name, String suffix) {
246 		final File packdir = db.getObjectDatabase().getPackDirectory();
247 		return new File(packdir, "pack-" + name.name() + suffix);
248 	}
249 
250 	private RevObject writeBlob(Repository repo, String data)
251 			throws IOException {
252 		final byte[] bytes = Constants.encode(data);
253 		final ObjectId id;
254 		try (ObjectInserter inserter = repo.newObjectInserter()) {
255 			id = inserter.insert(Constants.OBJ_BLOB, bytes);
256 			inserter.flush();
257 		}
258 		try {
259 			parse(id);
260 			fail("Object " + id.name() + " should not exist in test repository");
261 		} catch (MissingObjectException e) {
262 			// Ok
263 		}
264 		try (RevWalk revWalk = new RevWalk(repo)) {
265 			return revWalk.lookupBlob(id);
266 		}
267 	}
268 }