View Javadoc
1   /*
2    * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  
11  package org.eclipse.jgit.transport;
12  
13  import static org.eclipse.jgit.transport.WalkRemoteObjectDatabase.ROOT_DIR;
14  
15  import java.io.BufferedOutputStream;
16  import java.io.IOException;
17  import java.io.OutputStream;
18  import java.util.ArrayList;
19  import java.util.Collection;
20  import java.util.HashSet;
21  import java.util.LinkedHashMap;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Set;
25  import java.util.TreeMap;
26  
27  import org.eclipse.jgit.errors.TransportException;
28  import org.eclipse.jgit.internal.JGitText;
29  import org.eclipse.jgit.internal.storage.pack.PackWriter;
30  import org.eclipse.jgit.lib.AnyObjectId;
31  import org.eclipse.jgit.lib.Constants;
32  import org.eclipse.jgit.lib.ObjectId;
33  import org.eclipse.jgit.lib.ObjectIdRef;
34  import org.eclipse.jgit.lib.ProgressMonitor;
35  import org.eclipse.jgit.lib.Ref;
36  import org.eclipse.jgit.lib.Ref.Storage;
37  import org.eclipse.jgit.lib.RefWriter;
38  import org.eclipse.jgit.lib.Repository;
39  import org.eclipse.jgit.transport.RemoteRefUpdate.Status;
40  
41  /**
42   * Generic push support for dumb transport protocols.
43   * <p>
44   * Since there are no Git-specific smarts on the remote side of the connection
45   * the client side must handle everything on its own. The generic push support
46   * requires being able to delete, create and overwrite files on the remote side,
47   * as well as create any missing directories (if necessary). Typically this can
48   * be handled through an FTP style protocol.
49   * <p>
50   * Objects not on the remote side are uploaded as pack files, using one pack
51   * file per invocation. This simplifies the implementation as only two data
52   * files need to be written to the remote repository.
53   * <p>
54   * Push support supplied by this class is not multiuser safe. Concurrent pushes
55   * to the same repository may yield an inconsistent reference database which may
56   * confuse fetch clients.
57   * <p>
58   * A single push is concurrently safe with multiple fetch requests, due to the
59   * careful order of operations used to update the repository. Clients fetching
60   * may receive transient failures due to short reads on certain files if the
61   * protocol does not support atomic file replacement.
62   *
63   * @see WalkRemoteObjectDatabase
64   */
65  class WalkPushConnection extends BaseConnection implements PushConnection {
66  	/** The repository this transport pushes out of. */
67  	private final Repository local;
68  
69  	/** Location of the remote repository we are writing to. */
70  	private final URIish uri;
71  
72  	/** Database connection to the remote repository. */
73  	final WalkRemoteObjectDatabase dest;
74  
75  	/** The configured transport we were constructed by. */
76  	private final Transport transport;
77  
78  	/**
79  	 * Packs already known to reside in the remote repository.
80  	 * <p>
81  	 * This is a LinkedHashMap to maintain the original order.
82  	 */
83  	private LinkedHashMap<String, String> packNames;
84  
85  	/** Complete listing of refs the remote will have after our push. */
86  	private Map<String, Ref> newRefs;
87  
88  	/**
89  	 * Updates which require altering the packed-refs file to complete.
90  	 * <p>
91  	 * If this collection is non-empty then any refs listed in {@link #newRefs}
92  	 * with a storage class of {@link Storage#PACKED} will be written.
93  	 */
94  	private Collection<RemoteRefUpdate> packedRefUpdates;
95  
96  	WalkPushConnection(final WalkTransport walkTransport,
97  			final WalkRemoteObjectDatabase w) {
98  		transport = (Transport) walkTransport;
99  		local = transport.local;
100 		uri = transport.getURI();
101 		dest = w;
102 	}
103 
104 	/** {@inheritDoc} */
105 	@Override
106 	public void push(final ProgressMonitor monitor,
107 			final Map<String, RemoteRefUpdate> refUpdates)
108 			throws TransportException {
109 		push(monitor, refUpdates, null);
110 	}
111 
112 	/** {@inheritDoc} */
113 	@Override
114 	public void push(final ProgressMonitor monitor,
115 			final Map<String, RemoteRefUpdate> refUpdates, OutputStream out)
116 			throws TransportException {
117 		markStartedOperation();
118 		packNames = null;
119 		newRefs = new TreeMap<>(getRefsMap());
120 		packedRefUpdates = new ArrayList<>(refUpdates.size());
121 
122 		// Filter the commands and issue all deletes first. This way we
123 		// can correctly handle a directory being cleared out and a new
124 		// ref using the directory name being created.
125 		//
126 		final List<RemoteRefUpdate> updates = new ArrayList<>();
127 		for (RemoteRefUpdate u : refUpdates.values()) {
128 			final String n = u.getRemoteName();
129 			if (!n.startsWith("refs/") || !Repository.isValidRefName(n)) { //$NON-NLS-1$
130 				u.setStatus(Status.REJECTED_OTHER_REASON);
131 				u.setMessage(JGitText.get().funnyRefname);
132 				continue;
133 			}
134 
135 			if (AnyObjectId.isEqual(ObjectId.zeroId(), u.getNewObjectId()))
136 				deleteCommand(u);
137 			else
138 				updates.add(u);
139 		}
140 
141 		// If we have any updates we need to upload the objects first, to
142 		// prevent creating refs pointing at non-existent data. Then we
143 		// can update the refs, and the info-refs file for dumb transports.
144 		//
145 		if (!updates.isEmpty())
146 			sendpack(updates, monitor);
147 		for (RemoteRefUpdate u : updates)
148 			updateCommand(u);
149 
150 		// Is this a new repository? If so we should create additional
151 		// metadata files so it is properly initialized during the push.
152 		//
153 		if (!updates.isEmpty() && isNewRepository())
154 			createNewRepository(updates);
155 
156 		RefWriter refWriter = new RefWriter(newRefs.values()) {
157 			@Override
158 			protected void writeFile(String file, byte[] content)
159 					throws IOException {
160 				dest.writeFile(ROOT_DIR + file, content);
161 			}
162 		};
163 		if (!packedRefUpdates.isEmpty()) {
164 			try {
165 				refWriter.writePackedRefs();
166 				for (RemoteRefUpdate u : packedRefUpdates)
167 					u.setStatus(Status.OK);
168 			} catch (IOException err) {
169 				for (RemoteRefUpdate u : packedRefUpdates) {
170 					u.setStatus(Status.REJECTED_OTHER_REASON);
171 					u.setMessage(err.getMessage());
172 				}
173 				throw new TransportException(uri, JGitText.get().failedUpdatingRefs, err);
174 			}
175 		}
176 
177 		try {
178 			refWriter.writeInfoRefs();
179 		} catch (IOException err) {
180 			throw new TransportException(uri, JGitText.get().failedUpdatingRefs, err);
181 		}
182 	}
183 
184 	/** {@inheritDoc} */
185 	@Override
186 	public void close() {
187 		dest.close();
188 	}
189 
190 	private void sendpack(final List<RemoteRefUpdate> updates,
191 			final ProgressMonitor monitor) throws TransportException {
192 		String pathPack = null;
193 		String pathIdx = null;
194 
195 		try (PackWritere/pack/PackWriter.html#PackWriter">PackWriter writer = new PackWriter(transport.getPackConfig(),
196 				local.newObjectReader())) {
197 
198 			final Set<ObjectId> need = new HashSet<>();
199 			final Set<ObjectId> have = new HashSet<>();
200 			for (RemoteRefUpdate r : updates)
201 				need.add(r.getNewObjectId());
202 			for (Ref r : getRefs()) {
203 				have.add(r.getObjectId());
204 				if (r.getPeeledObjectId() != null)
205 					have.add(r.getPeeledObjectId());
206 			}
207 			writer.preparePack(monitor, need, have);
208 
209 			// We don't have to continue further if the pack will
210 			// be an empty pack, as the remote has all objects it
211 			// needs to complete this change.
212 			//
213 			if (writer.getObjectCount() == 0)
214 				return;
215 
216 			packNames = new LinkedHashMap<>();
217 			for (String n : dest.getPackNames())
218 				packNames.put(n, n);
219 
220 			final String base = "pack-" + writer.computeName().name(); //$NON-NLS-1$
221 			final String packName = base + ".pack"; //$NON-NLS-1$
222 			pathPack = "pack/" + packName; //$NON-NLS-1$
223 			pathIdx = "pack/" + base + ".idx"; //$NON-NLS-1$ //$NON-NLS-2$
224 
225 			if (packNames.remove(packName) != null) {
226 				// The remote already contains this pack. We should
227 				// remove the index before overwriting to prevent bad
228 				// offsets from appearing to clients.
229 				//
230 				dest.writeInfoPacks(packNames.keySet());
231 				dest.deleteFile(pathIdx);
232 			}
233 
234 			// Write the pack file, then the index, as readers look the
235 			// other direction (index, then pack file).
236 			//
237 			String wt = "Put " + base.substring(0, 12); //$NON-NLS-1$
238 			try (OutputStream os = new BufferedOutputStream(
239 					dest.writeFile(pathPack, monitor, wt + "..pack"))) { //$NON-NLS-1$
240 				writer.writePack(monitor, monitor, os);
241 			}
242 
243 			try (OutputStream os = new BufferedOutputStream(
244 					dest.writeFile(pathIdx, monitor, wt + "..idx"))) { //$NON-NLS-1$
245 				writer.writeIndex(os);
246 			}
247 
248 			// Record the pack at the start of the pack info list. This
249 			// way clients are likely to consult the newest pack first,
250 			// and discover the most recent objects there.
251 			//
252 			final ArrayList<String> infoPacks = new ArrayList<>();
253 			infoPacks.add(packName);
254 			infoPacks.addAll(packNames.keySet());
255 			dest.writeInfoPacks(infoPacks);
256 
257 		} catch (IOException err) {
258 			safeDelete(pathIdx);
259 			safeDelete(pathPack);
260 
261 			throw new TransportException(uri, JGitText.get().cannotStoreObjects, err);
262 		}
263 	}
264 
265 	private void safeDelete(String path) {
266 		if (path != null) {
267 			try {
268 				dest.deleteFile(path);
269 			} catch (IOException cleanupFailure) {
270 				// Ignore the deletion failure. We probably are
271 				// already failing and were just trying to pick
272 				// up after ourselves.
273 			}
274 		}
275 	}
276 
277 	private void deleteCommand(RemoteRefUpdate u) {
278 		final Ref r = newRefs.remove(u.getRemoteName());
279 		if (r == null) {
280 			// Already gone.
281 			//
282 			u.setStatus(Status.OK);
283 			return;
284 		}
285 
286 		if (r.getStorage().isPacked())
287 			packedRefUpdates.add(u);
288 
289 		if (r.getStorage().isLoose()) {
290 			try {
291 				dest.deleteRef(u.getRemoteName());
292 				u.setStatus(Status.OK);
293 			} catch (IOException e) {
294 				u.setStatus(Status.REJECTED_OTHER_REASON);
295 				u.setMessage(e.getMessage());
296 			}
297 		}
298 
299 		try {
300 			dest.deleteRefLog(u.getRemoteName());
301 		} catch (IOException e) {
302 			u.setStatus(Status.REJECTED_OTHER_REASON);
303 			u.setMessage(e.getMessage());
304 		}
305 	}
306 
307 	private void updateCommand(RemoteRefUpdate u) {
308 		try {
309 			dest.writeRef(u.getRemoteName(), u.getNewObjectId());
310 			newRefs.put(u.getRemoteName(), new ObjectIdRef.Unpeeled(
311 					Storage.LOOSE, u.getRemoteName(), u.getNewObjectId()));
312 			u.setStatus(Status.OK);
313 		} catch (IOException e) {
314 			u.setStatus(Status.REJECTED_OTHER_REASON);
315 			u.setMessage(e.getMessage());
316 		}
317 	}
318 
319 	private boolean isNewRepository() {
320 		return getRefsMap().isEmpty() && packNames != null
321 				&& packNames.isEmpty();
322 	}
323 
324 	private void createNewRepository(List<RemoteRefUpdate> updates)
325 			throws TransportException {
326 		try {
327 			final String ref = "ref: " + pickHEAD(updates) + "\n"; //$NON-NLS-1$ //$NON-NLS-2$
328 			final byte[] bytes = Constants.encode(ref);
329 			dest.writeFile(ROOT_DIR + Constants.HEAD, bytes);
330 		} catch (IOException e) {
331 			throw new TransportException(uri, JGitText.get().cannotCreateHEAD, e);
332 		}
333 
334 		try {
335 			final String config = "[core]\n" //$NON-NLS-1$
336 					+ "\trepositoryformatversion = 0\n"; //$NON-NLS-1$
337 			final byte[] bytes = Constants.encode(config);
338 			dest.writeFile(ROOT_DIR + Constants.CONFIG, bytes);
339 		} catch (IOException e) {
340 			throw new TransportException(uri, JGitText.get().cannotCreateConfig, e);
341 		}
342 	}
343 
344 	private static String pickHEAD(List<RemoteRefUpdate> updates) {
345 		// Try to use master if the user is pushing that, it is the
346 		// default branch and is likely what they want to remain as
347 		// the default on the new remote.
348 		//
349 		for (RemoteRefUpdate u : updates) {
350 			final String n = u.getRemoteName();
351 			if (n.equals(Constants.R_HEADS + Constants.MASTER))
352 				return n;
353 		}
354 
355 		// Pick any branch, under the assumption the user pushed only
356 		// one to the remote side.
357 		//
358 		for (RemoteRefUpdate u : updates) {
359 			final String n = u.getRemoteName();
360 			if (n.startsWith(Constants.R_HEADS))
361 				return n;
362 		}
363 		return updates.get(0).getRemoteName();
364 	}
365 }