View Javadoc
1   /*
2    * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com>
3    * and other copyright owners as documented in the project's IP log.
4    *
5    * This program and the accompanying materials are made available
6    * under the terms of the Eclipse Distribution License v1.0 which
7    * accompanies this distribution, is reproduced below, and is
8    * available at http://www.eclipse.org/org/documents/edl-v10.php
9    *
10   * All rights reserved.
11   *
12   * Redistribution and use in source and binary forms, with or
13   * without modification, are permitted provided that the following
14   * conditions are met:
15   *
16   * - Redistributions of source code must retain the above copyright
17   *   notice, this list of conditions and the following disclaimer.
18   *
19   * - Redistributions in binary form must reproduce the above
20   *   copyright notice, this list of conditions and the following
21   *   disclaimer in the documentation and/or other materials provided
22   *   with the distribution.
23   *
24   * - Neither the name of the Eclipse Foundation, Inc. nor the
25   *   names of its contributors may be used to endorse or promote
26   *   products derived from this software without specific prior
27   *   written permission.
28   *
29   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42   */
43  package org.eclipse.jgit.lfs.internal;
44  
45  import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
46  import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
47  import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
48  import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE;
49  
50  import java.io.BufferedReader;
51  import java.io.IOException;
52  import java.io.InputStreamReader;
53  import java.net.ProxySelector;
54  import java.net.URL;
55  import java.text.SimpleDateFormat;
56  import java.util.LinkedList;
57  import java.util.Map;
58  import java.util.TreeMap;
59  
60  import org.eclipse.jgit.annotations.NonNull;
61  import org.eclipse.jgit.lfs.LfsPointer;
62  import org.eclipse.jgit.lfs.Protocol;
63  import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException;
64  import org.eclipse.jgit.lib.ConfigConstants;
65  import org.eclipse.jgit.lib.Repository;
66  import org.eclipse.jgit.lib.StoredConfig;
67  import org.eclipse.jgit.transport.HttpConfig;
68  import org.eclipse.jgit.transport.HttpTransport;
69  import org.eclipse.jgit.transport.RemoteSession;
70  import org.eclipse.jgit.transport.SshSessionFactory;
71  import org.eclipse.jgit.transport.URIish;
72  import org.eclipse.jgit.transport.http.HttpConnection;
73  import org.eclipse.jgit.util.FS;
74  import org.eclipse.jgit.util.HttpSupport;
75  import org.eclipse.jgit.util.io.MessageWriter;
76  import org.eclipse.jgit.util.io.StreamCopyThread;
77  
78  /**
79   * Provides means to get a valid LFS connection for a given repository.
80   */
81  public class LfsConnectionFactory {
82  
83  	private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$
84  	private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$
85  	private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>();
86  
87  	/**
88  	 * Determine URL of LFS server by looking into config parameters lfs.url,
89  	 * lfs.[remote].url or remote.[remote].url. The LFS server URL is computed
90  	 * from remote.[remote].url by appending "/info/lfs". In case there is no
91  	 * URL configured, a SSH remote URI can be used to auto-detect the LFS URI
92  	 * by using the remote "git-lfs-authenticate" command.
93  	 *
94  	 * @param db
95  	 *            the repository to work with
96  	 * @param method
97  	 *            the method (GET,PUT,...) of the request this connection will
98  	 *            be used for
99  	 * @param purpose
100 	 *            the action, e.g. Protocol.OPERATION_DOWNLOAD
101 	 * @return the url for the lfs server. e.g.
102 	 *         "https://github.com/github/git-lfs.git/info/lfs"
103 	 * @throws IOException
104 	 */
105 	public static HttpConnection getLfsConnection(Repository db, String method,
106 			String purpose) throws IOException {
107 		StoredConfig config = db.getConfig();
108 		Map<String, String> additionalHeaders = new TreeMap<>();
109 		String lfsUrl = getLfsUrl(db, purpose, additionalHeaders);
110 		URL url = new URL(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT);
111 		HttpConnection connection = HttpTransport.getConnectionFactory().create(
112 				url, HttpSupport.proxyFor(ProxySelector.getDefault(), url));
113 		connection.setDoOutput(true);
114 		if (url.getProtocol().equals(SCHEME_HTTPS)
115 				&& !config.getBoolean(HttpConfig.HTTP,
116 						HttpConfig.SSL_VERIFY_KEY, true)) {
117 			HttpSupport.disableSslVerify(connection);
118 		}
119 		connection.setRequestMethod(method);
120 		connection.setRequestProperty(HDR_ACCEPT,
121 				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
122 		connection.setRequestProperty(HDR_CONTENT_TYPE,
123 				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
124 		additionalHeaders
125 				.forEach((k, v) -> connection.setRequestProperty(k, v));
126 		return connection;
127 	}
128 
129 	private static String getLfsUrl(Repository db, String purpose,
130 			Map<String, String> additionalHeaders)
131 			throws LfsConfigInvalidException {
132 		StoredConfig config = db.getConfig();
133 		String lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
134 				null,
135 				ConfigConstants.CONFIG_KEY_URL);
136 		if (lfsUrl == null) {
137 			String remoteUrl = null;
138 			for (String remote : db.getRemoteNames()) {
139 				lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
140 						remote,
141 						ConfigConstants.CONFIG_KEY_URL);
142 				// This could be done better (more precise logic), but according
143 				// to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs
144 				// generally only supports 'origin' in an integrated workflow.
145 				if (lfsUrl == null && (remote.equals(
146 						org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME))) {
147 					remoteUrl = config.getString(
148 							ConfigConstants.CONFIG_KEY_REMOTE, remote,
149 							ConfigConstants.CONFIG_KEY_URL);
150 				}
151 				break;
152 			}
153 			if (lfsUrl == null && remoteUrl != null) {
154 				lfsUrl = discoverLfsUrl(db, purpose, additionalHeaders,
155 						remoteUrl);
156 			} else {
157 				lfsUrl = lfsUrl + Protocol.INFO_LFS_ENDPOINT;
158 			}
159 		}
160 		if (lfsUrl == null) {
161 			throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl);
162 		}
163 		return lfsUrl;
164 	}
165 
166 	private static String discoverLfsUrl(Repository db, String purpose,
167 			Map<String, String> additionalHeaders, String remoteUrl) {
168 		try {
169 			URIish u = new URIish(remoteUrl);
170 			if (SCHEME_SSH.equals(u.getScheme())) {
171 				Protocol.ExpiringAction action = getSshAuthentication(
172 						db, purpose, remoteUrl, u);
173 				additionalHeaders.putAll(action.header);
174 				return action.href;
175 			} else {
176 				return remoteUrl + Protocol.INFO_LFS_ENDPOINT;
177 			}
178 		} catch (Exception e) {
179 			return null; // could not discover
180 		}
181 	}
182 
183 	private static Protocol.ExpiringAction getSshAuthentication(
184 			Repository db, String purpose, String remoteUrl, URIish u)
185 			throws IOException {
186 		AuthCache cached = sshAuthCache.get(remoteUrl);
187 		Protocol.ExpiringAction action = null;
188 		if (cached != null && cached.validUntil > System.currentTimeMillis()) {
189 			action = cached.cachedAction;
190 		}
191 
192 		if (action == null) {
193 			// discover and authenticate; git-lfs does "ssh
194 			// -p <port> -- <host> git-lfs-authenticate
195 			// <project> <upload/download>"
196 			String json = runSshCommand(u.setPath(""), //$NON-NLS-1$
197 					db.getFS(),
198 					"git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$
199 							+ purpose);
200 
201 			action = Protocol.gson().fromJson(json,
202 					Protocol.ExpiringAction.class);
203 
204 			// cache the result as long as possible.
205 			AuthCache c = new AuthCache(action);
206 			sshAuthCache.put(remoteUrl, c);
207 		}
208 		return action;
209 	}
210 
211 	/**
212 	 * Create a connection for the specified
213 	 * {@link org.eclipse.jgit.lfs.Protocol.Action}.
214 	 *
215 	 * @param repo
216 	 *            the repo to fetch required configuration from
217 	 * @param action
218 	 *            the action for which to create a connection
219 	 * @param method
220 	 *            the target method (GET or PUT)
221 	 * @return a connection. output mode is not set.
222 	 * @throws IOException
223 	 *             in case of any error.
224 	 */
225 	public static @NonNull HttpConnection getLfsContentConnection(
226 			Repository repo, Protocol.Action action, String method)
227 			throws IOException {
228 		URL contentUrl = new URL(action.href);
229 		HttpConnection contentServerConn = HttpTransport.getConnectionFactory()
230 				.create(contentUrl, HttpSupport
231 						.proxyFor(ProxySelector.getDefault(), contentUrl));
232 		contentServerConn.setRequestMethod(method);
233 		action.header
234 				.forEach((k, v) -> contentServerConn.setRequestProperty(k, v));
235 		if (contentUrl.getProtocol().equals(SCHEME_HTTPS)
236 				&& !repo.getConfig().getBoolean(HttpConfig.HTTP,
237 						HttpConfig.SSL_VERIFY_KEY, true)) {
238 			HttpSupport.disableSslVerify(contentServerConn);
239 		}
240 
241 		contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING,
242 				ENCODING_GZIP);
243 
244 		return contentServerConn;
245 	}
246 
247 	private static String extractProjectName(URIish u) {
248 		String path = u.getPath().substring(1);
249 		if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) {
250 			return path.substring(0, path.length() - 4);
251 		} else {
252 			return path;
253 		}
254 	}
255 
256 	private static String runSshCommand(URIish sshUri, FS fs, String command)
257 			throws IOException {
258 		RemoteSession session = null;
259 		Process process = null;
260 		StreamCopyThread errorThread = null;
261 		try (MessageWriter stderr = new MessageWriter()) {
262 			session = SshSessionFactory.getInstance().getSession(sshUri, null,
263 					fs, 5_000);
264 			process = session.exec(command, 0);
265 			errorThread = new StreamCopyThread(process.getErrorStream(),
266 					stderr.getRawStream());
267 			errorThread.start();
268 			try (BufferedReader reader = new BufferedReader(
269 					new InputStreamReader(process.getInputStream(),
270 							org.eclipse.jgit.lib.Constants.CHARSET))) {
271 				return reader.readLine();
272 			}
273 		} finally {
274 			if (process != null) {
275 				process.destroy();
276 			}
277 			if (errorThread != null) {
278 				try {
279 					errorThread.halt();
280 				} catch (InterruptedException e) {
281 					// Stop waiting and return anyway.
282 				} finally {
283 					errorThread = null;
284 				}
285 			}
286 			if (session != null) {
287 				SshSessionFactory.getInstance().releaseSession(session);
288 			}
289 		}
290 	}
291 
292 	/**
293 	 * @param operation
294 	 *            the operation to perform, e.g. Protocol.OPERATION_DOWNLOAD
295 	 * @param resources
296 	 *            the LFS resources affected
297 	 * @return a request that can be serialized to JSON
298 	 */
299 	public static Protocol.Request toRequest(String operation,
300 			LfsPointer... resources) {
301 		Protocol.Request req = new Protocol.Request();
302 		req.operation = operation;
303 		if (resources != null) {
304 			req.objects = new LinkedList<>();
305 			for (LfsPointer res : resources) {
306 				Protocol.ObjectSpec o = new Protocol.ObjectSpec();
307 				o.oid = res.getOid().getName();
308 				o.size = res.getSize();
309 				req.objects.add(o);
310 			}
311 		}
312 		return req;
313 	}
314 
315 	private static final class AuthCache {
316 		private static final long AUTH_CACHE_EAGER_TIMEOUT = 100;
317 
318 		private static final SimpleDateFormat ISO_FORMAT = new SimpleDateFormat(
319 				"yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$
320 
321 		/**
322 		 * Creates a cache entry for an authentication response.
323 		 * <p>
324 		 * The timeout of the cache token is extracted from the given action. If
325 		 * no timeout can be determined, the token will be used only once.
326 		 *
327 		 * @param action
328 		 */
329 		public AuthCache(Protocol.ExpiringAction action) {
330 			this.cachedAction = action;
331 			try {
332 				if (action.expiresIn != null && !action.expiresIn.isEmpty()) {
333 					this.validUntil = System.currentTimeMillis()
334 							+ Long.parseLong(action.expiresIn);
335 				} else if (action.expiresAt != null
336 						&& !action.expiresAt.isEmpty()) {
337 					this.validUntil = ISO_FORMAT.parse(action.expiresAt)
338 							.getTime() - AUTH_CACHE_EAGER_TIMEOUT;
339 				} else {
340 					this.validUntil = System.currentTimeMillis();
341 				}
342 			} catch (Exception e) {
343 				this.validUntil = System.currentTimeMillis();
344 			}
345 		}
346 
347 		long validUntil;
348 
349 		Protocol.ExpiringAction cachedAction;
350 	}
351 
352 }