View Javadoc
1   /*
2    * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  package org.eclipse.jgit.lfs.internal;
11  
12  import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
13  import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
14  import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
15  import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE;
16  
17  import java.io.IOException;
18  import java.net.ProxySelector;
19  import java.net.URISyntaxException;
20  import java.net.URL;
21  import java.text.SimpleDateFormat;
22  import java.util.LinkedList;
23  import java.util.Map;
24  import java.util.TreeMap;
25  
26  import org.eclipse.jgit.annotations.NonNull;
27  import org.eclipse.jgit.errors.CommandFailedException;
28  import org.eclipse.jgit.lfs.LfsPointer;
29  import org.eclipse.jgit.lfs.Protocol;
30  import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException;
31  import org.eclipse.jgit.lib.ConfigConstants;
32  import org.eclipse.jgit.lib.Repository;
33  import org.eclipse.jgit.lib.StoredConfig;
34  import org.eclipse.jgit.transport.HttpConfig;
35  import org.eclipse.jgit.transport.HttpTransport;
36  import org.eclipse.jgit.transport.URIish;
37  import org.eclipse.jgit.transport.http.HttpConnection;
38  import org.eclipse.jgit.util.HttpSupport;
39  import org.eclipse.jgit.util.SshSupport;
40  
41  /**
42   * Provides means to get a valid LFS connection for a given repository.
43   */
44  public class LfsConnectionFactory {
45  
46  	private static final int SSH_AUTH_TIMEOUT_SECONDS = 30;
47  	private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$
48  	private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$
49  	private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>();
50  
51  	/**
52  	 * Determine URL of LFS server by looking into config parameters lfs.url,
53  	 * lfs.[remote].url or remote.[remote].url. The LFS server URL is computed
54  	 * from remote.[remote].url by appending "/info/lfs". In case there is no
55  	 * URL configured, a SSH remote URI can be used to auto-detect the LFS URI
56  	 * by using the remote "git-lfs-authenticate" command.
57  	 *
58  	 * @param db
59  	 *            the repository to work with
60  	 * @param method
61  	 *            the method (GET,PUT,...) of the request this connection will
62  	 *            be used for
63  	 * @param purpose
64  	 *            the action, e.g. Protocol.OPERATION_DOWNLOAD
65  	 * @return the url for the lfs server. e.g.
66  	 *         "https://github.com/github/git-lfs.git/info/lfs"
67  	 * @throws IOException
68  	 */
69  	public static HttpConnection getLfsConnection(Repository db, String method,
70  			String purpose) throws IOException {
71  		StoredConfig config = db.getConfig();
72  		Map<String, String> additionalHeaders = new TreeMap<>();
73  		String lfsUrl = getLfsUrl(db, purpose, additionalHeaders);
74  		URL url = new URL(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT);
75  		HttpConnection connection = HttpTransport.getConnectionFactory().create(
76  				url, HttpSupport.proxyFor(ProxySelector.getDefault(), url));
77  		connection.setDoOutput(true);
78  		if (url.getProtocol().equals(SCHEME_HTTPS)
79  				&& !config.getBoolean(HttpConfig.HTTP,
80  						HttpConfig.SSL_VERIFY_KEY, true)) {
81  			HttpSupport.disableSslVerify(connection);
82  		}
83  		connection.setRequestMethod(method);
84  		connection.setRequestProperty(HDR_ACCEPT,
85  				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
86  		connection.setRequestProperty(HDR_CONTENT_TYPE,
87  				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
88  		additionalHeaders
89  				.forEach((k, v) -> connection.setRequestProperty(k, v));
90  		return connection;
91  	}
92  
93  	private static String getLfsUrl(Repository db, String purpose,
94  			Map<String, String> additionalHeaders)
95  			throws LfsConfigInvalidException {
96  		StoredConfig config = db.getConfig();
97  		String lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
98  				null,
99  				ConfigConstants.CONFIG_KEY_URL);
100 		Exception ex = null;
101 		if (lfsUrl == null) {
102 			String remoteUrl = null;
103 			for (String remote : db.getRemoteNames()) {
104 				lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
105 						remote,
106 						ConfigConstants.CONFIG_KEY_URL);
107 				// This could be done better (more precise logic), but according
108 				// to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs
109 				// generally only supports 'origin' in an integrated workflow.
110 				if (lfsUrl == null && (remote.equals(
111 						org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME))) {
112 					remoteUrl = config.getString(
113 							ConfigConstants.CONFIG_KEY_REMOTE, remote,
114 							ConfigConstants.CONFIG_KEY_URL);
115 				}
116 				break;
117 			}
118 			if (lfsUrl == null && remoteUrl != null) {
119 				try {
120 					lfsUrl = discoverLfsUrl(db, purpose, additionalHeaders,
121 							remoteUrl);
122 				} catch (URISyntaxException | IOException
123 						| CommandFailedException e) {
124 					ex = e;
125 				}
126 			} else {
127 				lfsUrl = lfsUrl + Protocol.INFO_LFS_ENDPOINT;
128 			}
129 		}
130 		if (lfsUrl == null) {
131 			if (ex != null) {
132 				throw new LfsConfigInvalidException(
133 						LfsText.get().lfsNoDownloadUrl, ex);
134 			}
135 			throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl);
136 		}
137 		return lfsUrl;
138 	}
139 
140 	private static String discoverLfsUrl(Repository db, String purpose,
141 			Map<String, String> additionalHeaders, String remoteUrl)
142 			throws URISyntaxException, IOException, CommandFailedException {
143 		URIish u = new URIish(remoteUrl);
144 		if (u.getScheme() == null || SCHEME_SSH.equals(u.getScheme())) {
145 			Protocol.ExpiringAction action = getSshAuthentication(db, purpose,
146 					remoteUrl, u);
147 			additionalHeaders.putAll(action.header);
148 			return action.href;
149 		}
150 		return remoteUrl + Protocol.INFO_LFS_ENDPOINT;
151 	}
152 
153 	private static Protocol.ExpiringAction getSshAuthentication(
154 			Repository db, String purpose, String remoteUrl, URIish u)
155 			throws IOException, CommandFailedException {
156 		AuthCache cached = sshAuthCache.get(remoteUrl);
157 		Protocol.ExpiringAction action = null;
158 		if (cached != null && cached.validUntil > System.currentTimeMillis()) {
159 			action = cached.cachedAction;
160 		}
161 
162 		if (action == null) {
163 			// discover and authenticate; git-lfs does "ssh
164 			// -p <port> -- <host> git-lfs-authenticate
165 			// <project> <upload/download>"
166 			String json = SshSupport.runSshCommand(u.setPath(""), //$NON-NLS-1$
167 					null, db.getFS(),
168 					"git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$
169 							+ purpose,
170 					SSH_AUTH_TIMEOUT_SECONDS);
171 
172 			action = Protocol.gson().fromJson(json,
173 					Protocol.ExpiringAction.class);
174 
175 			// cache the result as long as possible.
176 			AuthCache c = new AuthCache(action);
177 			sshAuthCache.put(remoteUrl, c);
178 		}
179 		return action;
180 	}
181 
182 	/**
183 	 * Create a connection for the specified
184 	 * {@link org.eclipse.jgit.lfs.Protocol.Action}.
185 	 *
186 	 * @param repo
187 	 *            the repo to fetch required configuration from
188 	 * @param action
189 	 *            the action for which to create a connection
190 	 * @param method
191 	 *            the target method (GET or PUT)
192 	 * @return a connection. output mode is not set.
193 	 * @throws IOException
194 	 *             in case of any error.
195 	 */
196 	@NonNull
197 	public static HttpConnection getLfsContentConnection(
198 			Repository repo, Protocol.Action action, String method)
199 			throws IOException {
200 		URL contentUrl = new URL(action.href);
201 		HttpConnection contentServerConn = HttpTransport.getConnectionFactory()
202 				.create(contentUrl, HttpSupport
203 						.proxyFor(ProxySelector.getDefault(), contentUrl));
204 		contentServerConn.setRequestMethod(method);
205 		if (action.header != null) {
206 			action.header.forEach(
207 					(k, v) -> contentServerConn.setRequestProperty(k, v));
208 		}
209 		if (contentUrl.getProtocol().equals(SCHEME_HTTPS)
210 				&& !repo.getConfig().getBoolean(HttpConfig.HTTP,
211 						HttpConfig.SSL_VERIFY_KEY, true)) {
212 			HttpSupport.disableSslVerify(contentServerConn);
213 		}
214 
215 		contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING,
216 				ENCODING_GZIP);
217 
218 		return contentServerConn;
219 	}
220 
221 	private static String extractProjectName(URIish u) {
222 		String path = u.getPath();
223 
224 		// begins with a slash if the url contains a port (gerrit vs. github).
225 		if (path.startsWith("/")) { //$NON-NLS-1$
226 			path = path.substring(1);
227 		}
228 
229 		if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) {
230 			return path.substring(0, path.length() - 4);
231 		}
232 		return path;
233 	}
234 
235 	/**
236 	 * @param operation
237 	 *            the operation to perform, e.g. Protocol.OPERATION_DOWNLOAD
238 	 * @param resources
239 	 *            the LFS resources affected
240 	 * @return a request that can be serialized to JSON
241 	 */
242 	public static Protocol.Request toRequest(String operation,
243 			LfsPointer... resources) {
244 		Protocol.Request req = new Protocol.Request();
245 		req.operation = operation;
246 		if (resources != null) {
247 			req.objects = new LinkedList<>();
248 			for (LfsPointer res : resources) {
249 				Protocol.ObjectSpec o = new Protocol.ObjectSpec();
250 				o.oid = res.getOid().getName();
251 				o.size = res.getSize();
252 				req.objects.add(o);
253 			}
254 		}
255 		return req;
256 	}
257 
258 	private static final class AuthCache {
259 		private static final long AUTH_CACHE_EAGER_TIMEOUT = 500;
260 
261 		private static final SimpleDateFormat ISO_FORMAT = new SimpleDateFormat(
262 				"yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$
263 
264 		/**
265 		 * Creates a cache entry for an authentication response.
266 		 * <p>
267 		 * The timeout of the cache token is extracted from the given action. If
268 		 * no timeout can be determined, the token will be used only once.
269 		 *
270 		 * @param action
271 		 */
272 		public AuthCache(Protocol.ExpiringAction action) {
273 			this.cachedAction = action;
274 			try {
275 				if (action.expiresIn != null && !action.expiresIn.isEmpty()) {
276 					this.validUntil = (System.currentTimeMillis()
277 							+ Long.parseLong(action.expiresIn))
278 							- AUTH_CACHE_EAGER_TIMEOUT;
279 				} else if (action.expiresAt != null
280 						&& !action.expiresAt.isEmpty()) {
281 					this.validUntil = ISO_FORMAT.parse(action.expiresAt)
282 							.getTime() - AUTH_CACHE_EAGER_TIMEOUT;
283 				} else {
284 					this.validUntil = System.currentTimeMillis();
285 				}
286 			} catch (Exception e) {
287 				this.validUntil = System.currentTimeMillis();
288 			}
289 		}
290 
291 		long validUntil;
292 
293 		Protocol.ExpiringAction cachedAction;
294 	}
295 
296 }