LfsConnectionFactory.java

  1. /*
  2.  * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> and others
  3.  *
  4.  * This program and the accompanying materials are made available under the
  5.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  6.  * https://www.eclipse.org/org/documents/edl-v10.php.
  7.  *
  8.  * SPDX-License-Identifier: BSD-3-Clause
  9.  */
  10. package org.eclipse.jgit.lfs.internal;

  12. import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
  13. import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
  14. import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
  15. import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE;

  17. import java.io.IOException;
  18. import java.net.ProxySelector;
  19. import java.net.URISyntaxException;
  20. import java.net.URL;
  21. import java.time.LocalDateTime;
  22. import java.time.ZoneOffset;
  23. import java.time.format.DateTimeFormatter;
  24. import java.util.LinkedList;
  25. import java.util.Map;
  26. import java.util.TreeMap;

  28. import org.eclipse.jgit.annotations.NonNull;
  29. import org.eclipse.jgit.errors.CommandFailedException;
  30. import org.eclipse.jgit.lfs.LfsPointer;
  31. import org.eclipse.jgit.lfs.Protocol;
  32. import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException;
  33. import org.eclipse.jgit.lib.ConfigConstants;
  34. import org.eclipse.jgit.lib.Repository;
  35. import org.eclipse.jgit.lib.StoredConfig;
  36. import org.eclipse.jgit.transport.HttpConfig;
  37. import org.eclipse.jgit.transport.HttpTransport;
  38. import org.eclipse.jgit.transport.URIish;
  39. import org.eclipse.jgit.transport.http.HttpConnection;
  40. import org.eclipse.jgit.util.HttpSupport;
  41. import org.eclipse.jgit.util.SshSupport;

  43. /**
  44.  * Provides means to get a valid LFS connection for a given repository.
  45.  */
  46. public class LfsConnectionFactory {

  48.     private static final int SSH_AUTH_TIMEOUT_SECONDS = 30;
  49.     private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$
  50.     private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$
  51.     private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>();

  53.     /**
  54.      * Determine URL of LFS server by looking into config parameters lfs.url,
  55.      * lfs.[remote].url or remote.[remote].url. The LFS server URL is computed
  56.      * from remote.[remote].url by appending "/info/lfs". In case there is no
  57.      * URL configured, a SSH remote URI can be used to auto-detect the LFS URI
  58.      * by using the remote "git-lfs-authenticate" command.
  59.      *
  60.      * @param db
  61.      *            the repository to work with
  62.      * @param method
  63.      *            the method (GET,PUT,...) of the request this connection will
  64.      *            be used for
  65.      * @param purpose
  66.      *            the action, e.g. Protocol.OPERATION_DOWNLOAD
  67.      * @return the url for the lfs server. e.g.
  68.      *         "https://github.com/github/git-lfs.git/info/lfs"
  69.      * @throws IOException
  70.      */
  71.     public static HttpConnection getLfsConnection(Repository db, String method,
  72.             String purpose) throws IOException {
  73.         StoredConfig config = db.getConfig();
  74.         Map<String, String> additionalHeaders = new TreeMap<>();
  75.         String lfsUrl = getLfsUrl(db, purpose, additionalHeaders);
  76.         URL url = new URL(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT);
  77.         HttpConnection connection = HttpTransport.getConnectionFactory().create(
  78.                 url, HttpSupport.proxyFor(ProxySelector.getDefault(), url));
  79.         connection.setDoOutput(true);
  80.         if (url.getProtocol().equals(SCHEME_HTTPS)
  81.                 && !config.getBoolean(HttpConfig.HTTP,
  82.                         HttpConfig.SSL_VERIFY_KEY, true)) {
  83.             HttpSupport.disableSslVerify(connection);
  84.         }
  85.         connection.setRequestMethod(method);
  86.         connection.setRequestProperty(HDR_ACCEPT,
  87.                 Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
  88.         connection.setRequestProperty(HDR_CONTENT_TYPE,
  89.                 Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
  90.         additionalHeaders
  91.                 .forEach((k, v) -> connection.setRequestProperty(k, v));
  92.         return connection;
  93.     }

  95.     private static String getLfsUrl(Repository db, String purpose,
  96.             Map<String, String> additionalHeaders)
  97.             throws LfsConfigInvalidException {
  98.         StoredConfig config = db.getConfig();
  99.         String lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
  100.                 null,
  101.                 ConfigConstants.CONFIG_KEY_URL);
  102.         Exception ex = null;
  103.         if (lfsUrl == null) {
  104.             String remoteUrl = null;
  105.             for (String remote : db.getRemoteNames()) {
  106.                 lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
  107.                         remote,
  108.                         ConfigConstants.CONFIG_KEY_URL);
  109.                 // This could be done better (more precise logic), but according
  110.                 // to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs
  111.                 // generally only supports 'origin' in an integrated workflow.
  112.                 if (lfsUrl == null && (remote.equals(
  113.                         org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME))) {
  114.                     remoteUrl = config.getString(
  115.                             ConfigConstants.CONFIG_KEY_REMOTE, remote,
  116.                             ConfigConstants.CONFIG_KEY_URL);
  117.                     break;
  118.                 }
  119.             }
  120.             if (lfsUrl == null && remoteUrl != null) {
  121.                 try {
  122.                     lfsUrl = discoverLfsUrl(db, purpose, additionalHeaders,
  123.                             remoteUrl);
  124.                 } catch (URISyntaxException | IOException
  125.                         | CommandFailedException e) {
  126.                     ex = e;
  127.                 }
  128.             } else {
  129.                 lfsUrl = lfsUrl + Protocol.INFO_LFS_ENDPOINT;
  130.             }
  131.         }
  132.         if (lfsUrl == null) {
  133.             if (ex != null) {
  134.                 throw new LfsConfigInvalidException(
  135.                         LfsText.get().lfsNoDownloadUrl, ex);
  136.             }
  137.             throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl);
  138.         }
  139.         return lfsUrl;
  140.     }

  142.     private static String discoverLfsUrl(Repository db, String purpose,
  143.             Map<String, String> additionalHeaders, String remoteUrl)
  144.             throws URISyntaxException, IOException, CommandFailedException {
  145.         URIish u = new URIish(remoteUrl);
  146.         if (u.getScheme() == null || SCHEME_SSH.equals(u.getScheme())) {
  147.             Protocol.ExpiringAction action = getSshAuthentication(db, purpose,
  148.                     remoteUrl, u);
  149.             additionalHeaders.putAll(action.header);
  150.             return action.href;
  151.         }
  152.         return remoteUrl + Protocol.INFO_LFS_ENDPOINT;
  153.     }

  155.     private static Protocol.ExpiringAction getSshAuthentication(
  156.             Repository db, String purpose, String remoteUrl, URIish u)
  157.             throws IOException, CommandFailedException {
  158.         AuthCache cached = sshAuthCache.get(remoteUrl);
  159.         Protocol.ExpiringAction action = null;
  160.         if (cached != null && cached.validUntil > System.currentTimeMillis()) {
  161.             action = cached.cachedAction;
  162.         }

  164.         if (action == null) {
  165.             // discover and authenticate; git-lfs does "ssh
  166.             // -p <port> -- <host> git-lfs-authenticate
  167.             // <project> <upload/download>"
  168.             String json = SshSupport.runSshCommand(u.setPath(""), //$NON-NLS-1$
  169.                     null, db.getFS(),
  170.                     "git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$
  171.                             + purpose,
  172.                     SSH_AUTH_TIMEOUT_SECONDS);

  174.             action = Protocol.gson().fromJson(json,
  175.                     Protocol.ExpiringAction.class);

  177.             // cache the result as long as possible.
  178.             AuthCache c = new AuthCache(action);
  179.             sshAuthCache.put(remoteUrl, c);
  180.         }
  181.         return action;
  182.     }

  184.     /**
  185.      * Create a connection for the specified
  186.      * {@link org.eclipse.jgit.lfs.Protocol.Action}.
  187.      *
  188.      * @param repo
  189.      *            the repo to fetch required configuration from
  190.      * @param action
  191.      *            the action for which to create a connection
  192.      * @param method
  193.      *            the target method (GET or PUT)
  194.      * @return a connection. output mode is not set.
  195.      * @throws IOException
  196.      *             in case of any error.
  197.      */
  198.     @NonNull
  199.     public static HttpConnection getLfsContentConnection(
  200.             Repository repo, Protocol.Action action, String method)
  201.             throws IOException {
  202.         URL contentUrl = new URL(action.href);
  203.         HttpConnection contentServerConn = HttpTransport.getConnectionFactory()
  204.                 .create(contentUrl, HttpSupport
  205.                         .proxyFor(ProxySelector.getDefault(), contentUrl));
  206.         contentServerConn.setRequestMethod(method);
  207.         if (action.header != null) {
  208.             action.header.forEach(
  209.                     (k, v) -> contentServerConn.setRequestProperty(k, v));
  210.         }
  211.         if (contentUrl.getProtocol().equals(SCHEME_HTTPS)
  212.                 && !repo.getConfig().getBoolean(HttpConfig.HTTP,
  213.                         HttpConfig.SSL_VERIFY_KEY, true)) {
  214.             HttpSupport.disableSslVerify(contentServerConn);
  215.         }

  217.         contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING,
  218.                 ENCODING_GZIP);

  220.         return contentServerConn;
  221.     }

  223.     private static String extractProjectName(URIish u) {
  224.         String path = u.getPath();

  226.         // begins with a slash if the url contains a port (gerrit vs. github).
  227.         if (path.startsWith("/")) { //$NON-NLS-1$
  228.             path = path.substring(1);
  229.         }

  231.         if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) {
  232.             return path.substring(0, path.length() - 4);
  233.         }
  234.         return path;
  235.     }

  237.     /**
  238.      * @param operation
  239.      *            the operation to perform, e.g. Protocol.OPERATION_DOWNLOAD
  240.      * @param resources
  241.      *            the LFS resources affected
  242.      * @return a request that can be serialized to JSON
  243.      */
  244.     public static Protocol.Request toRequest(String operation,
  245.             LfsPointer... resources) {
  246.         Protocol.Request req = new Protocol.Request();
  247.         req.operation = operation;
  248.         if (resources != null) {
  249.             req.objects = new LinkedList<>();
  250.             for (LfsPointer res : resources) {
  251.                 Protocol.ObjectSpec o = new Protocol.ObjectSpec();
  252.                 o.oid = res.getOid().getName();
  253.                 o.size = res.getSize();
  254.                 req.objects.add(o);
  255.             }
  256.         }
  257.         return req;
  258.     }

  260.     private static final class AuthCache {
  261.         private static final long AUTH_CACHE_EAGER_TIMEOUT = 500;

  263.         private static final DateTimeFormatter ISO_FORMAT = DateTimeFormatter
  264.                 .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$

  266.         /**
  267.          * Creates a cache entry for an authentication response.
  268.          * <p>
  269.          * The timeout of the cache token is extracted from the given action. If
  270.          * no timeout can be determined, the token will be used only once.
  271.          *
  272.          * @param action
  273.          */
  274.         public AuthCache(Protocol.ExpiringAction action) {
  275.             this.cachedAction = action;
  276.             try {
  277.                 if (action.expiresIn != null && !action.expiresIn.isEmpty()) {
  278.                     this.validUntil = (System.currentTimeMillis()
  279.                             + Long.parseLong(action.expiresIn))
  280.                             - AUTH_CACHE_EAGER_TIMEOUT;
  281.                 } else if (action.expiresAt != null
  282.                         && !action.expiresAt.isEmpty()) {
  283.                     this.validUntil = LocalDateTime
  284.                             .parse(action.expiresAt, ISO_FORMAT)
  285.                             .atZone(ZoneOffset.UTC).toInstant().toEpochMilli()
  286.                             - AUTH_CACHE_EAGER_TIMEOUT;
  287.                 } else {
  288.                     this.validUntil = System.currentTimeMillis();
  289.                 }
  290.             } catch (Exception e) {
  291.                 this.validUntil = System.currentTimeMillis();
  292.             }
  293.         }

  295.         long validUntil;

  297.         Protocol.ExpiringAction cachedAction;
  298.     }

  300. }
Created with JaCoCo 0.8.6.202009150832