S3Repository.java

  1. /*
  2.  * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com>
  3.  * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others
  4.  *
  5.  * This program and the accompanying materials are made available under the
  6.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  7.  * https://www.eclipse.org/org/documents/edl-v10.php.
  8.  *
  9.  * SPDX-License-Identifier: BSD-3-Clause
  10.  */
  11. package org.eclipse.jgit.lfs.server.s3;

  13. import static javax.servlet.http.HttpServletResponse.SC_OK;
  14. import static org.eclipse.jgit.lfs.server.s3.SignerV4.UNSIGNED_PAYLOAD;
  15. import static org.eclipse.jgit.lfs.server.s3.SignerV4.X_AMZ_CONTENT_SHA256;
  16. import static org.eclipse.jgit.lfs.server.s3.SignerV4.X_AMZ_EXPIRES;
  17. import static org.eclipse.jgit.lfs.server.s3.SignerV4.X_AMZ_STORAGE_CLASS;
  18. import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_LENGTH;
  19. import static org.eclipse.jgit.util.HttpSupport.METHOD_GET;
  20. import static org.eclipse.jgit.util.HttpSupport.METHOD_HEAD;
  21. import static org.eclipse.jgit.util.HttpSupport.METHOD_PUT;

  23. import java.io.IOException;
  24. import java.net.MalformedURLException;
  25. import java.net.Proxy;
  26. import java.net.ProxySelector;
  27. import java.net.URL;
  28. import java.text.MessageFormat;
  29. import java.util.HashMap;
  30. import java.util.Map;

  32. import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
  33. import org.eclipse.jgit.lfs.server.LargeFileRepository;
  34. import org.eclipse.jgit.lfs.server.Response;
  35. import org.eclipse.jgit.lfs.server.Response.Action;
  36. import org.eclipse.jgit.lfs.server.internal.LfsServerText;
  37. import org.eclipse.jgit.transport.http.HttpConnection;
  38. import org.eclipse.jgit.transport.http.apache.HttpClientConnectionFactory;
  39. import org.eclipse.jgit.util.HttpSupport;

  41. /**
  42.  * Repository storing LFS objects in Amazon S3
  43.  *
  44.  * @since 4.3
  45.  */
  46. public class S3Repository implements LargeFileRepository {

  48.     private S3Config s3Config;

  50.     /**
  51.      * Construct a LFS repository storing large objects in Amazon S3
  52.      *
  53.      * @param config
  54.      *            AWS S3 storage bucket configuration
  55.      */
  56.     public S3Repository(S3Config config) {
  57.         validateConfig(config);
  58.         this.s3Config = config;
  59.     }

  61.     /** {@inheritDoc} */
  62.     @Override
  63.     public Response.Action getDownloadAction(AnyLongObjectId oid) {
  64.         URL endpointUrl = getObjectUrl(oid);
  65.         Map<String, String> queryParams = new HashMap<>();
  66.         queryParams.put(X_AMZ_EXPIRES,
  67.                 Integer.toString(s3Config.getExpirationSeconds()));
  68.         Map<String, String> headers = new HashMap<>();
  69.         String authorizationQueryParameters = SignerV4.createAuthorizationQuery(
  70.                 s3Config, endpointUrl, METHOD_GET, headers, queryParams,
  71.                 UNSIGNED_PAYLOAD);

  73.         Response.Action a = new Response.Action();
  74.         a.href = endpointUrl.toString() + "?" + authorizationQueryParameters; //$NON-NLS-1$
  75.         return a;
  76.     }

  78.     /** {@inheritDoc} */
  79.     @Override
  80.     public Response.Action getUploadAction(AnyLongObjectId oid, long size) {
  81.         cacheObjectMetaData(oid, size);
  82.         URL objectUrl = getObjectUrl(oid);
  83.         Map<String, String> headers = new HashMap<>();
  84.         headers.put(X_AMZ_CONTENT_SHA256, oid.getName());
  85.         headers.put(HDR_CONTENT_LENGTH, Long.toString(size));
  86.         headers.put(X_AMZ_STORAGE_CLASS, s3Config.getStorageClass());
  87.         headers.put(HttpSupport.HDR_CONTENT_TYPE, "application/octet-stream"); //$NON-NLS-1$
  88.         headers = SignerV4.createHeaderAuthorization(s3Config, objectUrl,
  89.                 METHOD_PUT, headers, oid.getName());

  91.         Response.Action a = new Response.Action();
  92.         a.href = objectUrl.toString();
  93.         a.header = new HashMap<>();
  94.         a.header.putAll(headers);
  95.         return a;
  96.     }

  98.     /** {@inheritDoc} */
  99.     @Override
  100.     public Action getVerifyAction(AnyLongObjectId id) {
  101.         return null; // TODO(ms) implement this
  102.     }

  104.     /** {@inheritDoc} */
  105.     @Override
  106.     public long getSize(AnyLongObjectId oid) throws IOException {
  107.         URL endpointUrl = getObjectUrl(oid);
  108.         Map<String, String> queryParams = new HashMap<>();
  109.         queryParams.put(X_AMZ_EXPIRES,
  110.                 Integer.toString(s3Config.getExpirationSeconds()));
  111.         Map<String, String> headers = new HashMap<>();

  113.         String authorizationQueryParameters = SignerV4.createAuthorizationQuery(
  114.                 s3Config, endpointUrl, METHOD_HEAD, headers, queryParams,
  115.                 UNSIGNED_PAYLOAD);
  116.         String href = endpointUrl.toString() + "?" //$NON-NLS-1$
  117.                 + authorizationQueryParameters;

  119.         Proxy proxy = HttpSupport.proxyFor(ProxySelector.getDefault(),
  120.                 endpointUrl);
  121.         HttpClientConnectionFactory f = new HttpClientConnectionFactory();
  122.         HttpConnection conn = f.create(new URL(href), proxy);
  123.         if (s3Config.isDisableSslVerify()) {
  124.             HttpSupport.disableSslVerify(conn);
  125.         }
  126.         conn.setRequestMethod(METHOD_HEAD);
  127.         conn.connect();
  128.         int status = conn.getResponseCode();
  129.         if (status == SC_OK) {
  130.             String contentLengthHeader = conn
  131.                     .getHeaderField(HDR_CONTENT_LENGTH);
  132.             if (contentLengthHeader != null) {
  133.                 return Integer.parseInt(contentLengthHeader);
  134.             }
  135.         }
  136.         return -1;
  137.     }

  139.     /**
  140.      * Cache metadata (size) for an object to avoid extra roundtrip to S3 in
  141.      * order to retrieve this metadata for a given object. Subclasses can
  142.      * implement a local cache and override {{@link #getSize(AnyLongObjectId)}
  143.      * to retrieve the object size from the local cache to eliminate the need
  144.      * for another roundtrip to S3
  145.      *
  146.      * @param oid
  147.      *            the object id identifying the object to be cached
  148.      * @param size
  149.      *            the object's size (in bytes)
  150.      */
  151.     protected void cacheObjectMetaData(AnyLongObjectId oid, long size) {
  152.         // no caching
  153.     }

  155.     private void validateConfig(S3Config config) {
  156.         assertNotEmpty(LfsServerText.get().undefinedS3AccessKey,
  157.                 config.getAccessKey());
  158.         assertNotEmpty(LfsServerText.get().undefinedS3Bucket,
  159.                 config.getBucket());
  160.         assertNotEmpty(LfsServerText.get().undefinedS3Region,
  161.                 config.getRegion());
  162.         assertNotEmpty(LfsServerText.get().undefinedS3Hostname,
  163.                 config.getHostname());
  164.         assertNotEmpty(LfsServerText.get().undefinedS3SecretKey,
  165.                 config.getSecretKey());
  166.         assertNotEmpty(LfsServerText.get().undefinedS3StorageClass,
  167.                 config.getStorageClass());
  168.     }

  170.     private void assertNotEmpty(String message, String value) {
  171.         if (value == null || value.trim().length() == 0) {
  172.             throw new IllegalArgumentException(message);
  173.         }
  174.     }

  176.     private URL getObjectUrl(AnyLongObjectId oid) {
  177.         try {
  178.             return new URL(String.format("https://%s/%s/%s", //$NON-NLS-1$
  179.                     s3Config.getHostname(), s3Config.getBucket(),
  180.                     getPath(oid)));
  181.         } catch (MalformedURLException e) {
  182.             throw new IllegalArgumentException(MessageFormat.format(
  183.                     LfsServerText.get().unparsableEndpoint, e.getMessage()));
  184.         }
  185.     }

  187.     private String getPath(AnyLongObjectId oid) {
  188.         return oid.getName();
  189.     }
  190. }
Created with JaCoCo 0.8.6.202009150832