1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.transport;
45
46 import java.io.BufferedReader;
47 import java.io.File;
48 import java.io.FileNotFoundException;
49 import java.io.IOException;
50 import java.io.InputStream;
51 import java.io.OutputStream;
52 import java.net.URLConnection;
53 import java.text.MessageFormat;
54 import java.util.ArrayList;
55 import java.util.Collection;
56 import java.util.Collections;
57 import java.util.EnumSet;
58 import java.util.HashSet;
59 import java.util.Map;
60 import java.util.Properties;
61 import java.util.Set;
62 import java.util.TreeMap;
63
64 import org.eclipse.jgit.errors.NotSupportedException;
65 import org.eclipse.jgit.errors.TransportException;
66 import org.eclipse.jgit.internal.JGitText;
67 import org.eclipse.jgit.lib.Constants;
68 import org.eclipse.jgit.lib.ObjectId;
69 import org.eclipse.jgit.lib.ObjectIdRef;
70 import org.eclipse.jgit.lib.ProgressMonitor;
71 import org.eclipse.jgit.lib.Ref;
72 import org.eclipse.jgit.lib.Ref.Storage;
73 import org.eclipse.jgit.lib.Repository;
74 import org.eclipse.jgit.lib.SymbolicRef;
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100 public class TransportAmazonS3 extends HttpTransport implements WalkTransport {
101 static final String S3_SCHEME = "amazon-s3";
102
103 static final TransportProtocol PROTO_S3 = new TransportProtocol() {
104 @Override
105 public String getName() {
106 return "Amazon S3";
107 }
108
109 @Override
110 public Set<String> getSchemes() {
111 return Collections.singleton(S3_SCHEME);
112 }
113
114 @Override
115 public Set<URIishField> getRequiredFields() {
116 return Collections.unmodifiableSet(EnumSet.of(URIishField.USER,
117 URIishField.HOST, URIishField.PATH));
118 }
119
120 @Override
121 public Set<URIishField> getOptionalFields() {
122 return Collections.unmodifiableSet(EnumSet.of(URIishField.PASS));
123 }
124
125 @Override
126 public Transport open(URIish uri, Repository local, String remoteName)
127 throws NotSupportedException {
128 return new TransportAmazonS3(local, uri);
129 }
130 };
131
132
133 final AmazonS3 s3;
134
135
136 final String bucket;
137
138
139
140
141
142
143
144
145
146
147
148
149 private final String keyPrefix;
150
151 TransportAmazonS3(final Repository local, final URIish uri)
152 throws NotSupportedException {
153 super(local, uri);
154
155 Properties props = loadProperties();
156 File directory = local.getDirectory();
157 if (!props.containsKey("tmpdir") && directory != null)
158 props.put("tmpdir", directory.getPath());
159
160 s3 = new AmazonS3(props);
161 bucket = uri.getHost();
162
163 String p = uri.getPath();
164 if (p.startsWith("/"))
165 p = p.substring(1);
166 if (p.endsWith("/"))
167 p = p.substring(0, p.length() - 1);
168 keyPrefix = p;
169 }
170
171 private Properties loadProperties() throws NotSupportedException {
172 if (local.getDirectory() != null) {
173 File propsFile = new File(local.getDirectory(), uri.getUser());
174 if (propsFile.isFile())
175 return loadPropertiesFile(propsFile);
176 }
177
178 File propsFile = new File(local.getFS().userHome(), uri.getUser());
179 if (propsFile.isFile())
180 return loadPropertiesFile(propsFile);
181
182 Properties props = new Properties();
183 String user = uri.getUser();
184 String pass = uri.getPass();
185 if (user != null && pass != null) {
186 props.setProperty("accesskey", user);
187 props.setProperty("secretkey", pass);
188 } else
189 throw new NotSupportedException(MessageFormat.format(
190 JGitText.get().cannotReadFile, propsFile));
191 return props;
192 }
193
194 private static Properties loadPropertiesFile(File propsFile)
195 throws NotSupportedException {
196 try {
197 return AmazonS3.properties(propsFile);
198 } catch (IOException e) {
199 throw new NotSupportedException(MessageFormat.format(
200 JGitText.get().cannotReadFile, propsFile), e);
201 }
202 }
203
204 @Override
205 public FetchConnection openFetch() throws TransportException {
206 final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
207 final WalkFetchConnection r = new WalkFetchConnection(this, c);
208 r.available(c.readAdvertisedRefs());
209 return r;
210 }
211
212 @Override
213 public PushConnection openPush() throws TransportException {
214 final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
215 final WalkPushConnection r = new WalkPushConnection(this, c);
216 r.available(c.readAdvertisedRefs());
217 return r;
218 }
219
220 @Override
221 public void close() {
222
223 }
224
225 class DatabaseS3 extends WalkRemoteObjectDatabase {
226 private final String bucketName;
227
228 private final String objectsKey;
229
230 DatabaseS3(final String b, final String o) {
231 bucketName = b;
232 objectsKey = o;
233 }
234
235 private String resolveKey(String subpath) {
236 if (subpath.endsWith("/"))
237 subpath = subpath.substring(0, subpath.length() - 1);
238 String k = objectsKey;
239 while (subpath.startsWith(ROOT_DIR)) {
240 k = k.substring(0, k.lastIndexOf('/'));
241 subpath = subpath.substring(3);
242 }
243 return k + "/" + subpath;
244 }
245
246 @Override
247 URIish getURI() {
248 URIish u = new URIish();
249 u = u.setScheme(S3_SCHEME);
250 u = u.setHost(bucketName);
251 u = u.setPath("/" + objectsKey);
252 return u;
253 }
254
255 @Override
256 Collection<WalkRemoteObjectDatabase> getAlternates() throws IOException {
257 try {
258 return readAlternates(INFO_ALTERNATES);
259 } catch (FileNotFoundException err) {
260
261 }
262 return null;
263 }
264
265 @Override
266 WalkRemoteObjectDatabase openAlternate(final String location)
267 throws IOException {
268 return new DatabaseS3(bucketName, resolveKey(location));
269 }
270
271 @Override
272 Collection<String> getPackNames() throws IOException {
273 final HashSet<String> have = new HashSet<>();
274 have.addAll(s3.list(bucket, resolveKey("pack")));
275
276 final Collection<String> packs = new ArrayList<>();
277 for (final String n : have) {
278 if (!n.startsWith("pack-") || !n.endsWith(".pack"))
279 continue;
280
281 final String in = n.substring(0, n.length() - 5) + ".idx";
282 if (have.contains(in))
283 packs.add(n);
284 }
285 return packs;
286 }
287
288 @Override
289 FileStream open(final String path) throws IOException {
290 final URLConnection c = s3.get(bucket, resolveKey(path));
291 final InputStream raw = c.getInputStream();
292 final InputStream in = s3.decrypt(c);
293 final int len = c.getContentLength();
294 return new FileStream(in, raw == in ? len : -1);
295 }
296
297 @Override
298 void deleteFile(final String path) throws IOException {
299 s3.delete(bucket, resolveKey(path));
300 }
301
302 @Override
303 OutputStream writeFile(final String path,
304 final ProgressMonitor monitor, final String monitorTask)
305 throws IOException {
306 return s3.beginPut(bucket, resolveKey(path), monitor, monitorTask);
307 }
308
309 @Override
310 void writeFile(final String path, final byte[] data) throws IOException {
311 s3.put(bucket, resolveKey(path), data);
312 }
313
314 Map<String, Ref> readAdvertisedRefs() throws TransportException {
315 final TreeMap<String, Ref> avail = new TreeMap<>();
316 readPackedRefs(avail);
317 readLooseRefs(avail);
318 readRef(avail, Constants.HEAD);
319 return avail;
320 }
321
322 private void readLooseRefs(final TreeMap<String, Ref> avail)
323 throws TransportException {
324 try {
325 for (final String n : s3.list(bucket, resolveKey(ROOT_DIR
326 + "refs")))
327 readRef(avail, "refs/" + n);
328 } catch (IOException e) {
329 throw new TransportException(getURI(), JGitText.get().cannotListRefs, e);
330 }
331 }
332
333 private Ref readRef(final TreeMap<String, Ref> avail, final String rn)
334 throws TransportException {
335 final String s;
336 String ref = ROOT_DIR + rn;
337 try {
338 final BufferedReader br = openReader(ref);
339 try {
340 s = br.readLine();
341 } finally {
342 br.close();
343 }
344 } catch (FileNotFoundException noRef) {
345 return null;
346 } catch (IOException err) {
347 throw new TransportException(getURI(), MessageFormat.format(
348 JGitText.get().transportExceptionReadRef, ref), err);
349 }
350
351 if (s == null)
352 throw new TransportException(getURI(), MessageFormat.format(JGitText.get().transportExceptionEmptyRef, rn));
353
354 if (s.startsWith("ref: ")) {
355 final String target = s.substring("ref: ".length());
356 Ref r = avail.get(target);
357 if (r == null)
358 r = readRef(avail, target);
359 if (r == null)
360 r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null);
361 r = new SymbolicRef(rn, r);
362 avail.put(r.getName(), r);
363 return r;
364 }
365
366 if (ObjectId.isId(s)) {
367 final Ref r = new ObjectIdRef.Unpeeled(loose(avail.get(rn)),
368 rn, ObjectId.fromString(s));
369 avail.put(r.getName(), r);
370 return r;
371 }
372
373 throw new TransportException(getURI(), MessageFormat.format(JGitText.get().transportExceptionBadRef, rn, s));
374 }
375
376 private Storage loose(final Ref r) {
377 if (r != null && r.getStorage() == Storage.PACKED)
378 return Storage.LOOSE_PACKED;
379 return Storage.LOOSE;
380 }
381
382 @Override
383 void close() {
384
385 }
386 }
387 }