1 /* 2 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> 3 * and other copyright owners as documented in the project's IP log. 4 * 5 * This program and the accompanying materials are made available 6 * under the terms of the Eclipse Distribution License v1.0 which 7 * accompanies this distribution, is reproduced below, and is 8 * available at http://www.eclipse.org/org/documents/edl-v10.php 9 * 10 * All rights reserved. 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials provided 22 * with the distribution. 23 * 24 * - Neither the name of the Eclipse Foundation, Inc. nor the 25 * names of its contributors may be used to endorse or promote 26 * products derived from this software without specific prior 27 * written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 30 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 31 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 32 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 33 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 34 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 35 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 36 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 37 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 38 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 39 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 40 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 41 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 */ 43 44 package org.eclipse.jgit.transport; 45 46 import java.io.BufferedReader; 47 import java.io.ByteArrayOutputStream; 48 import java.io.FileNotFoundException; 49 import java.io.IOException; 50 import java.io.InputStream; 51 import java.io.InputStreamReader; 52 import java.io.OutputStream; 53 import java.text.MessageFormat; 54 import java.util.ArrayList; 55 import java.util.Collection; 56 import java.util.Map; 57 58 import org.eclipse.jgit.errors.TransportException; 59 import org.eclipse.jgit.internal.JGitText; 60 import org.eclipse.jgit.internal.storage.file.RefDirectory; 61 import org.eclipse.jgit.lib.Constants; 62 import org.eclipse.jgit.lib.ObjectId; 63 import org.eclipse.jgit.lib.ObjectIdRef; 64 import org.eclipse.jgit.lib.ProgressMonitor; 65 import org.eclipse.jgit.lib.Ref; 66 import org.eclipse.jgit.util.IO; 67 68 /** 69 * Transfers object data through a dumb transport. 70 * <p> 71 * Implementations are responsible for resolving path names relative to the 72 * <code>objects/</code> subdirectory of a single remote Git repository or 73 * naked object database and make the content available as a Java input stream 74 * for reading during fetch. The actual object traversal logic to determine the 75 * names of files to retrieve is handled through the generic, protocol 76 * independent {@link WalkFetchConnection}. 77 */ 78 abstract class WalkRemoteObjectDatabase { 79 static final String ROOT_DIR = "../"; //$NON-NLS-1$ 80 81 static final String INFO_PACKS = "info/packs"; //$NON-NLS-1$ 82 83 static final String INFO_ALTERNATES = "info/alternates"; //$NON-NLS-1$ 84 85 static final String INFO_HTTP_ALTERNATES = "info/http-alternates"; //$NON-NLS-1$ 86 87 static final String INFO_REFS = ROOT_DIR + Constants.INFO_REFS; 88 89 abstract URIish getURI(); 90 91 /** 92 * Obtain the list of available packs (if any). 93 * <p> 94 * Pack names should be the file name in the packs directory, that is 95 * <code>pack-035760ab452d6eebd123add421f253ce7682355a.pack</code>. Index 96 * names should not be included in the returned collection. 97 * 98 * @return list of pack names; null or empty list if none are available. 99 * @throws IOException 100 * The connection is unable to read the remote repository's list 101 * of available pack files. 102 */ 103 abstract Collection<String> getPackNames() throws IOException; 104 105 /** 106 * Obtain alternate connections to alternate object databases (if any). 107 * <p> 108 * Alternates are typically read from the file {@link #INFO_ALTERNATES} or 109 * {@link #INFO_HTTP_ALTERNATES}. The content of each line must be resolved 110 * by the implementation and a new database reference should be returned to 111 * represent the additional location. 112 * <p> 113 * Alternates may reuse the same network connection handle, however the 114 * fetch connection will {@link #close()} each created alternate. 115 * 116 * @return list of additional object databases the caller could fetch from; 117 * null or empty list if none are configured. 118 * @throws IOException 119 * The connection is unable to read the remote repository's list 120 * of configured alternates. 121 */ 122 abstract Collection<WalkRemoteObjectDatabase> getAlternates() 123 throws IOException; 124 125 /** 126 * Open a single file for reading. 127 * <p> 128 * Implementors should make every attempt possible to ensure 129 * {@link FileNotFoundException} is used when the remote object does not 130 * exist. However when fetching over HTTP some misconfigured servers may 131 * generate a 200 OK status message (rather than a 404 Not Found) with an 132 * HTML formatted message explaining the requested resource does not exist. 133 * Callers such as {@link WalkFetchConnection} are prepared to handle this 134 * by validating the content received, and assuming content that fails to 135 * match its hash is an incorrectly phrased FileNotFoundException. 136 * 137 * @param path 138 * location of the file to read, relative to this objects 139 * directory (e.g. 140 * <code>cb/95df6ab7ae9e57571511ef451cf33767c26dd2</code> or 141 * <code>pack/pack-035760ab452d6eebd123add421f253ce7682355a.pack</code>). 142 * @return a stream to read from the file. Never null. 143 * @throws FileNotFoundException 144 * the requested file does not exist at the given location. 145 * @throws IOException 146 * The connection is unable to read the remote's file, and the 147 * failure occurred prior to being able to determine if the file 148 * exists, or after it was determined to exist but before the 149 * stream could be created. 150 */ 151 abstract FileStream open(String path) throws FileNotFoundException, 152 IOException; 153 154 /** 155 * Create a new connection for a discovered alternate object database 156 * <p> 157 * This method is typically called by {@link #readAlternates(String)} when 158 * subclasses us the generic alternate parsing logic for their 159 * implementation of {@link #getAlternates()}. 160 * 161 * @param location 162 * the location of the new alternate, relative to the current 163 * object database. 164 * @return a new database connection that can read from the specified 165 * alternate. 166 * @throws IOException 167 * The database connection cannot be established with the 168 * alternate, such as if the alternate location does not 169 * actually exist and the connection's constructor attempts to 170 * verify that. 171 */ 172 abstract WalkRemoteObjectDatabase openAlternate(String location) 173 throws IOException; 174 175 /** 176 * Close any resources used by this connection. 177 * <p> 178 * If the remote repository is contacted by a network socket this method 179 * must close that network socket, disconnecting the two peers. If the 180 * remote repository is actually local (same system) this method must close 181 * any open file handles used to read the "remote" repository. 182 */ 183 abstract void close(); 184 185 /** 186 * Delete a file from the object database. 187 * <p> 188 * Path may start with <code>../</code> to request deletion of a file that 189 * resides in the repository itself. 190 * <p> 191 * When possible empty directories must be removed, up to but not including 192 * the current object database directory itself. 193 * <p> 194 * This method does not support deletion of directories. 195 * 196 * @param path 197 * name of the item to be removed, relative to the current object 198 * database. 199 * @throws IOException 200 * deletion is not supported, or deletion failed. 201 */ 202 void deleteFile(final String path) throws IOException { 203 throw new IOException(MessageFormat.format(JGitText.get().deletingNotSupported, path)); 204 } 205 206 /** 207 * Open a remote file for writing. 208 * <p> 209 * Path may start with <code>../</code> to request writing of a file that 210 * resides in the repository itself. 211 * <p> 212 * The requested path may or may not exist. If the path already exists as a 213 * file the file should be truncated and completely replaced. 214 * <p> 215 * This method creates any missing parent directories, if necessary. 216 * 217 * @param path 218 * name of the file to write, relative to the current object 219 * database. 220 * @return stream to write into this file. Caller must close the stream to 221 * complete the write request. The stream is not buffered and each 222 * write may cause a network request/response so callers should 223 * buffer to smooth out small writes. 224 * @param monitor 225 * (optional) progress monitor to post write completion to during 226 * the stream's close method. 227 * @param monitorTask 228 * (optional) task name to display during the close method. 229 * @throws IOException 230 * writing is not supported, or attempting to write the file 231 * failed, possibly due to permissions or remote disk full, etc. 232 */ 233 OutputStream writeFile(final String path, final ProgressMonitor monitor, 234 final String monitorTask) throws IOException { 235 throw new IOException(MessageFormat.format(JGitText.get().writingNotSupported, path)); 236 } 237 238 /** 239 * Atomically write a remote file. 240 * <p> 241 * This method attempts to perform as atomic of an update as it can, 242 * reducing (or eliminating) the time that clients might be able to see 243 * partial file content. This method is not suitable for very large 244 * transfers as the complete content must be passed as an argument. 245 * <p> 246 * Path may start with <code>../</code> to request writing of a file that 247 * resides in the repository itself. 248 * <p> 249 * The requested path may or may not exist. If the path already exists as a 250 * file the file should be truncated and completely replaced. 251 * <p> 252 * This method creates any missing parent directories, if necessary. 253 * 254 * @param path 255 * name of the file to write, relative to the current object 256 * database. 257 * @param data 258 * complete new content of the file. 259 * @throws IOException 260 * writing is not supported, or attempting to write the file 261 * failed, possibly due to permissions or remote disk full, etc. 262 */ 263 void writeFile(final String path, final byte[] data) throws IOException { 264 final OutputStream os = writeFile(path, null, null); 265 try { 266 os.write(data); 267 } finally { 268 os.close(); 269 } 270 } 271 272 /** 273 * Delete a loose ref from the remote repository. 274 * 275 * @param name 276 * name of the ref within the ref space, for example 277 * <code>refs/heads/pu</code>. 278 * @throws IOException 279 * deletion is not supported, or deletion failed. 280 */ 281 void deleteRef(final String name) throws IOException { 282 deleteFile(ROOT_DIR + name); 283 } 284 285 /** 286 * Delete a reflog from the remote repository. 287 * 288 * @param name 289 * name of the ref within the ref space, for example 290 * <code>refs/heads/pu</code>. 291 * @throws IOException 292 * deletion is not supported, or deletion failed. 293 */ 294 void deleteRefLog(final String name) throws IOException { 295 deleteFile(ROOT_DIR + Constants.LOGS + "/" + name); //$NON-NLS-1$ 296 } 297 298 /** 299 * Overwrite (or create) a loose ref in the remote repository. 300 * <p> 301 * This method creates any missing parent directories, if necessary. 302 * 303 * @param name 304 * name of the ref within the ref space, for example 305 * <code>refs/heads/pu</code>. 306 * @param value 307 * new value to store in this ref. Must not be null. 308 * @throws IOException 309 * writing is not supported, or attempting to write the file 310 * failed, possibly due to permissions or remote disk full, etc. 311 */ 312 void writeRef(final String name, final ObjectId value) throws IOException { 313 final ByteArrayOutputStream b; 314 315 b = new ByteArrayOutputStream(Constants.OBJECT_ID_STRING_LENGTH + 1); 316 value.copyTo(b); 317 b.write('\n'); 318 319 writeFile(ROOT_DIR + name, b.toByteArray()); 320 } 321 322 /** 323 * Rebuild the {@link #INFO_PACKS} for dumb transport clients. 324 * <p> 325 * This method rebuilds the contents of the {@link #INFO_PACKS} file to 326 * match the passed list of pack names. 327 * 328 * @param packNames 329 * names of available pack files, in the order they should appear 330 * in the file. Valid pack name strings are of the form 331 * <code>pack-035760ab452d6eebd123add421f253ce7682355a.pack</code>. 332 * @throws IOException 333 * writing is not supported, or attempting to write the file 334 * failed, possibly due to permissions or remote disk full, etc. 335 */ 336 void writeInfoPacks(final Collection<String> packNames) throws IOException { 337 final StringBuilder w = new StringBuilder(); 338 for (final String n : packNames) { 339 w.append("P "); //$NON-NLS-1$ 340 w.append(n); 341 w.append('\n'); 342 } 343 writeFile(INFO_PACKS, Constants.encodeASCII(w.toString())); 344 } 345 346 /** 347 * Open a buffered reader around a file. 348 * <p> 349 * This is shorthand for calling {@link #open(String)} and then wrapping it 350 * in a reader suitable for line oriented files like the alternates list. 351 * 352 * @return a stream to read from the file. Never null. 353 * @param path 354 * location of the file to read, relative to this objects 355 * directory (e.g. <code>info/packs</code>). 356 * @throws FileNotFoundException 357 * the requested file does not exist at the given location. 358 * @throws IOException 359 * The connection is unable to read the remote's file, and the 360 * failure occurred prior to being able to determine if the file 361 * exists, or after it was determined to exist but before the 362 * stream could be created. 363 */ 364 BufferedReader openReader(final String path) throws IOException { 365 final InputStream is = open(path).in; 366 return new BufferedReader(new InputStreamReader(is, Constants.CHARSET)); 367 } 368 369 /** 370 * Read a standard Git alternates file to discover other object databases. 371 * <p> 372 * This method is suitable for reading the standard formats of the 373 * alternates file, such as found in <code>objects/info/alternates</code> 374 * or <code>objects/info/http-alternates</code> within a Git repository. 375 * <p> 376 * Alternates appear one per line, with paths expressed relative to this 377 * object database. 378 * 379 * @param listPath 380 * location of the alternate file to read, relative to this 381 * object database (e.g. <code>info/alternates</code>). 382 * @return the list of discovered alternates. Empty list if the file exists, 383 * but no entries were discovered. 384 * @throws FileNotFoundException 385 * the requested file does not exist at the given location. 386 * @throws IOException 387 * The connection is unable to read the remote's file, and the 388 * failure occurred prior to being able to determine if the file 389 * exists, or after it was determined to exist but before the 390 * stream could be created. 391 */ 392 Collection<WalkRemoteObjectDatabase> readAlternates(final String listPath) 393 throws IOException { 394 final BufferedReader br = openReader(listPath); 395 try { 396 final Collection<WalkRemoteObjectDatabase> alts = new ArrayList<WalkRemoteObjectDatabase>(); 397 for (;;) { 398 String line = br.readLine(); 399 if (line == null) 400 break; 401 if (!line.endsWith("/")) //$NON-NLS-1$ 402 line += "/"; //$NON-NLS-1$ 403 alts.add(openAlternate(line)); 404 } 405 return alts; 406 } finally { 407 br.close(); 408 } 409 } 410 411 /** 412 * Read a standard Git packed-refs file to discover known references. 413 * 414 * @param avail 415 * return collection of references. Any existing entries will be 416 * replaced if they are found in the packed-refs file. 417 * @throws TransportException 418 * an error occurred reading from the packed refs file. 419 */ 420 protected void readPackedRefs(final Map<String, Ref> avail) 421 throws TransportException { 422 try { 423 final BufferedReader br = openReader(ROOT_DIR 424 + Constants.PACKED_REFS); 425 try { 426 readPackedRefsImpl(avail, br); 427 } finally { 428 br.close(); 429 } 430 } catch (FileNotFoundException notPacked) { 431 // Perhaps it wasn't worthwhile, or is just an older repository. 432 } catch (IOException e) { 433 throw new TransportException(getURI(), JGitText.get().errorInPackedRefs, e); 434 } 435 } 436 437 private void readPackedRefsImpl(final Map<String, Ref> avail, 438 final BufferedReader br) throws IOException { 439 Ref last = null; 440 boolean peeled = false; 441 for (;;) { 442 String line = br.readLine(); 443 if (line == null) 444 break; 445 if (line.charAt(0) == '#') { 446 if (line.startsWith(RefDirectory.PACKED_REFS_HEADER)) { 447 line = line.substring(RefDirectory.PACKED_REFS_HEADER.length()); 448 peeled = line.contains(RefDirectory.PACKED_REFS_PEELED); 449 } 450 continue; 451 } 452 if (line.charAt(0) == '^') { 453 if (last == null) 454 throw new TransportException(JGitText.get().peeledLineBeforeRef); 455 final ObjectId id = ObjectId.fromString(line.substring(1)); 456 last = new ObjectIdRef.PeeledTag(Ref.Storage.PACKED, last 457 .getName(), last.getObjectId(), id); 458 avail.put(last.getName(), last); 459 continue; 460 } 461 462 final int sp = line.indexOf(' '); 463 if (sp < 0) 464 throw new TransportException(MessageFormat.format(JGitText.get().unrecognizedRef, line)); 465 final ObjectId id = ObjectId.fromString(line.substring(0, sp)); 466 final String name = line.substring(sp + 1); 467 if (peeled) 468 last = new ObjectIdRef.PeeledNonTag(Ref.Storage.PACKED, name, id); 469 else 470 last = new ObjectIdRef.Unpeeled(Ref.Storage.PACKED, name, id); 471 avail.put(last.getName(), last); 472 } 473 } 474 475 static final class FileStream { 476 final InputStream in; 477 478 final long length; 479 480 /** 481 * Create a new stream of unknown length. 482 * 483 * @param i 484 * stream containing the file data. This stream will be 485 * closed by the caller when reading is complete. 486 */ 487 FileStream(final InputStream i) { 488 in = i; 489 length = -1; 490 } 491 492 /** 493 * Create a new stream of known length. 494 * 495 * @param i 496 * stream containing the file data. This stream will be 497 * closed by the caller when reading is complete. 498 * @param n 499 * total number of bytes available for reading through 500 * <code>i</code>. 501 */ 502 FileStream(final InputStream i, final long n) { 503 in = i; 504 length = n; 505 } 506 507 byte[] toArray() throws IOException { 508 try { 509 if (length >= 0) { 510 final byte[] r = new byte[(int) length]; 511 IO.readFully(in, r, 0, r.length); 512 return r; 513 } 514 515 final ByteArrayOutputStream r = new ByteArrayOutputStream(); 516 final byte[] buf = new byte[2048]; 517 int n; 518 while ((n = in.read(buf)) >= 0) 519 r.write(buf, 0, n); 520 return r.toByteArray(); 521 } finally { 522 in.close(); 523 } 524 } 525 } 526 }