WalkFetchConnection.java

/*
 * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
 *
 * This program and the accompanying materials are made available under the
 * terms of the Eclipse Distribution License v. 1.0 which is available at
 * https://www.eclipse.org/org/documents/edl-v10.php.
 *
 * SPDX-License-Identifier: BSD-3-Clause
 */

package org.eclipse.jgit.transport;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import org.eclipse.jgit.errors.CompoundException;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.file.ObjectDirectory;
import org.eclipse.jgit.internal.storage.file.PackIndex;
import org.eclipse.jgit.internal.storage.file.PackLock;
import org.eclipse.jgit.internal.storage.file.UnpackedObject;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.FileMode;
import org.eclipse.jgit.lib.MutableObjectId;
import org.eclipse.jgit.lib.ObjectChecker;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectInserter;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.DateRevQueue;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevFlag;
import org.eclipse.jgit.revwalk.RevObject;
import org.eclipse.jgit.revwalk.RevTag;
import org.eclipse.jgit.revwalk.RevTree;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.util.FileUtils;

/**
 * Generic fetch support for dumb transport protocols.
 * <p>
 * Since there are no Git-specific smarts on the remote side of the connection
 * the client side must determine which objects it needs to copy in order to
 * completely fetch the requested refs and their history. The generic walk
 * support in this class parses each individual object (once it has been copied
 * to the local repository) and examines the list of objects that must also be
 * copied to create a complete history. Objects which are already available
 * locally are retained (and not copied), saving bandwidth for incremental
 * fetches. Pack files are copied from the remote repository only as a last
 * resort, as the entire pack must be copied locally in order to access any
 * single object.
 * <p>
 * This fetch connection does not actually perform the object data transfer.
 * Instead it delegates the transfer to a {@link WalkRemoteObjectDatabase},
 * which knows how to read individual files from the remote repository and
 * supply the data as a standard Java InputStream.
 *
 * @see WalkRemoteObjectDatabase
 */
class WalkFetchConnection extends BaseFetchConnection {
    /** The repository this transport fetches into, or pushes out of. */
    final Repository local;

    /** If not null the validator for received objects. */
    final ObjectChecker objCheck;

    /**
     * List of all remote repositories we may need to get objects out of.
     * <p>
     * The first repository in the list is the one we were asked to fetch from;
     * the remaining repositories point to the alternate locations we can fetch
     * objects through.
     */
    private final List<WalkRemoteObjectDatabase> remotes;

    /** Most recently used item in {@link #remotes}. */
    private int lastRemoteIdx;

    private final RevWalk revWalk;

    private final TreeWalk treeWalk;

    /** Objects whose direct dependents we know we have (or will have). */
    private final RevFlag COMPLETE;

    /** Objects that have already entered {@link #workQueue}. */
    private final RevFlag IN_WORK_QUEUE;

    /** Commits that have already entered {@link #localCommitQueue}. */
    private final RevFlag LOCALLY_SEEN;

    /** Commits already reachable from all local refs. */
    private final DateRevQueue localCommitQueue;

    /** Objects we need to copy from the remote repository. */
    private LinkedList<ObjectId> workQueue;

    /** Databases we have not yet obtained the list of packs from. */
    private final LinkedList<WalkRemoteObjectDatabase> noPacksYet;

    /** Databases we have not yet obtained the alternates from. */
    private final LinkedList<WalkRemoteObjectDatabase> noAlternatesYet;

    /** Packs we have discovered, but have not yet fetched locally. */
    private final LinkedList<RemotePack> unfetchedPacks;

    /**
     * Packs whose indexes we have looked at in {@link #unfetchedPacks}.
     * <p>
     * We try to avoid getting duplicate copies of the same pack through
     * multiple alternates by only looking at packs whose names are not yet in
     * this collection.
     */
    private final Set<String> packsConsidered;

    private final MutableObjectId idBuffer = new MutableObjectId();

    /**
     * Errors received while trying to obtain an object.
     * <p>
     * If the fetch winds up failing because we cannot locate a specific object
     * then we need to report all errors related to that object back to the
     * caller as there may be cascading failures.
     */
    private final HashMap<ObjectId, List<Throwable>> fetchErrors;

    String lockMessage;

    final List<PackLock> packLocks;

    /** Inserter to write objects onto {@link #local}. */
    final ObjectInserter inserter;

    /** Inserter to read objects from {@link #local}. */
    private final ObjectReader reader;

    WalkFetchConnection(WalkTransport t, WalkRemoteObjectDatabase w) {
        Transport wt = (Transport)t;
        local = wt.local;
        objCheck = wt.getObjectChecker();
        inserter = local.newObjectInserter();
        reader = inserter.newReader();

        remotes = new ArrayList<>();
        remotes.add(w);

        unfetchedPacks = new LinkedList<>();
        packsConsidered = new HashSet<>();

        noPacksYet = new LinkedList<>();
        noPacksYet.add(w);

        noAlternatesYet = new LinkedList<>();
        noAlternatesYet.add(w);

        fetchErrors = new HashMap<>();
        packLocks = new ArrayList<>(4);

        revWalk = new RevWalk(reader);
        revWalk.setRetainBody(false);
        treeWalk = new TreeWalk(reader);
        COMPLETE = revWalk.newFlag("COMPLETE"); //$NON-NLS-1$
        IN_WORK_QUEUE = revWalk.newFlag("IN_WORK_QUEUE"); //$NON-NLS-1$
        LOCALLY_SEEN = revWalk.newFlag("LOCALLY_SEEN"); //$NON-NLS-1$

        localCommitQueue = new DateRevQueue();
        workQueue = new LinkedList<>();
    }

    /** {@inheritDoc} */
    @Override
    public boolean didFetchTestConnectivity() {
        return true;
    }

    /** {@inheritDoc} */
    @Override
    protected void doFetch(final ProgressMonitor monitor,
            final Collection<Ref> want, final Set<ObjectId> have)
            throws TransportException {
        markLocalRefsComplete(have);
        queueWants(want);

        while (!monitor.isCancelled() && !workQueue.isEmpty()) {
            final ObjectId id = workQueue.removeFirst();
            if (!(id instanceof RevObject) || !((RevObject) id).has(COMPLETE))
                downloadObject(monitor, id);
            process(id);
        }

        try {
            inserter.flush();
        } catch (IOException e) {
            throw new TransportException(e.getMessage(), e);
        }
    }

    /** {@inheritDoc} */
    @Override
    public Collection<PackLock> getPackLocks() {
        return packLocks;
    }

    /** {@inheritDoc} */
    @Override
    public void setPackLockMessage(String message) {
        lockMessage = message;
    }

    /** {@inheritDoc} */
    @Override
    public void close() {
        inserter.close();
        reader.close();
        for (RemotePack p : unfetchedPacks) {
            if (p.tmpIdx != null)
                p.tmpIdx.delete();
        }
        for (WalkRemoteObjectDatabase r : remotes)
            r.close();
    }

    private void queueWants(Collection<Ref> want)
            throws TransportException {
        final HashSet<ObjectId> inWorkQueue = new HashSet<>();
        for (Ref r : want) {
            final ObjectId id = r.getObjectId();
            if (id == null) {
                throw new NullPointerException(MessageFormat.format(
                        JGitText.get().transportProvidedRefWithNoObjectId, r.getName()));
            }
            try {
                final RevObject obj = revWalk.parseAny(id);
                if (obj.has(COMPLETE))
                    continue;
                if (inWorkQueue.add(id)) {
                    obj.add(IN_WORK_QUEUE);
                    workQueue.add(obj);
                }
            } catch (MissingObjectException e) {
                if (inWorkQueue.add(id))
                    workQueue.add(id);
            } catch (IOException e) {
                throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
            }
        }
    }

    private void process(ObjectId id) throws TransportException {
        final RevObject obj;
        try {
            if (id instanceof RevObject) {
                obj = (RevObject) id;
                if (obj.has(COMPLETE))
                    return;
                revWalk.parseHeaders(obj);
            } else {
                obj = revWalk.parseAny(id);
                if (obj.has(COMPLETE))
                    return;
            }
        } catch (IOException e) {
            throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
        }

        switch (obj.getType()) {
        case Constants.OBJ_BLOB:
            processBlob(obj);
            break;
        case Constants.OBJ_TREE:
            processTree(obj);
            break;
        case Constants.OBJ_COMMIT:
            processCommit(obj);
            break;
        case Constants.OBJ_TAG:
            processTag(obj);
            break;
        default:
            throw new TransportException(MessageFormat.format(JGitText.get().unknownObjectType, id.name()));
        }

        // If we had any prior errors fetching this object they are
        // now resolved, as the object was parsed successfully.
        //
        fetchErrors.remove(id);
    }

    private void processBlob(RevObject obj) throws TransportException {
        try {
            if (reader.has(obj, Constants.OBJ_BLOB))
                obj.add(COMPLETE);
            else
                throw new TransportException(MessageFormat.format(JGitText
                        .get().cannotReadBlob, obj.name()),
                        new MissingObjectException(obj, Constants.TYPE_BLOB));
        } catch (IOException error) {
            throw new TransportException(MessageFormat.format(
                    JGitText.get().cannotReadBlob, obj.name()), error);
        }
    }

    private void processTree(RevObject obj) throws TransportException {
        try {
            treeWalk.reset(obj);
            while (treeWalk.next()) {
                final FileMode mode = treeWalk.getFileMode(0);
                final int sType = mode.getObjectType();

                switch (sType) {
                case Constants.OBJ_BLOB:
                case Constants.OBJ_TREE:
                    treeWalk.getObjectId(idBuffer, 0);
                    needs(revWalk.lookupAny(idBuffer, sType));
                    continue;

                default:
                    if (FileMode.GITLINK.equals(mode))
                        continue;
                    treeWalk.getObjectId(idBuffer, 0);
                    throw new CorruptObjectException(MessageFormat.format(JGitText.get().invalidModeFor
                            , mode, idBuffer.name(), treeWalk.getPathString(), obj.getId().name()));
                }
            }
        } catch (IOException ioe) {
            throw new TransportException(MessageFormat.format(JGitText.get().cannotReadTree, obj.name()), ioe);
        }
        obj.add(COMPLETE);
    }

    private void processCommit(RevObject obj) throws TransportException {
        final RevCommit commit = (RevCommit) obj;
        markLocalCommitsComplete(commit.getCommitTime());
        needs(commit.getTree());
        for (RevCommit p : commit.getParents())
            needs(p);
        obj.add(COMPLETE);
    }

    private void processTag(RevObject obj) {
        final RevTag tag = (RevTag) obj;
        needs(tag.getObject());
        obj.add(COMPLETE);
    }

    private void needs(RevObject obj) {
        if (obj.has(COMPLETE))
            return;
        if (!obj.has(IN_WORK_QUEUE)) {
            obj.add(IN_WORK_QUEUE);
            workQueue.add(obj);
        }
    }

    private void downloadObject(ProgressMonitor pm, AnyObjectId id)
            throws TransportException {
        if (alreadyHave(id))
            return;

        for (;;) {
            // Try a pack file we know about, but don't have yet. Odds are
            // that if it has this object, it has others related to it so
            // getting the pack is a good bet.
            //
            if (downloadPackedObject(pm, id))
                return;

            // Search for a loose object over all alternates, starting
            // from the one we last successfully located an object through.
            //
            final String idStr = id.name();
            final String subdir = idStr.substring(0, 2);
            final String file = idStr.substring(2);
            final String looseName = subdir + "/" + file; //$NON-NLS-1$

            for (int i = lastRemoteIdx; i < remotes.size(); i++) {
                if (downloadLooseObject(id, looseName, remotes.get(i))) {
                    lastRemoteIdx = i;
                    return;
                }
            }
            for (int i = 0; i < lastRemoteIdx; i++) {
                if (downloadLooseObject(id, looseName, remotes.get(i))) {
                    lastRemoteIdx = i;
                    return;
                }
            }

            // Try to obtain more pack information and search those.
            //
            while (!noPacksYet.isEmpty()) {
                final WalkRemoteObjectDatabase wrr = noPacksYet.removeFirst();
                final Collection<String> packNameList;
                try {
                    pm.beginTask(JGitText.get().listingPacks,
                            ProgressMonitor.UNKNOWN);
                    packNameList = wrr.getPackNames();
                } catch (IOException e) {
                    // Try another repository.
                    //
                    recordError(id, e);
                    continue;
                } finally {
                    pm.endTask();
                }

                if (packNameList == null || packNameList.isEmpty())
                    continue;
                for (String packName : packNameList) {
                    if (packsConsidered.add(packName))
                        unfetchedPacks.add(new RemotePack(wrr, packName));
                }
                if (downloadPackedObject(pm, id))
                    return;
            }

            // Try to expand the first alternate we haven't expanded yet.
            //
            Collection<WalkRemoteObjectDatabase> al = expandOneAlternate(id, pm);
            if (al != null && !al.isEmpty()) {
                for (WalkRemoteObjectDatabase alt : al) {
                    remotes.add(alt);
                    noPacksYet.add(alt);
                    noAlternatesYet.add(alt);
                }
                continue;
            }

            // We could not obtain the object. There may be reasons why.
            //
            List<Throwable> failures = fetchErrors.get(id);
            final TransportException te;

            te = new TransportException(MessageFormat.format(JGitText.get().cannotGet, id.name()));
            if (failures != null && !failures.isEmpty()) {
                if (failures.size() == 1)
                    te.initCause(failures.get(0));
                else
                    te.initCause(new CompoundException(failures));
            }
            throw te;
        }
    }

    private boolean alreadyHave(AnyObjectId id) throws TransportException {
        try {
            return reader.has(id);
        } catch (IOException error) {
            throw new TransportException(MessageFormat.format(
                    JGitText.get().cannotReadObject, id.name()), error);
        }
    }

    private boolean downloadPackedObject(final ProgressMonitor monitor,
            final AnyObjectId id) throws TransportException {
        // Search for the object in a remote pack whose index we have,
        // but whose pack we do not yet have.
        //
        final Iterator<RemotePack> packItr = unfetchedPacks.iterator();
        while (packItr.hasNext() && !monitor.isCancelled()) {
            final RemotePack pack = packItr.next();
            try {
                pack.openIndex(monitor);
            } catch (IOException err) {
                // If the index won't open its either not found or
                // its a format we don't recognize. In either case
                // we may still be able to obtain the object from
                // another source, so don't consider it a failure.
                //
                recordError(id, err);
                packItr.remove();
                continue;
            }

            if (monitor.isCancelled()) {
                // If we were cancelled while the index was opening
                // the open may have aborted. We can't search an
                // unopen index.
                //
                return false;
            }

            if (!pack.index.hasObject(id)) {
                // Not in this pack? Try another.
                //
                continue;
            }

            // It should be in the associated pack. Download that
            // and attach it to the local repository so we can use
            // all of the contained objects.
            //
            Throwable e1 = null;
            try {
                pack.downloadPack(monitor);
            } catch (IOException err) {
                // If the pack failed to download, index correctly,
                // or open in the local repository we may still be
                // able to obtain this object from another pack or
                // an alternate.
                //
                recordError(id, err);
                e1 = err;
                continue;
            } finally {
                // If the pack was good its in the local repository
                // and Repository.getObjectDatabase().has(id) will
                // succeed in the future, so we do not need this
                // data any more. If it failed the index and pack
                // are unusable and we shouldn't consult them again.
                //
                try {
                    if (pack.tmpIdx != null)
                        FileUtils.delete(pack.tmpIdx);
                } catch (IOException e) {
                    if (e1 != null) {
                        e.addSuppressed(e1);
                    }
                    throw new TransportException(e.getMessage(), e);
                }
                packItr.remove();
            }

            if (!alreadyHave(id)) {
                // What the hell? This pack claimed to have
                // the object, but after indexing we didn't
                // actually find it in the pack.
                //
                recordError(id, new FileNotFoundException(MessageFormat.format(
                        JGitText.get().objectNotFoundIn, id.name(), pack.packName)));
                continue;
            }

            // Complete any other objects that we can.
            //
            final Iterator<ObjectId> pending = swapFetchQueue();
            while (pending.hasNext()) {
                final ObjectId p = pending.next();
                if (pack.index.hasObject(p)) {
                    pending.remove();
                    process(p);
                } else {
                    workQueue.add(p);
                }
            }
            return true;

        }
        return false;
    }

    private Iterator<ObjectId> swapFetchQueue() {
        final Iterator<ObjectId> r = workQueue.iterator();
        workQueue = new LinkedList<>();
        return r;
    }

    private boolean downloadLooseObject(final AnyObjectId id,
            final String looseName, final WalkRemoteObjectDatabase remote)
            throws TransportException {
        try {
            final byte[] compressed = remote.open(looseName).toArray();
            verifyAndInsertLooseObject(id, compressed);
            return true;
        } catch (FileNotFoundException e) {
            // Not available in a loose format from this alternate?
            // Try another strategy to get the object.
            //
            recordError(id, e);
            return false;
        } catch (IOException e) {
            throw new TransportException(MessageFormat.format(JGitText.get().cannotDownload, id.name()), e);
        }
    }

    private void verifyAndInsertLooseObject(final AnyObjectId id,
            final byte[] compressed) throws IOException {
        final ObjectLoader uol;
        try {
            uol = UnpackedObject.parse(compressed, id);
        } catch (CorruptObjectException parsingError) {
            // Some HTTP servers send back a "200 OK" status with an HTML
            // page that explains the requested file could not be found.
            // These servers are most certainly misconfigured, but many
            // of them exist in the world, and many of those are hosting
            // Git repositories.
            //
            // Since an HTML page is unlikely to hash to one of our loose
            // objects we treat this condition as a FileNotFoundException
            // and attempt to recover by getting the object from another
            // source.
            //
            final FileNotFoundException e;
            e = new FileNotFoundException(id.name());
            e.initCause(parsingError);
            throw e;
        }

        final int type = uol.getType();
        final byte[] raw = uol.getCachedBytes();
        if (objCheck != null) {
            try {
                objCheck.check(id, type, raw);
            } catch (CorruptObjectException e) {
                throw new TransportException(MessageFormat.format(
                        JGitText.get().transportExceptionInvalid,
                        Constants.typeString(type), id.name(), e.getMessage()));
            }
        }

        ObjectId act = inserter.insert(type, raw);
        if (!AnyObjectId.isEqual(id, act)) {
            throw new TransportException(MessageFormat.format(
                    JGitText.get().incorrectHashFor, id.name(), act.name(),
                    Constants.typeString(type),
                    Integer.valueOf(compressed.length)));
        }
    }

    private Collection<WalkRemoteObjectDatabase> expandOneAlternate(
            final AnyObjectId id, final ProgressMonitor pm) {
        while (!noAlternatesYet.isEmpty()) {
            final WalkRemoteObjectDatabase wrr = noAlternatesYet.removeFirst();
            try {
                pm.beginTask(JGitText.get().listingAlternates, ProgressMonitor.UNKNOWN);
                Collection<WalkRemoteObjectDatabase> altList = wrr
                        .getAlternates();
                if (altList != null && !altList.isEmpty())
                    return altList;
            } catch (IOException e) {
                // Try another repository.
                //
                recordError(id, e);
            } finally {
                pm.endTask();
            }
        }
        return null;
    }

    private void markLocalRefsComplete(Set<ObjectId> have) throws TransportException {
        List<Ref> refs;
        try {
            refs = local.getRefDatabase().getRefs();
        } catch (IOException e) {
            throw new TransportException(e.getMessage(), e);
        }
        for (Ref r : refs) {
            try {
                markLocalObjComplete(revWalk.parseAny(r.getObjectId()));
            } catch (IOException readError) {
                throw new TransportException(MessageFormat.format(JGitText.get().localRefIsMissingObjects, r.getName()), readError);
            }
        }
        for (ObjectId id : have) {
            try {
                markLocalObjComplete(revWalk.parseAny(id));
            } catch (IOException readError) {
                throw new TransportException(MessageFormat.format(JGitText.get().transportExceptionMissingAssumed, id.name()), readError);
            }
        }
    }

    private void markLocalObjComplete(RevObject obj) throws IOException {
        while (obj.getType() == Constants.OBJ_TAG) {
            obj.add(COMPLETE);
            obj = ((RevTag) obj).getObject();
            revWalk.parseHeaders(obj);
        }

        switch (obj.getType()) {
        case Constants.OBJ_BLOB:
            obj.add(COMPLETE);
            break;
        case Constants.OBJ_COMMIT:
            pushLocalCommit((RevCommit) obj);
            break;
        case Constants.OBJ_TREE:
            markTreeComplete((RevTree) obj);
            break;
        }
    }

    private void markLocalCommitsComplete(int until)
            throws TransportException {
        try {
            for (;;) {
                final RevCommit c = localCommitQueue.peek();
                if (c == null || c.getCommitTime() < until)
                    return;
                localCommitQueue.next();

                markTreeComplete(c.getTree());
                for (RevCommit p : c.getParents())
                    pushLocalCommit(p);
            }
        } catch (IOException err) {
            throw new TransportException(JGitText.get().localObjectsIncomplete, err);
        }
    }

    private void pushLocalCommit(RevCommit p)
            throws MissingObjectException, IOException {
        if (p.has(LOCALLY_SEEN))
            return;
        revWalk.parseHeaders(p);
        p.add(LOCALLY_SEEN);
        p.add(COMPLETE);
        p.carry(COMPLETE);
        localCommitQueue.add(p);
    }

    private void markTreeComplete(RevTree tree) throws IOException {
        if (tree.has(COMPLETE))
            return;
        tree.add(COMPLETE);
        treeWalk.reset(tree);
        while (treeWalk.next()) {
            final FileMode mode = treeWalk.getFileMode(0);
            final int sType = mode.getObjectType();

            switch (sType) {
            case Constants.OBJ_BLOB:
                treeWalk.getObjectId(idBuffer, 0);
                revWalk.lookupAny(idBuffer, sType).add(COMPLETE);
                continue;

            case Constants.OBJ_TREE: {
                treeWalk.getObjectId(idBuffer, 0);
                final RevObject o = revWalk.lookupAny(idBuffer, sType);
                if (!o.has(COMPLETE)) {
                    o.add(COMPLETE);
                    treeWalk.enterSubtree();
                }
                continue;
            }
            default:
                if (FileMode.GITLINK.equals(mode))
                    continue;
                treeWalk.getObjectId(idBuffer, 0);
                throw new CorruptObjectException(MessageFormat.format(JGitText.get().corruptObjectInvalidMode3
                        , mode, idBuffer.name(), treeWalk.getPathString(), tree.name()));
            }
        }
    }

    private void recordError(AnyObjectId id, Throwable what) {
        final ObjectId objId = id.copy();
        List<Throwable> errors = fetchErrors.get(objId);
        if (errors == null) {
            errors = new ArrayList<>(2);
            fetchErrors.put(objId, errors);
        }
        errors.add(what);
    }

    private class RemotePack {
        final WalkRemoteObjectDatabase connection;

        final String packName;

        final String idxName;

        File tmpIdx;

        PackIndex index;

        RemotePack(WalkRemoteObjectDatabase c, String pn) {
            connection = c;
            packName = pn;
            idxName = packName.substring(0, packName.length() - 5) + ".idx"; //$NON-NLS-1$

            String tn = idxName;
            if (tn.startsWith("pack-")) //$NON-NLS-1$
                tn = tn.substring(5);
            if (tn.endsWith(".idx")) //$NON-NLS-1$
                tn = tn.substring(0, tn.length() - 4);

            if (local.getObjectDatabase() instanceof ObjectDirectory) {
                tmpIdx = new File(((ObjectDirectory) local.getObjectDatabase())
                                .getDirectory(),
                        "walk-" + tn + ".walkidx"); //$NON-NLS-1$ //$NON-NLS-2$
            }
        }

        void openIndex(ProgressMonitor pm) throws IOException {
            if (index != null)
                return;
            if (tmpIdx == null)
                tmpIdx = File.createTempFile("jgit-walk-", ".idx"); //$NON-NLS-1$ //$NON-NLS-2$
            else if (tmpIdx.isFile()) {
                try {
                    index = PackIndex.open(tmpIdx);
                    return;
                } catch (FileNotFoundException err) {
                    // Fall through and get the file.
                }
            }

            final WalkRemoteObjectDatabase.FileStream s;
            s = connection.open("pack/" + idxName); //$NON-NLS-1$
            pm.beginTask("Get " + idxName.substring(0, 12) + "..idx", //$NON-NLS-1$ //$NON-NLS-2$
                    s.length < 0 ? ProgressMonitor.UNKNOWN
                            : (int) (s.length / 1024));
            try (FileOutputStream fos = new FileOutputStream(tmpIdx)) {
                final byte[] buf = new byte[2048];
                int cnt;
                while (!pm.isCancelled() && (cnt = s.in.read(buf)) >= 0) {
                    fos.write(buf, 0, cnt);
                    pm.update(cnt / 1024);
                }
            } catch (IOException err) {
                FileUtils.delete(tmpIdx);
                throw err;
            } finally {
                s.in.close();
            }
            pm.endTask();

            if (pm.isCancelled()) {
                FileUtils.delete(tmpIdx);
                return;
            }

            try {
                index = PackIndex.open(tmpIdx);
            } catch (IOException e) {
                FileUtils.delete(tmpIdx);
                throw e;
            }
        }

        void downloadPack(ProgressMonitor monitor) throws IOException {
            String name = "pack/" + packName; //$NON-NLS-1$
            WalkRemoteObjectDatabase.FileStream s = connection.open(name);
            try {
                PackParser parser = inserter.newPackParser(s.in);
                parser.setAllowThin(false);
                parser.setObjectChecker(objCheck);
                parser.setLockMessage(lockMessage);
                PackLock lock = parser.parse(monitor);
                if (lock != null)
                    packLocks.add(lock);
            } finally {
                s.in.close();
            }
        }
    }
}