Teach gitscanner how to filter in ScanTree()

commands/command_checkout.go

@@ -35,12 +35,10 @@ func checkoutCommand(cmd *cobra.Command, args []string) {
 	close(inchan)
 
 	filter := filepathfilter.New(rootedpaths, nil)
-	gitscanner := lfs.NewGitScanner(nil)
-	defer gitscanner.Close()
-	checkoutWithIncludeExclude(gitscanner, filter)
+	checkoutWithIncludeExclude(filter)
 }
 
-func checkoutFromFetchChan(gitscanner *lfs.GitScanner, filter *filepathfilter.Filter, in chan *lfs.WrappedPointer) {
+func checkoutFromFetchChan(in chan *lfs.WrappedPointer) {
 	ref, err := git.CurrentRef()
 	if err != nil {
 		Panic(err, "Could not checkout")
@@ -54,10 +52,7 @@ func checkoutFromFetchChan(gitscanner *lfs.GitScanner, filter *filepathfilter.Fi
 			Panic(err, "Could not scan for Git LFS files")
 			return
 		}
-
-		if filter.Allows(p.Name) {
-			mapping[p.Oid] = append(mapping[p.Oid], p)
-		}
+		mapping[p.Oid] = append(mapping[p.Oid], p)
 	})
 
 	if err := chgitscanner.ScanTree(ref.Sha, nil); err != nil {
@@ -88,7 +83,7 @@ func checkoutFromFetchChan(gitscanner *lfs.GitScanner, filter *filepathfilter.Fi
 	wait.Wait()
 }
 
-func checkoutWithIncludeExclude(gitscanner *lfs.GitScanner, filter *filepathfilter.Filter) {
+func checkoutWithIncludeExclude(filter *filepathfilter.Filter) {
 	ref, err := git.CurrentRef()
 	if err != nil {
 		Panic(err, "Could not checkout")
@@ -110,6 +105,8 @@ func checkoutWithIncludeExclude(gitscanner *lfs.GitScanner, filter *filepathfilt
 		pointers = append(pointers, p)
 	})
 
+	chgitscanner.Filter = filter
+
 	if err := chgitscanner.ScanTree(ref.Sha, nil); err != nil {
 		ExitWithError(err)
 	}
@@ -141,16 +138,12 @@ func checkoutWithIncludeExclude(gitscanner *lfs.GitScanner, filter *filepathfilt
 	totalBytes = 0
 	for _, pointer := range pointers {
 		totalBytes += pointer.Size
-		if filter.Allows(pointer.Name) {
-			progress.Add(pointer.Name)
-			c <- pointer
-			// not strictly correct (parallel) but we don't have a callback & it's just local
-			// plus only 1 slot in channel so it'll block & be close
-			progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size))
-			progress.FinishTransfer(pointer.Name)
-		} else {
-			progress.Skip(pointer.Size)
-		}
+		progress.Add(pointer.Name)
+		c <- pointer
+		// not strictly correct (parallel) but we don't have a callback & it's just local
+		// plus only 1 slot in channel so it'll block & be close
+		progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size))
+		progress.FinishTransfer(pointer.Name)
 	}
 	close(c)
 	wait.Wait()

commands/command_clone.go

@@ -6,7 +6,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/git-lfs/git-lfs/lfs"
 	"github.com/git-lfs/git-lfs/localstorage"
 	"github.com/git-lfs/git-lfs/subprocess"
 
@@ -71,13 +70,11 @@ func cloneCommand(cmd *cobra.Command, args []string) {
 
 	includeArg, excludeArg := getIncludeExcludeArgs(cmd)
 	filter := buildFilepathFilter(cfg, includeArg, excludeArg)
-	gitscanner := lfs.NewGitScanner(nil)
-	defer gitscanner.Close()
 	if cloneFlags.NoCheckout || cloneFlags.Bare {
 		// If --no-checkout or --bare then we shouldn't check out, just fetch instead
-		fetchRef(gitscanner, "HEAD", filter)
+		fetchRef("HEAD", filter)
 	} else {
-		pull(gitscanner, filter)
+		pull(filter)
 		err := postCloneSubmodules(args)
 		if err != nil {
 			Exit("Error performing 'git lfs pull' for submodules: %v", err)

commands/command_fetch.go

@@ -76,20 +76,20 @@ func fetchCommand(cmd *cobra.Command, args []string) {
 		if len(cfg.FetchIncludePaths()) > 0 || len(cfg.FetchExcludePaths()) > 0 {
 			Print("Ignoring global include / exclude paths to fulfil --all")
 		}
-		success = fetchAll(gitscanner)
+		success = fetchAll()
 
 	} else { // !all
 		filter := buildFilepathFilter(cfg, include, exclude)
 
 		// Fetch refs sequentially per arg order; duplicates in later refs will be ignored
 		for _, ref := range refs {
 			Print("Fetching %v", ref.Name)
-			s := fetchRef(gitscanner, ref.Sha, filter)
+			s := fetchRef(ref.Sha, filter)
 			success = success && s
 		}
 
 		if fetchRecentArg || cfg.FetchPruneConfig().FetchRecentAlways {
-			s := fetchRecent(gitscanner, refs, filter)
+			s := fetchRecent(refs, filter)
 			success = success && s
 		}
 	}
@@ -106,7 +106,7 @@ func fetchCommand(cmd *cobra.Command, args []string) {
 	}
 }
 
-func pointersToFetchForRef(gitscanner *lfs.GitScanner, ref string) ([]*lfs.WrappedPointer, error) {
+func pointersToFetchForRef(ref string, filter *filepathfilter.Filter) ([]*lfs.WrappedPointer, error) {
 	var pointers []*lfs.WrappedPointer
 	var multiErr error
 	tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
@@ -122,6 +122,8 @@ func pointersToFetchForRef(gitscanner *lfs.GitScanner, ref string) ([]*lfs.Wrapp
 		pointers = append(pointers, p)
 	})
 
+	tempgitscanner.Filter = filter
+
 	if err := tempgitscanner.ScanTree(ref, nil); err != nil {
 		return nil, err
 	}
@@ -130,9 +132,9 @@ func pointersToFetchForRef(gitscanner *lfs.GitScanner, ref string) ([]*lfs.Wrapp
 	return pointers, multiErr
 }
 
-func fetchRefToChan(gitscanner *lfs.GitScanner, ref string, filter *filepathfilter.Filter) chan *lfs.WrappedPointer {
+func fetchRefToChan(ref string, filter *filepathfilter.Filter) chan *lfs.WrappedPointer {
 	c := make(chan *lfs.WrappedPointer)
-	pointers, err := pointersToFetchForRef(gitscanner, ref)
+	pointers, err := pointersToFetchForRef(ref, filter)
 	if err != nil {
 		Panic(err, "Could not scan for Git LFS files")
 	}
@@ -143,8 +145,8 @@ func fetchRefToChan(gitscanner *lfs.GitScanner, ref string, filter *filepathfilt
 }
 
 // Fetch all binaries for a given ref (that we don't have already)
-func fetchRef(gitscanner *lfs.GitScanner, ref string, filter *filepathfilter.Filter) bool {
-	pointers, err := pointersToFetchForRef(gitscanner, ref)
+func fetchRef(ref string, filter *filepathfilter.Filter) bool {
+	pointers, err := pointersToFetchForRef(ref, filter)
 	if err != nil {
 		Panic(err, "Could not scan for Git LFS files")
 	}
@@ -153,7 +155,7 @@ func fetchRef(gitscanner *lfs.GitScanner, ref string, filter *filepathfilter.Fil
 
 // Fetch all previous versions of objects from since to ref (not including final state at ref)
 // So this will fetch all the '-' sides of the diff from since to ref
-func fetchPreviousVersions(gitscanner *lfs.GitScanner, ref string, since time.Time, filter *filepathfilter.Filter) bool {
+func fetchPreviousVersions(ref string, since time.Time, filter *filepathfilter.Filter) bool {
 	var pointers []*lfs.WrappedPointer
 
 	tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
@@ -165,6 +167,8 @@ func fetchPreviousVersions(gitscanner *lfs.GitScanner, ref string, since time.Ti
 		pointers = append(pointers, p)
 	})
 
+	tempgitscanner.Filter = filter
+
 	if err := tempgitscanner.ScanPreviousVersions(ref, since, nil); err != nil {
 		ExitWithError(err)
 	}
@@ -174,7 +178,7 @@ func fetchPreviousVersions(gitscanner *lfs.GitScanner, ref string, since time.Ti
 }
 
 // Fetch recent objects based on config
-func fetchRecent(gitscanner *lfs.GitScanner, alreadyFetchedRefs []*git.Ref, filter *filepathfilter.Filter) bool {
+func fetchRecent(alreadyFetchedRefs []*git.Ref, filter *filepathfilter.Filter) bool {
 	fetchconf := cfg.FetchPruneConfig()
 
 	if fetchconf.FetchRecentRefsDays == 0 && fetchconf.FetchRecentCommitsDays == 0 {
@@ -204,7 +208,7 @@ func fetchRecent(gitscanner *lfs.GitScanner, alreadyFetchedRefs []*git.Ref, filt
 			} else {
 				uniqueRefShas[ref.Sha] = ref.Name
 				Print("Fetching %v", ref.Name)
-				k := fetchRef(gitscanner, ref.Sha, filter)
+				k := fetchRef(ref.Sha, filter)
 				ok = ok && k
 			}
 		}
@@ -220,21 +224,21 @@ func fetchRecent(gitscanner *lfs.GitScanner, alreadyFetchedRefs []*git.Ref, filt
 			}
 			Print("Fetching changes within %v days of %v", fetchconf.FetchRecentCommitsDays, refName)
 			commitsSince := summ.CommitDate.AddDate(0, 0, -fetchconf.FetchRecentCommitsDays)
-			k := fetchPreviousVersions(gitscanner, commit, commitsSince, filter)
+			k := fetchPreviousVersions(commit, commitsSince, filter)
 			ok = ok && k
 		}
 
 	}
 	return ok
 }
 
-func fetchAll(gitscanner *lfs.GitScanner) bool {
-	pointers := scanAll(gitscanner)
+func fetchAll() bool {
+	pointers := scanAll()
 	Print("Fetching objects...")
 	return fetchAndReportToChan(pointers, nil, nil)
 }
 
-func scanAll(gitscanner *lfs.GitScanner) []*lfs.WrappedPointer {
+func scanAll() []*lfs.WrappedPointer {
 	// This could be a long process so use the chan version & report progress
 	Print("Scanning for all objects ever referenced...")
 	spinner := progress.NewSpinner()
@@ -343,11 +347,6 @@ func readyAndMissingPointers(allpointers []*lfs.WrappedPointer, filter *filepath
 	ready := make([]*lfs.WrappedPointer, 0, len(allpointers))
 
 	for _, p := range allpointers {
-		// Filtered out by --include or --exclude
-		if !filter.Allows(p.Name) {
-			continue
-		}
-
 		// no need to download the same object multiple times
 		if seen[p.Oid] {
 			continue

commands/command_pull.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/git-lfs/git-lfs/filepathfilter"
 	"github.com/git-lfs/git-lfs/git"
-	"github.com/git-lfs/git-lfs/lfs"
 	"github.com/spf13/cobra"
 )
 
@@ -30,19 +29,17 @@ func pullCommand(cmd *cobra.Command, args []string) {
 
 	includeArg, excludeArg := getIncludeExcludeArgs(cmd)
 	filter := buildFilepathFilter(cfg, includeArg, excludeArg)
-	gitscanner := lfs.NewGitScanner(nil)
-	defer gitscanner.Close()
-	pull(gitscanner, filter)
+	pull(filter)
 }
 
-func pull(gitscanner *lfs.GitScanner, filter *filepathfilter.Filter) {
+func pull(filter *filepathfilter.Filter) {
 	ref, err := git.CurrentRef()
 	if err != nil {
 		Panic(err, "Could not pull")
 	}
 
-	c := fetchRefToChan(gitscanner, ref.Sha, filter)
-	checkoutFromFetchChan(gitscanner, filter, c)
+	c := fetchRefToChan(ref.Sha, filter)
+	checkoutFromFetchChan(c)
 }
 
 func init() {

lfs/gitscanner.go

@@ -137,7 +137,7 @@ func (s *GitScanner) ScanTree(ref string, cb GitScannerCallback) error {
 	if err != nil {
 		return err
 	}
-	return runScanTree(callback, ref)
+	return runScanTree(callback, ref, s.Filter)
 }
 
 // ScanUnpushed scans history for all LFS pointers which have been added but not

lfs/gitscanner_tree.go

@@ -8,6 +8,8 @@ import (
 	"io/ioutil"
 	"strconv"
 	"strings"
+
+	"github.com/git-lfs/git-lfs/filepathfilter"
 )
 
 // An entry from ls-tree or rev-list including a blob sha and tree path
@@ -16,10 +18,10 @@ type TreeBlob struct {
 	Filename string
 }
 
-func runScanTree(cb GitScannerCallback, ref string) error {
+func runScanTree(cb GitScannerCallback, ref string, filter *filepathfilter.Filter) error {
 	// We don't use the nameMap approach here since that's imprecise when >1 file
 	// can be using the same content
-	treeShas, err := lsTreeBlobs(ref)
+	treeShas, err := lsTreeBlobs(ref, filter)
 	if err != nil {
 		return err
 	}
@@ -96,7 +98,7 @@ func catFileBatchTree(treeblobs *TreeBlobChannelWrapper) (*PointerChannelWrapper
 // Use ls-tree at ref to find a list of candidate tree blobs which might be lfs files
 // The returned channel will be sent these blobs which should be sent to catFileBatchTree
 // for final check & conversion to Pointer
-func lsTreeBlobs(ref string) (*TreeBlobChannelWrapper, error) {
+func lsTreeBlobs(ref string, filter *filepathfilter.Filter) (*TreeBlobChannelWrapper, error) {
 	cmd, err := startCommand("git", "ls-tree",
 		"-r",          // recurse
 		"-l",          // report object size (we'll need this)
@@ -117,7 +119,7 @@ func lsTreeBlobs(ref string) (*TreeBlobChannelWrapper, error) {
 	go func() {
 		scanner := newLsTreeScanner(cmd.Stdout)
 		for scanner.Scan() {
-			if t := scanner.TreeBlob(); t != nil {
+			if t := scanner.TreeBlob(); t != nil && filter.Allows(t.Filename) {
 				blobs <- *t
 			}
 		}