diff --git a/README.md b/README.md index 44e8642..9502504 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,14 @@ m.AddFromFile("/path/to/repo/src/.gitignore", "src") m.AddPatterns([]byte("*.log\nbuild/\n"), "") ``` +To limit the bytes read from each ignore file, pass `MaxIgnoreFileSize` to any constructor or walk function: + +```go +m := gitignore.NewFromDirectory("/path/to/repo", gitignore.MaxIgnoreFileSize(1<<20)) +``` + +The limit applies to global excludes, `.git/info/exclude`, and `.gitignore` files, including later `AddFromFile` calls. Oversized files are skipped entirely and recorded in `Errors()` with their source path and line zero. Nonpositive limits are unlimited. `AddPatterns` is unaffected, and the limit does not cap total memory across files. + ## Matching `Match` uses the trailing-slash convention to distinguish files from directories. If you already know whether the path is a directory, `MatchPath` avoids that: @@ -85,6 +93,8 @@ gitignore.WalkFrom("/path/to/repo", "src/pkg", func(path string, d fs.DirEntry) }) ``` +`Walk` and `WalkFrom` accept the same options as trailing arguments. With `MaxIgnoreFileSize` set they stop and return an `*IgnoreFileSizeError` when an oversized file is encountered. Its `Path` and `Limit` fields identify the file and configured byte limit; use `errors.As` to inspect it. Callbacks may already have run for earlier entries. + ## Error handling Invalid patterns (like unknown POSIX character classes) are silently skipped during matching. To inspect them: diff --git a/gitignore.go b/gitignore.go index 2b3588b..7c30608 100644 --- a/gitignore.go +++ b/gitignore.go @@ -41,19 +41,23 @@ type pattern struct { // AddPatterns/AddFromFile call). Do not call AddPatterns or AddFromFile // concurrently with Match. type Matcher struct { - patterns []pattern - errors []PatternError + maxIgnoreFileSize int64 + patterns []pattern + errors []PatternError } -// PatternError records a pattern that could not be compiled. +// PatternError records a pattern compilation error or a skipped oversized file. type PatternError struct { Pattern string // the original pattern text Source string // file path, empty for programmatic patterns - Line int // 1-based line number + Line int // 1-based line number; zero for a file-size error Message string } func (e PatternError) Error() string { + if e.Line == 0 && e.Source != "" { + return e.Source + ": " + e.Message + } if e.Source != "" { return e.Source + ":" + itoa(e.Line) + ": invalid pattern: " + e.Pattern + ": " + e.Message } @@ -74,9 +78,8 @@ func itoa(n int) string { return string(buf[i:]) } -// Errors returns any pattern compilation errors encountered while loading -// patterns. Invalid patterns are silently skipped during matching; this -// method lets callers detect and report them. +// Errors returns pattern compilation errors and skipped oversized files. +// File-size errors have a source path and a zero line number. func (m *Matcher) Errors() []PatternError { return m.errors } @@ -92,33 +95,39 @@ func (m *Matcher) Errors() []PatternError { // (containing .git/). If root is empty, no filesystem patterns are // loaded and the returned Matcher is empty. Use AddPatterns or // AddFromFile to add patterns programmatically. -func New(root string) *Matcher { +// +// Options such as MaxIgnoreFileSize apply to files loaded here and to +// later AddFromFile calls; oversized files are skipped and recorded in +// Errors with Line set to zero. +func New(root string, opts ...Option) *Matcher { + m, _ := newMatcher(root, opts) + return m +} + +func newMatcher(root string, opts []Option) (*Matcher, error) { m := &Matcher{} + for _, opt := range opts { + opt(m) + } if root == "" { - return m + return m, nil } - // Read global excludes (lowest priority) - if gef := globalExcludesFile(); gef != "" { - if data, err := os.ReadFile(gef); err == nil { - m.addPatterns(data, "", gef) + var firstErr error + for _, path := range []string{ + globalExcludesFile(), + filepath.Join(root, ".git", "info", "exclude"), + filepath.Join(root, ".gitignore"), + } { + if path == "" { + continue + } + if err := m.addFromFile(path, ""); firstErr == nil { + firstErr = err } } - - // Read .git/info/exclude - excludePath := filepath.Join(root, ".git", "info", "exclude") - if data, err := os.ReadFile(excludePath); err == nil { - m.addPatterns(data, "", excludePath) - } - - // Read root .gitignore (highest priority) - ignorePath := filepath.Join(root, ".gitignore") - if data, err := os.ReadFile(ignorePath); err == nil { - m.addPatterns(data, "", ignorePath) - } - - return m + return m, firstErr } // globalExcludesFile returns the path to the user's global gitignore file. @@ -170,10 +179,11 @@ func expandTilde(path string) string { // NewFromDirectory creates a Matcher by walking the directory tree rooted // at root, loading every .gitignore file found along the way. Each nested // .gitignore is scoped to its containing directory. The .git directory is -// skipped. -func NewFromDirectory(root string) *Matcher { - m := New(root) - _ = walkRecursive(root, "", m, nil) +// skipped. Oversized files under MaxIgnoreFileSize are skipped and recorded +// in Errors. +func NewFromDirectory(root string, opts ...Option) *Matcher { + m := New(root, opts...) + _ = walkRecursive(root, "", m, nil, false) return m } @@ -184,9 +194,15 @@ func NewFromDirectory(root string) *Matcher { // // Paths passed to fn are relative to root and use the OS path separator. // The root directory itself is not passed to fn. -func Walk(root string, fn func(path string, d fs.DirEntry) error) error { - m := New(root) - return walkRecursive(root, "", m, fn) +// +// With MaxIgnoreFileSize set, an oversized ignore file stops the walk and +// is returned as an *IgnoreFileSizeError. +func Walk(root string, fn func(path string, d fs.DirEntry) error, opts ...Option) error { + m, err := newMatcher(root, opts) + if err != nil { + return err + } + return walkRecursive(root, "", m, fn, true) } // WalkFrom walks the directory tree starting at a subdirectory of root, @@ -200,17 +216,23 @@ func Walk(root string, fn func(path string, d fs.DirEntry) error) error { // using either forward slashes or the OS path separator. Paths passed // to fn are relative to root (not to start) and use the OS path // separator. The start directory itself is passed to fn. -func WalkFrom(root, start string, fn func(path string, d fs.DirEntry) error) error { +// +// With MaxIgnoreFileSize set, an oversized ignore file stops the walk and +// is returned as an *IgnoreFileSizeError. +func WalkFrom(root, start string, fn func(path string, d fs.DirEntry) error, opts ...Option) error { if start == "" || start == "." { - return Walk(root, fn) + return Walk(root, fn, opts...) } start = filepath.Clean(start) if start == "." { - return Walk(root, fn) + return Walk(root, fn, opts...) } - m := New(root) + m, err := newMatcher(root, opts) + if err != nil { + return err + } // Load .gitignore from each ancestor directory between root and start // (exclusive of start itself, which walkRecursive loads). @@ -222,7 +244,9 @@ func WalkFrom(root, start string, fn func(path string, d fs.DirEntry) error) err break } prefix := slashed[:off+i] - m.AddFromFile(filepath.Join(root, prefix, ".gitignore"), prefix) + if err := m.addFromFile(filepath.Join(root, prefix, ".gitignore"), prefix); err != nil { + return err + } off += i + 1 } } @@ -239,10 +263,10 @@ func WalkFrom(root, start string, fn func(path string, d fs.DirEntry) error) err } } - return walkRecursive(root, start, m, fn) + return walkRecursive(root, start, m, fn, true) } -func walkRecursive(root, rel string, m *Matcher, fn func(string, fs.DirEntry) error) error { +func walkRecursive(root, rel string, m *Matcher, fn func(string, fs.DirEntry) error, stopOnSizeError bool) error { dir := root if rel != "" { dir = filepath.Join(root, rel) @@ -250,7 +274,9 @@ func walkRecursive(root, rel string, m *Matcher, fn func(string, fs.DirEntry) er // Load .gitignore for this directory before processing entries. if rel != "" { - m.AddFromFile(filepath.Join(dir, ".gitignore"), filepath.ToSlash(rel)) + if err := m.addFromFile(filepath.Join(dir, ".gitignore"), filepath.ToSlash(rel)); err != nil && stopOnSizeError { + return err + } } entries, err := os.ReadDir(dir) @@ -282,7 +308,7 @@ func walkRecursive(root, rel string, m *Matcher, fn func(string, fs.DirEntry) er } if entry.IsDir() { - if err := walkRecursive(root, entryRel, m, fn); err != nil { + if err := walkRecursive(root, entryRel, m, fn, stopOnSizeError); err != nil { return err } } @@ -298,13 +324,23 @@ func (m *Matcher) AddPatterns(data []byte, dir string) { } // AddFromFile reads a .gitignore file at the given absolute path and scopes -// its patterns to the given relative directory. +// its patterns to the given relative directory. It uses the matcher's file-size +// limit, if set, and records oversized files in Errors without applying any rules. func (m *Matcher) AddFromFile(absPath, relDir string) { - data, err := os.ReadFile(absPath) + _ = m.addFromFile(absPath, relDir) +} + +func (m *Matcher) addFromFile(absPath, relDir string) error { + data, err := readIgnoreFile(absPath, m.maxIgnoreFileSize) if err != nil { - return + if sizeErr, ok := err.(*IgnoreFileSizeError); ok { + m.errors = append(m.errors, PatternError{Source: absPath, Message: sizeErr.message()}) + return err + } + return nil } m.addPatterns(data, relDir, absPath) + return nil } // Match returns true if the given path should be ignored. diff --git a/options.go b/options.go new file mode 100644 index 0000000..0d36beb --- /dev/null +++ b/options.go @@ -0,0 +1,63 @@ +package gitignore + +import ( + "io" + "os" + "strconv" +) + +// Option configures a Matcher at construction time. +type Option func(*Matcher) + +// MaxIgnoreFileSize limits the bytes read from each ignore file. Nonpositive +// values are unlimited. AddPatterns is unaffected because its data is already +// in memory. +func MaxIgnoreFileSize(n int64) Option { + return func(m *Matcher) { m.maxIgnoreFileSize = n } +} + +// IgnoreFileSizeError reports an ignore file that exceeded its byte limit. +type IgnoreFileSizeError struct { + Path string + Limit int64 +} + +func (e *IgnoreFileSizeError) Error() string { + return e.Path + ": " + e.message() +} + +func (e *IgnoreFileSizeError) message() string { + return "ignore file exceeds size limit of " + strconv.FormatInt(e.Limit, 10) + " bytes" +} + +func readIgnoreFile(path string, limit int64) ([]byte, error) { + if limit <= 0 { + return os.ReadFile(path) + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + info, err := f.Stat() + if err != nil { + return nil, err + } + if info.Size() > limit { + return nil, &IgnoreFileSizeError{Path: path, Limit: limit} + } + data, err := io.ReadAll(io.LimitReader(f, limit)) + if err != nil { + return nil, err + } + if int64(len(data)) < limit { + return data, nil + } + // The file can grow after Stat. Probe without overflowing limit+1. + if n, err := io.CopyN(io.Discard, f, 1); n != 0 { + return nil, &IgnoreFileSizeError{Path: path, Limit: limit} + } else if err != io.EOF { + return nil, err + } + return data, nil +} diff --git a/options_test.go b/options_test.go new file mode 100644 index 0000000..42df7b2 --- /dev/null +++ b/options_test.go @@ -0,0 +1,125 @@ +package gitignore_test + +import ( + "errors" + "math" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/git-pkgs/gitignore" +) + +func writeIgnoreFile(t *testing.T, path, data string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(data), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestIgnoreFileSizeBoundary(t *testing.T) { + isolateGitEnv(t) + root := t.TempDir() + path := filepath.Join(root, ".gitignore") + writeIgnoreFile(t, path, "*.log") + for _, limit := range []int64{-1, 0, 4, 5, 6, math.MaxInt64} { + m := gitignore.New(root, gitignore.MaxIgnoreFileSize(limit)) + if got := m.Match("app.log"); got != (limit != 4) { + t.Errorf("limit %d: Match = %v", limit, got) + } + if limit == 4 { + checkSizeDiagnostic(t, m, path) + } else if len(m.Errors()) != 0 { + t.Errorf("limit %d: Errors = %v", limit, m.Errors()) + } + } + m := gitignore.New("", gitignore.MaxIgnoreFileSize(4)) + m.AddFromFile(path, "src") + checkSizeDiagnostic(t, m, path) + if m.Match("src/app.log") { + t.Fatal("oversized file was partially applied") + } + m.AddPatterns([]byte("*.log"), "src") + if !m.Match("src/app.log") { + t.Fatal("file limit applied to AddPatterns") + } + if !gitignore.New(root).Match("app.log") { + t.Fatal("default constructor changed") + } +} + +func checkSizeDiagnostic(t *testing.T, m *gitignore.Matcher, path string) { + t.Helper() + errs := m.Errors() + if len(errs) != 1 { + t.Fatalf("Errors = %v", errs) + } + if errs[0].Source != path || errs[0].Line != 0 || errs[0].Pattern != "" { + t.Errorf("diagnostic = %+v", errs[0]) + } + if !strings.Contains(errs[0].Error(), "size limit") || strings.Contains(errs[0].Error(), "invalid pattern") { + t.Errorf("diagnostic text = %s", errs[0].Error()) + } +} + +func TestIgnoreFileSizeSources(t *testing.T) { + isolateGitEnv(t) + for _, source := range []string{".gitignore", ".git/info/exclude", ".global/git/ignore", "src/.gitignore"} { + t.Run(source, func(t *testing.T) { checkSizeLimitedSource(t, source) }) + } +} + +func checkSizeLimitedSource(t *testing.T, source string) { + t.Helper() + root := t.TempDir() + t.Setenv("XDG_CONFIG_HOME", filepath.Join(root, ".global")) + path := filepath.Join(root, source) + writeIgnoreFile(t, path, "*.log\n") + writeIgnoreFile(t, filepath.Join(root, "src/deep/file.log"), "") + opt := gitignore.MaxIgnoreFileSize(5) + m := gitignore.NewFromDirectory(root, opt) + checkSizeDiagnostic(t, m, path) + if m.Match("src/deep/file.log") { + t.Fatal("skipped rules were applied") + } + for _, start := range []string{"", ".", "src", "src/deep"} { + err := gitignore.WalkFrom(root, start, func(path string, _ os.DirEntry) error { + if filepath.ToSlash(path) == "src/deep/file.log" { + t.Error("walk continued past oversized ignore file") + } + return nil + }, opt) + checkSizeError(t, err, path, 5) + } + checkSizeError(t, gitignore.Walk(root, nil, opt), path, 5) + if err := gitignore.Walk(root, nil); err != nil { + t.Fatal(err) + } +} + +func checkSizeError(t *testing.T, err error, path string, limit int64) { + t.Helper() + var sizeErr *gitignore.IgnoreFileSizeError + if !errors.As(err, &sizeErr) { + t.Fatalf("error = %v, want IgnoreFileSizeError", err) + } + if sizeErr.Path != path || sizeErr.Limit != limit { + t.Errorf("error = %+v", sizeErr) + } +} + +func TestSizeLimitedDiscoveryContinues(t *testing.T) { + isolateGitEnv(t) + root := t.TempDir() + writeIgnoreFile(t, filepath.Join(root, "a/.gitignore"), "*.log\n") + writeIgnoreFile(t, filepath.Join(root, "b/.gitignore"), "*.go") + m := gitignore.NewFromDirectory(root, gitignore.MaxIgnoreFileSize(5)) + if !m.Match("b/main.go") { + t.Fatal("discovery stopped after oversized file") + } + checkSizeDiagnostic(t, m, filepath.Join(root, "a/.gitignore")) +}