fork/gdu

git clone https://git.julia.blue/fork/gdu

Daniel Mildefix: StoreDir fixes (#644)35cba00

master
14.8 KiB587 linesraw
1package analyze
2
3import (
4	"iter"
5	"os"
6	"path/filepath"
7	"sort"
8	"sync"
9	"time"
10
11	"github.com/dundee/gdu/v5/pkg/fs"
12)
13
14// fileFlagMu protects the mutable Flag field on published File values. File
15// keeps its fields public for compatibility, so hard-link accounting can
16// update Flag after a file has been exposed to preview readers.
17var fileFlagMu sync.RWMutex
18
19var _ fs.SymlinkItem = (*File)(nil)
20
21// File struct
22type File struct {
23	Mtime   time.Time
24	Parent  fs.Item
25	Name    string
26	Symlink string
27	Size    int64
28	Usage   int64
29	Mli     uint64
30	Flag    rune
31}
32
33// GetName returns name of dir
34func (f *File) GetName() string {
35	return f.Name
36}
37
38// IsDir returns false for file
39func (f *File) IsDir() bool {
40	return false
41}
42
43// GetParent returns parent dir
44func (f *File) GetParent() fs.Item {
45	return f.Parent
46}
47
48// SetParent sets parent dir
49func (f *File) SetParent(parent fs.Item) {
50	f.Parent = parent
51}
52
53// GetPath returns absolute Get of the file
54func (f *File) GetPath() string {
55	return filepath.Join(f.Parent.GetPath(), f.Name)
56}
57
58// GetFlag returns flag of the file
59func (f *File) GetFlag() rune {
60	fileFlagMu.RLock()
61	defer fileFlagMu.RUnlock()
62	return f.Flag
63}
64
65// GetSize returns size of the file
66func (f *File) GetSize() int64 {
67	return f.Size
68}
69
70// GetUsage returns usage of the file
71func (f *File) GetUsage() int64 {
72	return f.Usage
73}
74
75// GetMtime returns mtime of the file
76func (f *File) GetMtime() time.Time {
77	return f.Mtime
78}
79
80// GetType returns name type of item
81func (f *File) GetType() string {
82	fileFlagMu.RLock()
83	defer fileFlagMu.RUnlock()
84	if f.Flag == '@' {
85		if f.Symlink != "" {
86			return "Symlink"
87		}
88		return "Other"
89	}
90	return "File"
91}
92
93// GetSymlinkTarget returns the target of a symlink, or empty string
94func (f *File) GetSymlinkTarget() string {
95	return f.Symlink
96}
97
98// GetItemCount returns 1 for file
99func (f *File) GetItemCount() int64 {
100	return 1
101}
102
103// GetMultiLinkedInode returns inode number of multilinked file
104func (f *File) GetMultiLinkedInode() uint64 {
105	return f.Mli
106}
107
108func (f *File) alreadyCounted(linkedItems fs.HardLinkedItems) bool {
109	mli := f.Mli
110	counted := false
111	if mli > 0 {
112		fileFlagMu.Lock()
113		f.Flag = 'H'
114		fileFlagMu.Unlock()
115		if _, ok := linkedItems[mli]; ok {
116			counted = true
117		}
118		linkedItems[mli] = append(linkedItems[mli], f)
119	}
120	return counted
121}
122
123// CreateFileItem creates a File from an os.FileInfo with correct platform-specific attributes
124func CreateFileItem(name string, info os.FileInfo) *File {
125	file := &File{
126		Name: name,
127		Size: info.Size(),
128		Flag: getFlag(info),
129	}
130	setPlatformSpecificAttrs(file, info)
131	return file
132}
133
134// GetItemStats returns 1 as count of items, apparent usage and real usage of this file
135func (f *File) GetItemStats(linkedItems fs.HardLinkedItems, filteringFiles bool) (itemCount, size, usage int64) {
136	if f.alreadyCounted(linkedItems) {
137		return 1, 0, 0
138	}
139	return 1, f.GetSize(), f.GetUsage()
140}
141
142// UpdateStats does nothing on file
143func (f *File) UpdateStats(linkedItems fs.HardLinkedItems)                  {}
144func (f *File) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {}
145
146// GetFiles returns all files in directory
147func (f *File) GetFiles(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
148	return func(yield func(fs.Item) bool) {}
149}
150
151// GetFilesLocked returns all files in directory
152func (f *File) GetFilesLocked(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
153	return f.GetFiles(sortBy, order)
154}
155
156// RLock panics on file
157func (f *File) RLock() func() {
158	panic("RLock should not be called on file")
159}
160
161// AddFile panics on file
162func (f *File) AddFile(item fs.Item) {
163	panic("AddFile should not be called on file")
164}
165
166// RemoveFile panics on file
167func (f *File) RemoveFile(item fs.Item) {
168	panic("RemoveFile should not be called on file")
169}
170
171// RemoveFileByName panics on file
172func (f *File) RemoveFileByName(name string) {
173	panic("RemoveFileByName should not be called on file")
174}
175
176// EmptyDirSize is the apparent size gdu assigns to a directory that has no
177// counted children. It is a fixed sentinel (not the real filesystem block
178// size) written by updateStats, SimpleDir.UpdateStats and the parallel top-dir
179// analyzer, and recognised again on import. Keep every producer and consumer
180// of this value referencing this constant so they can never drift apart.
181const EmptyDirSize int64 = 512
182
183// dirTotals is the aggregate of a directory's counted children, before gdu's
184// empty-directory rule is applied.
185//
186// itemCount includes the directory itself, so it starts at 1. entries is the
187// number of children that were counted, and hasFiles records whether any of
188// them was not itself a directory.
189type dirTotals struct {
190	size      int64
191	usage     int64
192	itemCount int64
193	entries   int
194	hasFiles  bool
195}
196
197// resolveDirStats applies gdu's empty-directory rule to a directory's
198// aggregated child totals.
199//
200// A directory that counted no children — or, when file filtering is active,
201// one whose entries are all themselves empty directories — reports EmptyDirSize
202// as its apparent size and zero real usage. gdu deliberately does not charge a
203// directory for the block the filesystem allocates to it, so a tree containing
204// nothing but empty directories reports no disk usage at all.
205//
206// This is the single definition of that rule. Every analyzer must route through
207// it, otherwise the numbers gdu reports depend on which analyzer produced them.
208func resolveDirStats(totals dirTotals, filteringFiles bool) (itemCount, size, usage int64) {
209	onlyEmptyDirs := !totals.hasFiles && filteringFiles &&
210		totals.itemCount == int64(totals.entries+1)
211
212	if totals.entries == 0 || onlyEmptyDirs {
213		return 1, totals.size + EmptyDirSize, 0
214	}
215	return totals.itemCount, totals.size, totals.usage
216}
217
218// aggregateDirEntries folds the stats of entries into the totals their parent
219// directory should report, and returns the parent's mtime and flag updated from
220// its children. mtime and flag are seeded with the parent's current values.
221func aggregateDirEntries(
222	entries fs.Files,
223	mtime time.Time,
224	flag rune,
225	linkedItems fs.HardLinkedItems,
226	filteringFiles bool,
227) (totals dirTotals, newMtime time.Time, newFlag rune) {
228	totals = dirTotals{itemCount: 1, entries: len(entries)}
229
230	for _, entry := range entries {
231		count, size, usage := entry.GetItemStats(linkedItems, filteringFiles)
232		totals.size += size
233		totals.usage += usage
234		totals.itemCount += count
235
236		if entryMtime := entry.GetMtime(); entryMtime.After(mtime) {
237			mtime = entryMtime
238		}
239
240		if !entry.IsDir() {
241			totals.hasFiles = true
242		}
243
244		switch entry.GetFlag() {
245		case '!', '.':
246			if flag != '!' {
247				flag = '.'
248			}
249		}
250	}
251
252	return totals, mtime, flag
253}
254
255// Dir struct
256type Dir struct {
257	*File
258	BasePath      string
259	Files         fs.Files
260	ItemCount     int64
261	statsFromJSON bool
262	m             sync.RWMutex
263}
264
265func snapshotDir(source *Dir, parent fs.Item) *Dir {
266	source.m.RLock()
267	snapshot := &Dir{
268		File: &File{
269			Mtime:  source.Mtime,
270			Parent: parent,
271			Name:   source.Name,
272			Size:   source.Size,
273			Usage:  source.Usage,
274			Mli:    source.Mli,
275			Flag:   source.Flag,
276		},
277		BasePath:  source.BasePath,
278		Files:     make(fs.Files, 0, len(source.Files)),
279		ItemCount: source.ItemCount,
280	}
281	children := append(fs.Files(nil), source.Files...)
282	source.m.RUnlock()
283
284	for _, child := range children {
285		snapshot.Files = append(snapshot.Files, snapshotItem(child, snapshot))
286	}
287	return snapshot
288}
289
290func snapshotItem(source, parent fs.Item) fs.Item {
291	switch item := source.(type) {
292	case *ZipDir:
293		snapshot := &ZipDir{Dir: snapshotDir(item.Dir, parent), zipPath: item.zipPath}
294		reparentSnapshotChildren(snapshot.Dir, snapshot)
295		return snapshot
296	case *TarDir:
297		snapshot := &TarDir{Dir: snapshotDir(item.Dir, parent), tarPath: item.tarPath}
298		reparentSnapshotChildren(snapshot.Dir, snapshot)
299		return snapshot
300	case *ZipFile:
301		return &ZipFile{
302			File:      snapshotFile(item.File, parent),
303			zipPath:   item.zipPath,
304			inZipPath: item.inZipPath,
305		}
306	case *TarFile:
307		return &TarFile{
308			File:      snapshotFile(item.File, parent),
309			tarPath:   item.tarPath,
310			inTarPath: item.inTarPath,
311		}
312	case *Dir:
313		return snapshotDir(item, parent)
314	case *File:
315		return snapshotFile(item, parent)
316	}
317
318	if source.IsDir() {
319		snapshot := &Dir{
320			File:      snapshotFile(source, parent),
321			Files:     make(fs.Files, 0),
322			ItemCount: source.GetItemCount(),
323		}
324		for child := range source.GetFilesLocked(fs.SortByName, fs.SortAsc) {
325			snapshot.Files = append(snapshot.Files, snapshotItem(child, snapshot))
326		}
327		return snapshot
328	}
329	return snapshotFile(source, parent)
330}
331
332func reparentSnapshotChildren(dir *Dir, parent fs.Item) {
333	for _, child := range dir.Files {
334		child.SetParent(parent)
335	}
336}
337
338func snapshotFile(source, parent fs.Item) *File {
339	return &File{
340		Mtime:  source.GetMtime(),
341		Parent: parent,
342		Name:   source.GetName(),
343		Size:   source.GetSize(),
344		Usage:  source.GetUsage(),
345		Mli:    source.GetMultiLinkedInode(),
346		Flag:   source.GetFlag(),
347	}
348}
349
350// AddFile add item to files
351func (f *Dir) AddFile(item fs.Item) {
352	f.m.Lock()
353	defer f.m.Unlock()
354	f.Files = append(f.Files, item)
355}
356
357// SetFlag updates the directory flag while preserving snapshot consistency.
358func (f *Dir) SetFlag(flag rune) {
359	f.m.Lock()
360	f.Flag = flag
361	f.m.Unlock()
362}
363
364// SetStatsFromJSON marks directory statistics decoded from an export as authoritative.
365func (f *Dir) SetStatsFromJSON() {
366	f.statsFromJSON = true
367}
368
369// HasEmptyDirPlaceholderStats reports whether this directory's stats are
370// indistinguishable from the placeholder updateStats produces for a directory
371// with no counted children (see EmptyDirSize). Such stats carry no information
372// worth pinning from an import: recomputation reproduces exactly the same
373// values, so callers should let updateStats run rather than preserving them.
374//
375// The items == 1 check is load-bearing: a depth-truncated directory that held
376// real content always exports with ItemCount >= 2, so it can never be mistaken
377// for a placeholder and its stats are always preserved.
378func (f *Dir) HasEmptyDirPlaceholderStats() bool {
379	return f.Size == EmptyDirSize && f.Usage == 0 && f.ItemCount == 1
380}
381
382// GetFiles returns all files in directory as a sorted iterator
383func (f *Dir) GetFiles(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
384	return func(yield func(fs.Item) bool) {
385		// Make a copy to avoid modifying the original slice
386		sorted := make(fs.Files, len(f.Files))
387		copy(sorted, f.Files)
388		sortFiles(sorted, sortBy, order)
389
390		for _, item := range sorted {
391			if !yield(item) {
392				return
393			}
394		}
395	}
396}
397
398// GetFilesLocked returns all files in directory as a sorted iterator
399// It is safe to call this function from multiple goroutines
400func (f *Dir) GetFilesLocked(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
401	return func(yield func(fs.Item) bool) {
402		f.m.RLock()
403		defer f.m.RUnlock()
404
405		// Make a copy to avoid modifying the original slice
406		sorted := make(fs.Files, len(f.Files))
407		copy(sorted, f.Files)
408		sortFiles(sorted, sortBy, order)
409
410		for _, item := range sorted {
411			if !yield(item) {
412				return
413			}
414		}
415	}
416}
417
418// GetType returns name type of item
419func (f *Dir) GetType() string {
420	return "Directory"
421}
422
423// GetFlag returns the current directory flag.
424func (f *Dir) GetFlag() rune {
425	f.m.RLock()
426	defer f.m.RUnlock()
427	return f.Flag
428}
429
430// GetSize returns the current apparent size.
431func (f *Dir) GetSize() int64 {
432	f.m.RLock()
433	defer f.m.RUnlock()
434	return f.Size
435}
436
437// GetUsage returns the current disk usage.
438func (f *Dir) GetUsage() int64 {
439	f.m.RLock()
440	defer f.m.RUnlock()
441	return f.Usage
442}
443
444// GetMtime returns the current modification time.
445func (f *Dir) GetMtime() time.Time {
446	f.m.RLock()
447	defer f.m.RUnlock()
448	return f.Mtime
449}
450
451// GetItemCount returns number of files in dir
452func (f *Dir) GetItemCount() int64 {
453	f.m.RLock()
454	defer f.m.RUnlock()
455	return f.ItemCount
456}
457
458// IsDir returns true for dir
459func (f *Dir) IsDir() bool {
460	return true
461}
462
463// GetPath returns absolute path of the file
464func (f *Dir) GetPath() string {
465	if f.BasePath != "" {
466		return filepath.Join(f.BasePath, f.Name)
467	}
468	if f.Parent != nil {
469		return filepath.Join(f.Parent.GetPath(), f.Name)
470	}
471	return f.Name
472}
473
474// GetItemStats returns item count, apparent usage and real usage of this dir
475func (f *Dir) GetItemStats(linkedItems fs.HardLinkedItems, filteringFiles bool) (itemCount, size, usage int64) {
476	f.updateStats(linkedItems, filteringFiles)
477	return f.GetItemCount(), f.GetSize(), f.GetUsage()
478}
479
480func (f *Dir) UpdateStats(linkedItems fs.HardLinkedItems) {
481	f.updateStats(linkedItems, false)
482}
483
484func (f *Dir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {
485	f.updateStats(linkedItems, true)
486}
487
488// UpdateStats recursively updates size and item count
489func (f *Dir) updateStats(linkedItems fs.HardLinkedItems, filteringFiles bool) {
490	if f.statsFromJSON {
491		return
492	}
493
494	// Snapshot the file list under the read lock so it is safe to compute stats
495	// even while the analyzer is still appending items in another goroutine
496	// (e.g. when previewing a directory mid-scan).
497	f.m.RLock()
498	files := make(fs.Files, len(f.Files))
499	copy(files, f.Files)
500	mtime := f.Mtime
501	flag := f.Flag
502	f.m.RUnlock()
503
504	totals, mtime, flag := aggregateDirEntries(files, mtime, flag, linkedItems, filteringFiles)
505
506	f.m.Lock()
507	defer f.m.Unlock()
508	f.Mtime = mtime
509	f.Flag = flag
510	f.ItemCount, f.Size, f.Usage = resolveDirStats(totals, filteringFiles)
511}
512
513// RemoveFile removes item from dir, updates size and item count
514func (f *Dir) RemoveFile(item fs.Item) {
515	f.m.Lock()
516	f.Files = f.Files.Remove(item)
517	f.m.Unlock()
518
519	f.subtractStats(item)
520}
521
522// subtractStats removes item's totals from this dir and every ancestor. Each
523// directory is updated under its own lock, because holding only this dir's
524// mutex would leave the ancestors' writes unsynchronized against readers
525// calling GetSize, GetUsage or GetItemCount on them.
526func (f *Dir) subtractStats(item fs.Item) {
527	itemCount, size, usage := item.GetItemCount(), item.GetSize(), item.GetUsage()
528
529	cur := f
530	for {
531		cur.m.Lock()
532		cur.ItemCount -= itemCount
533		cur.Size -= size
534		cur.Usage -= usage
535		parent := cur.Parent
536		cur.m.Unlock()
537
538		if parent == nil {
539			break
540		}
541		cur = parent.(*Dir)
542	}
543}
544
545// sortFiles sorts files in place according to sortBy and order
546func sortFiles(files fs.Files, sortBy fs.SortBy, order fs.SortOrder) {
547	var sorter sort.Interface
548	switch sortBy {
549	case fs.SortByName:
550		sorter = fs.ByName(files)
551	case fs.SortByItemCount:
552		sorter = fs.ByItemCount(files)
553	case fs.SortByMtime:
554		sorter = fs.ByMtime(files)
555	case fs.SortByApparentSize:
556		sorter = fs.ByApparentSize(files)
557	case fs.SortBySize:
558		sorter = files
559	}
560
561	if order == fs.SortDesc {
562		sort.Sort(sort.Reverse(sorter))
563	} else {
564		sort.Sort(sorter)
565	}
566}
567
568// RLock read locks dir
569func (f *Dir) RLock() func() {
570	f.m.RLock()
571	return f.m.RUnlock
572}
573
574// RemoveFileByName removes item by name from dir
575func (f *Dir) RemoveFileByName(name string) {
576	f.m.Lock()
577	idx, ok := f.Files.FindByName(name)
578	if !ok {
579		f.m.Unlock()
580		return
581	}
582	item := f.Files[idx]
583	f.Files = append(f.Files[:idx], f.Files[idx+1:]...)
584	f.m.Unlock()
585
586	f.subtractStats(item)
587}