fork/gdu

git clone https://git.julia.blue/fork/gdu

Daniel Mildefix: StoreDir fixes (#644)35cba00

master
13.2 KiB485 linesraw
1package analyze
2
3import (
4	"io"
5	"iter"
6	"os"
7	"path/filepath"
8	"sync"
9	"time"
10
11	"github.com/dundee/gdu/v5/internal/common"
12	"github.com/dundee/gdu/v5/pkg/fs"
13	log "github.com/sirupsen/logrus"
14)
15
16// StoredAnalyzer implements Analyzer
17type StoredAnalyzer struct {
18	BaseAnalyzer
19	storage     *Storage
20	storagePath string
21}
22
23// CreateStoredAnalyzer returns Analyzer
24func CreateStoredAnalyzer(storagePath string) *StoredAnalyzer {
25	a := &StoredAnalyzer{
26		storagePath: storagePath,
27	}
28	a.Init()
29	return a
30}
31
32// AnalyzeDir analyzes given path
33func (a *StoredAnalyzer) AnalyzeDir(
34	path string, ignore common.ShouldDirBeIgnored, fileTypeFilter common.ShouldFileBeIgnored,
35) fs.Item {
36	a.ignoreDir = ignore
37	a.ignoreFileType = fileTypeFilter
38
39	a.storage = NewStorage(a.storagePath, path)
40	closeFn := a.storage.Open()
41	defer func() {
42		// nasty hack to close storage after all goroutines are done
43		// Wait returns immediately if value is 0
44		// few last goroutines might still start after that
45		time.Sleep(1 * time.Second)
46		closeFn()
47	}()
48
49	a.ignoreDir = ignore
50
51	go a.UpdateProgress()
52	dir := a.processDir(path)
53
54	a.wait.Wait()
55
56	a.progressDoneChan <- struct{}{}
57	a.doneChan.Broadcast()
58
59	return dir
60}
61
62func (a *StoredAnalyzer) processDir(path string) *StoredDir {
63	var (
64		file       fs.Item
65		err        error
66		totalUsage int64
67		info       os.FileInfo
68		dirCount   int
69	)
70
71	a.wait.Add(1)
72
73	files, err := os.ReadDir(path)
74	if err != nil {
75		log.Print(err.Error())
76	}
77
78	dir := &StoredDir{
79		Dir: &Dir{
80			File: &File{
81				Name: filepath.Base(path),
82				Flag: getDirFlag(err, len(files)),
83			},
84			BasePath:  filepath.Dir(path),
85			ItemCount: 1,
86			Files:     make(fs.Files, 0, len(files)),
87		},
88	}
89	parent := &ParentDir{Path: path}
90
91	setDirPlatformSpecificAttrs(dir.Dir, path)
92
93	for _, f := range files {
94		if a.IsCancelled() {
95			break
96		}
97		name := f.Name()
98		entryPath := filepath.Join(path, name)
99		if f.IsDir() {
100			if a.shouldSkipDir(name, entryPath) {
101				continue
102			}
103			dirCount++
104
105			subdir := &StoredDir{
106				Dir: &Dir{
107					File: &File{
108						Name: name,
109					},
110					BasePath: path,
111				},
112			}
113			dir.AddFile(subdir)
114
115			go func(entryPath string) {
116				concurrencyLimit <- struct{}{}
117				a.processDir(entryPath)
118				<-concurrencyLimit
119			}(entryPath)
120		} else {
121			// Apply file type filter if set
122			if a.ignoreFileType != nil && a.ignoreFileType(name) {
123				continue // Skip this file
124			}
125
126			info, err = f.Info()
127			if err != nil {
128				log.Print(err.Error())
129				continue
130			}
131
132			if a.followSymlinks && info.Mode()&os.ModeSymlink != 0 {
133				infoF, err := followSymlink(entryPath, a.gitAnnexedSize)
134				if err != nil {
135					log.Print(err.Error())
136					continue
137				}
138				if infoF != nil {
139					info = infoF
140				}
141			}
142
143			symlinkTarget := readSymlinkTarget(f.Type(), entryPath)
144
145			// Apply time filter if set
146			if a.matchesTimeFilterFn != nil && !a.matchesTimeFilterFn(info.ModTime()) {
147				continue // Skip this file
148			}
149
150			// Check if it's a zip or jar file
151			if a.archiveBrowsing && isZipFile(name) {
152				zipDir, err := processZipFile(entryPath, info)
153				if err != nil {
154					// If unable to process zip file, treat as regular file
155					log.Printf("Failed to process zip file %s: %v", entryPath, err)
156					file = &File{
157						Name:   name,
158						Flag:   getFlag(info),
159						Size:   info.Size(),
160						Parent: parent,
161					}
162				} else {
163					// Successfully processed zip file, use zip content size
164					uncompressedSize, compressedSize, err := getZipFileSize(entryPath)
165					if err == nil {
166						zipDir.Size = uncompressedSize
167						zipDir.Usage = compressedSize
168					}
169					zipDir.Parent = parent
170					file = zipDir
171				}
172			} else {
173				file = &File{
174					Name:    name,
175					Flag:    getFlag(info),
176					Size:    info.Size(),
177					Parent:  parent,
178					Symlink: symlinkTarget,
179				}
180			}
181
182			if file != nil {
183				// Only set platform-specific attributes for regular files
184				if regularFile, ok := file.(*File); ok {
185					setPlatformSpecificAttrs(regularFile, info)
186				}
187				totalUsage += file.GetUsage()
188				dir.AddFile(file)
189			}
190		}
191	}
192
193	err = a.storage.StoreDir(dir)
194	if err != nil {
195		log.Print(err.Error())
196	}
197
198	a.wait.Done()
199
200	a.progressCurrentItemName.Store(path)
201	a.progressItemCount.Add(int64(len(files)))
202	a.progressTotalUsage.Add(totalUsage)
203
204	return dir
205}
206
207// StoredDir implements Dir item stored on disk
208//
209// The stat fields inherited from Dir follow Dir's locking protocol: they are
210// written under Dir.m, so readers calling the synchronized accessors (GetSize,
211// GetUsage, GetFlag, GetMtime, GetItemCount) never observe a partially applied
212// stats update.
213//
214// cachedFiles gets its own leaf mutex rather than sharing Dir.m, because
215// loadFiles runs underneath GetFiles, and GetFiles is the caller-locked half
216// of the fs.Item contract: callers such as the TUI hold RLock across the whole
217// iteration (see tui.(*UI).showDir). Taking Dir.m in loadFiles would therefore
218// re-enter a lock the caller already holds, which sync.RWMutex does not allow.
219// cacheLock is never held while Dir.m is held, so the two cannot deadlock.
220type StoredDir struct {
221	*Dir
222	cachedFiles fs.Files
223	cacheLock   sync.Mutex
224}
225
226// GetParent returns parent dir
227func (f *StoredDir) GetParent() fs.Item {
228	if DefaultStorage.GetTopDir() == f.GetPath() {
229		return nil
230	}
231
232	// Storage.Open is reference counted, so taking a reference unconditionally
233	// is both cheap and free of the check-then-open race that an IsOpen guard
234	// has: another goroutine could close the database between the two.
235	closeFn := DefaultStorage.Open()
236	defer closeFn()
237
238	dir, err := DefaultStorage.GetDirForPath(f.BasePath)
239	if err != nil {
240		log.Print(err.Error())
241	}
242	return dir
243}
244
245// GetFiles returns files in directory as a sorted iterator
246// If files are already cached, use them
247// Otherwise load them from storage
248func (f *StoredDir) GetFiles(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
249	return func(yield func(fs.Item) bool) {
250		files := f.loadFiles()
251		sortFiles(files, sortBy, order)
252
253		for _, item := range files {
254			if !yield(item) {
255				return
256			}
257		}
258	}
259}
260
261// loadFiles loads files from storage or returns cached files
262//
263// Like Dir.GetFiles, this reads f.Files without taking Dir.m: it is reached
264// through GetFiles, whose callers are the ones holding the read lock.
265func (f *StoredDir) loadFiles() fs.Files {
266	return f.materializeFiles(f.Files)
267}
268
269// loadFilesLocked is the self-locking counterpart of loadFiles, mirroring the
270// GetFiles / GetFilesLocked split on Dir. Callers that do not already hold the
271// read lock use this so the entry list cannot change underneath them.
272func (f *StoredDir) loadFilesLocked() fs.Files {
273	f.m.RLock()
274	entries := copyFiles(f.Files)
275	f.m.RUnlock()
276
277	return f.materializeFiles(entries)
278}
279
280// materializeFiles turns raw stored entries into usable items, reading child
281// directories back out of storage, and memoizes the result in cachedFiles.
282// Only cachedFiles is synchronized here, under its own leaf mutex.
283func (f *StoredDir) materializeFiles(entries fs.Files) fs.Files {
284	closeFn := DefaultStorage.Open()
285	defer closeFn()
286
287	f.cacheLock.Lock()
288	defer f.cacheLock.Unlock()
289
290	if cached := copyFiles(f.cachedFiles); cached != nil {
291		return cached
292	}
293
294	var files fs.Files
295	for _, file := range entries {
296		if file.IsDir() {
297			dir := &StoredDir{
298				Dir: &Dir{
299					File: &File{
300						Name: file.GetName(),
301					},
302					BasePath: f.GetPath(),
303				},
304			}
305
306			err := DefaultStorage.LoadDir(dir)
307			if err != nil {
308				log.Print(err.Error())
309			}
310			files = append(files, dir)
311		} else {
312			files = append(files, file)
313		}
314	}
315
316	f.cachedFiles = files
317	return copyFiles(files)
318}
319
320// copyFiles returns a copy of files so callers cannot mutate the cached slice.
321// It returns nil for a nil input, letting callers use the result to distinguish
322// "not cached" from "cached but empty".
323func copyFiles(files fs.Files) fs.Files {
324	if files == nil {
325		return nil
326	}
327	result := make(fs.Files, len(files))
328	copy(result, files)
329	return result
330}
331
332// RemoveFile removes file from stored directory
333// It also updates size and item count of parent directories
334func (f *StoredDir) RemoveFile(item fs.Item) {
335	closeFn := DefaultStorage.Open()
336	defer closeFn()
337
338	f.m.Lock()
339	f.Files = f.Files.Remove(item)
340	f.m.Unlock()
341	f.invalidateCache()
342
343	f.subtractStats(item)
344}
345
346// invalidateCache drops the loaded children so the next loadFiles re-reads
347// them from storage.
348func (f *StoredDir) invalidateCache() {
349	f.cacheLock.Lock()
350	f.cachedFiles = nil
351	f.cacheLock.Unlock()
352}
353
354// subtractStats removes item's totals from this dir and every ancestor,
355// persisting each one. Every directory is updated and stored under its own
356// write lock so readers never see the stats and the stored copy disagree.
357func (f *StoredDir) subtractStats(item fs.Item) {
358	itemCount, size, usage := item.GetItemCount(), item.GetSize(), item.GetUsage()
359
360	cur := f
361	for {
362		cur.m.Lock()
363		cur.ItemCount -= itemCount
364		cur.Size -= size
365		cur.Usage -= usage
366		err := DefaultStorage.StoreDir(cur)
367		cur.m.Unlock()
368
369		if err != nil {
370			log.Print(err.Error())
371		}
372
373		parent := cur.GetParent()
374		if parent == nil {
375			break
376		}
377		cur = parent.(*StoredDir)
378	}
379}
380
381// GetItemStats returns item count, apparent usage and real usage of this dir
382func (f *StoredDir) GetItemStats(linkedItems fs.HardLinkedItems, filteringFiles bool) (itemCount, size, usage int64) {
383	f.updateStats(linkedItems, filteringFiles)
384	return f.GetItemCount(), f.GetSize(), f.GetUsage()
385}
386
387func (f *StoredDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {
388	f.updateStats(linkedItems, true)
389}
390
391// UpdateStats recursively updates size and item count
392func (f *StoredDir) UpdateStats(linkedItems fs.HardLinkedItems) {
393	f.updateStats(linkedItems, false)
394}
395
396func (f *StoredDir) updateStats(linkedItems fs.HardLinkedItems, filteringFiles bool) {
397	closeFn := DefaultStorage.Open()
398	defer closeFn()
399
400	// Drop the cache and snapshot the fields we accumulate into, so the stats
401	// are computed from locals while no lock is held. Recursing into
402	// entry.GetItemStats below must not happen under f.m.
403	f.invalidateCache()
404
405	f.m.RLock()
406	curMtime := f.Mtime
407	curFlag := f.Flag
408	f.m.RUnlock()
409
410	files := f.loadFilesLocked()
411	totals, mtime, flag := aggregateDirEntries(files, curMtime, curFlag, linkedItems, filteringFiles)
412
413	f.invalidateCache()
414
415	// Commit and persist under one write lock so neither a concurrent reader
416	// nor the gob encoder can observe a half-updated directory.
417	f.m.Lock()
418	defer f.m.Unlock()
419
420	f.Mtime = mtime
421	f.Flag = flag
422	f.ItemCount, f.Size, f.Usage = resolveDirStats(totals, filteringFiles)
423
424	if err := DefaultStorage.StoreDir(f); err != nil {
425		log.Print(err.Error())
426	}
427}
428
429// RemoveFileByName removes file by name from stored directory
430func (f *StoredDir) RemoveFileByName(name string) {
431	closeFn := DefaultStorage.Open()
432	defer closeFn()
433
434	f.m.Lock()
435	idx, ok := f.Files.FindByName(name)
436	if !ok {
437		f.m.Unlock()
438		return
439	}
440	item := f.Files[idx]
441	f.Files = append(f.Files[:idx], f.Files[idx+1:]...)
442	f.m.Unlock()
443	f.invalidateCache()
444
445	f.subtractStats(item)
446}
447
448// ParentDir represents parent directory of single file
449// It is used to get path to parent directory of a file
450type ParentDir struct {
451	Path string
452}
453
454func (p *ParentDir) GetPath() string {
455	return p.Path
456}
457func (p *ParentDir) GetName() string                                     { panic("must not be called") }
458func (p *ParentDir) GetFlag() rune                                       { panic("must not be called") }
459func (p *ParentDir) IsDir() bool                                         { panic("must not be called") }
460func (p *ParentDir) GetSize() int64                                      { panic("must not be called") }
461func (p *ParentDir) GetType() string                                     { panic("must not be called") }
462func (p *ParentDir) GetUsage() int64                                     { panic("must not be called") }
463func (p *ParentDir) GetMtime() time.Time                                 { panic("must not be called") }
464func (p *ParentDir) GetItemCount() int64                                 { panic("must not be called") }
465func (p *ParentDir) GetParent() fs.Item                                  { panic("must not be called") }
466func (p *ParentDir) SetParent(fs.Item)                                   { panic("must not be called") }
467func (p *ParentDir) GetMultiLinkedInode() uint64                         { panic("must not be called") }
468func (p *ParentDir) EncodeJSON(io.Writer, bool, fs.JSONAttributes) error { panic("must not be called") }
469func (p *ParentDir) UpdateStats(linkedItems fs.HardLinkedItems)          { panic("must not be called") }
470func (p *ParentDir) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {
471	panic("must not be called")
472}
473func (p *ParentDir) AddFile(fs.Item)                                    { panic("must not be called") }
474func (p *ParentDir) GetFiles(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] { panic("must not be called") }
475func (p *ParentDir) GetFilesLocked(fs.SortBy, fs.SortOrder) iter.Seq[fs.Item] {
476	panic("must not be called")
477}
478func (p *ParentDir) RLock() func()                { panic("must not be called") }
479func (p *ParentDir) RemoveFile(item fs.Item)      { panic("must not be called") }
480func (p *ParentDir) RemoveFileByName(name string) { panic("must not be called") }
481func (p *ParentDir) GetItemStats(
482	linkedItems fs.HardLinkedItems, filteringFiles bool,
483) (itemCount, size, usage int64) {
484	panic("must not be called")
485}