fork/gdu
git clone https://git.julia.blue/fork/gdu
35cba00
master
1package analyze 2 3import ( 4"io" 5"iter" 6"os" 7"path/filepath" 8"sync" 9"time" 10 11"github.com/dundee/gdu/v5/internal/common" 12"github.com/dundee/gdu/v5/pkg/fs" 13 log"github.com/sirupsen/logrus" 14) 15 16// StoredAnalyzer implements Analyzer 17type StoredAnalyzer struct { 18BaseAnalyzer 19storage * Storage 20storagePath string 21} 22 23// CreateStoredAnalyzer returns Analyzer 24func CreateStoredAnalyzer (storagePath string )* StoredAnalyzer { 25a := & StoredAnalyzer { 26storagePath :storagePath , 27 } 28a .Init () 29return a 30} 31 32// AnalyzeDir analyzes given path 33func (a * StoredAnalyzer )AnalyzeDir ( 34path string ,ignore common.ShouldDirBeIgnored ,fileTypeFilter common.ShouldFileBeIgnored , 35) fs.Item { 36a .ignoreDir = ignore 37a .ignoreFileType = fileTypeFilter 38 39a .storage = NewStorage (a .storagePath ,path ) 40closeFn := a .storage .Open () 41defer func () { 42// nasty hack to close storage after all goroutines are done 43// Wait returns immediately if value is 0 44// few last goroutines might still start after that 45time .Sleep (1 * time .Second ) 46closeFn () 47 }() 48 49a .ignoreDir = ignore 50 51go a .UpdateProgress () 52dir := a .processDir (path ) 53 54a .wait .Wait () 55 56a .progressDoneChan <- struct {}{} 57a .doneChan .Broadcast () 58 59return dir 60} 61 62func (a * StoredAnalyzer )processDir (path string )* StoredDir { 63var ( 64file fs.Item 65err error 66totalUsage int64 67info os.FileInfo 68dirCount int 69 ) 70 71a .wait .Add (1 ) 72 73files ,err := os .ReadDir (path ) 74if err != nil { 75log .err .Error ()) 76 } 77 78dir := & StoredDir { 79Dir :& Dir { 80File :& File { 81Name :filepath .Base (path ), 82Flag :getDirFlag (err ,len (files )), 83 }, 84BasePath :filepath .Dir (path ), 85ItemCount :1 , 86Files :make (fs.Files ,0 ,len (files )), 87 }, 88 } 89parent := & ParentDir {Path :path } 90 91setDirPlatformSpecificAttrs (dir .Dir ,path ) 92 93for _ ,f := range files { 94if a .IsCancelled () { 95break 96 } 97name := f .Name () 98entryPath := filepath .Join (path ,name ) 99if f .IsDir () { 100if a .shouldSkipDir (name ,entryPath ) { 101continue 102 } 103dirCount ++ 104 105subdir := & StoredDir { 106Dir :& Dir { 107File :& File { 108Name :name , 109 }, 110BasePath :path , 111 }, 112 } 113dir .AddFile (subdir ) 114 115go func (entryPath string ) { 116concurrencyLimit <- struct {}{} 117a .processDir (entryPath ) 118<- concurrencyLimit 119 }(entryPath ) 120 }else { 121// Apply file type filter if set 122if a .ignoreFileType != nil && a .ignoreFileType (name ) { 123continue // Skip this file 124 } 125 126info ,err = f .Info () 127if err != nil { 128log .err .Error ()) 129continue 130 } 131 132if a .followSymlinks && info .Mode ()& os .ModeSymlink != 0 { 133infoF ,err := followSymlink (entryPath ,a .gitAnnexedSize ) 134if err != nil { 135log .err .Error ()) 136continue 137 } 138if infoF != nil { 139info = infoF 140 } 141 } 142 143symlinkTarget := readSymlinkTarget (f .Type (),entryPath ) 144 145// Apply time filter if set 146if a .matchesTimeFilterFn != nil && ! a .matchesTimeFilterFn (info .ModTime ()) { 147continue // Skip this file 148 } 149 150// Check if it's a zip or jar file 151if a .archiveBrowsing && isZipFile (name ) { 152zipDir ,err := processZipFile (entryPath ,info ) 153if err != nil { 154// If unable to process zip file, treat as regular file 155log .Printf ("Failed to process zip file %s: %v" ,entryPath ,err ) 156file = & File { 157Name :name , 158Flag :getFlag (info ), 159Size :info .Size (), 160Parent :parent , 161 } 162 }else { 163// Successfully processed zip file, use zip content size 164uncompressedSize ,compressedSize ,err := getZipFileSize (entryPath ) 165if err == nil { 166zipDir .Size = uncompressedSize 167zipDir .Usage = compressedSize 168 } 169zipDir .Parent = parent 170file = zipDir 171 } 172 }else { 173file = & File { 174Name :name , 175Flag :getFlag (info ), 176Size :info .Size (), 177Parent :parent , 178Symlink :symlinkTarget , 179 } 180 } 181 182if file != nil { 183// Only set platform-specific attributes for regular files 184if regularFile ,ok := file .(* File );ok { 185setPlatformSpecificAttrs (regularFile ,info ) 186 } 187totalUsage += file .GetUsage () 188dir .AddFile (file ) 189 } 190 } 191 } 192 193err = a .storage .StoreDir (dir ) 194if err != nil { 195log .err .Error ()) 196 } 197 198a .wait .Done () 199 200a .progressCurrentItemName .Store (path ) 201a .progressItemCount .Add (int64 (len (files ))) 202a .progressTotalUsage .Add (totalUsage ) 203 204return dir 205} 206 207// StoredDir implements Dir item stored on disk 208// 209// The stat fields inherited from Dir follow Dir's locking protocol: they are 210// written under Dir.m, so readers calling the synchronized accessors (GetSize, 211// GetUsage, GetFlag, GetMtime, GetItemCount) never observe a partially applied 212// stats update. 213// 214// cachedFiles gets its own leaf mutex rather than sharing Dir.m, because 215// loadFiles runs underneath GetFiles, and GetFiles is the caller-locked half 216// of the fs.Item contract: callers such as the TUI hold RLock across the whole 217// iteration (see tui.(*UI).showDir). Taking Dir.m in loadFiles would therefore 218// re-enter a lock the caller already holds, which sync.RWMutex does not allow. 219// cacheLock is never held while Dir.m is held, so the two cannot deadlock. 220type StoredDir struct { 221* Dir 222cachedFiles fs.Files 223cacheLock sync.Mutex 224} 225 226// GetParent returns parent dir 227func (f * StoredDir )GetParent () fs.Item { 228if DefaultStorage .GetTopDir ()== f .GetPath () { 229return nil 230 } 231 232// Storage.Open is reference counted, so taking a reference unconditionally 233// is both cheap and free of the check-then-open race that an IsOpen guard 234// has: another goroutine could close the database between the two. 235closeFn := DefaultStorage .Open () 236defer closeFn () 237 238dir ,err := DefaultStorage .GetDirForPath (f .BasePath ) 239if err != nil { 240log .err .Error ()) 241 } 242return dir 243} 244 245// GetFiles returns files in directory as a sorted iterator 246// If files are already cached, use them 247// Otherwise load them from storage 248func (f * StoredDir )GetFiles (sortBy fs.SortBy ,order fs.SortOrder ) iter.Seq [fs.Item ] { 249return func (yield func (fs.Item )bool ) { 250files := f .loadFiles () 251sortFiles (files ,sortBy ,order ) 252 253for _ ,item := range files { 254if ! yield (item ) { 255return 256 } 257 } 258 } 259} 260 261// loadFiles loads files from storage or returns cached files 262// 263// Like Dir.GetFiles, this reads f.Files without taking Dir.m: it is reached 264// through GetFiles, whose callers are the ones holding the read lock. 265func (f * StoredDir )loadFiles () fs.Files { 266return f .materializeFiles (f .Files ) 267} 268 269// loadFilesLocked is the self-locking counterpart of loadFiles, mirroring the 270// GetFiles / GetFilesLocked split on Dir. Callers that do not already hold the 271// read lock use this so the entry list cannot change underneath them. 272func (f * StoredDir )loadFilesLocked () fs.Files { 273f .m .RLock () 274entries := copyFiles (f .Files ) 275f .m .RUnlock () 276 277return f .materializeFiles (entries ) 278} 279 280// materializeFiles turns raw stored entries into usable items, reading child 281// directories back out of storage, and memoizes the result in cachedFiles. 282// Only cachedFiles is synchronized here, under its own leaf mutex. 283func (f * StoredDir )materializeFiles (entries fs.Files ) fs.Files { 284closeFn := DefaultStorage .Open () 285defer closeFn () 286 287f .cacheLock .Lock () 288defer f .cacheLock .Unlock () 289 290if cached := copyFiles (f .cachedFiles );cached != nil { 291return cached 292 } 293 294var files fs.Files 295for _ ,file := range entries { 296if file .IsDir () { 297dir := & StoredDir { 298Dir :& Dir { 299File :& File { 300Name :file .GetName (), 301 }, 302BasePath :f .GetPath (), 303 }, 304 } 305 306err := DefaultStorage .LoadDir (dir ) 307if err != nil { 308log .err .Error ()) 309 } 310files = append (files ,dir ) 311 }else { 312files = append (files ,file ) 313 } 314 } 315 316f .cachedFiles = files 317return copyFiles (files ) 318} 319 320// copyFiles returns a copy of files so callers cannot mutate the cached slice. 321// It returns nil for a nil input, letting callers use the result to distinguish 322// "not cached" from "cached but empty". 323func copyFiles (files fs.Files ) fs.Files { 324if files == nil { 325return nil 326 } 327result := make (fs.Files ,len (files )) 328copy (result ,files ) 329return result 330} 331 332// RemoveFile removes file from stored directory 333// It also updates size and item count of parent directories 334func (f * StoredDir )RemoveFile (item fs.Item ) { 335closeFn := DefaultStorage .Open () 336defer closeFn () 337 338f .m .Lock () 339f .Files = f .Files .Remove (item ) 340f .m .Unlock () 341f .invalidateCache () 342 343f .subtractStats (item ) 344} 345 346// invalidateCache drops the loaded children so the next loadFiles re-reads 347// them from storage. 348func (f * StoredDir )invalidateCache () { 349f .cacheLock .Lock () 350f .cachedFiles = nil 351f .cacheLock .Unlock () 352} 353 354// subtractStats removes item's totals from this dir and every ancestor, 355// persisting each one. Every directory is updated and stored under its own 356// write lock so readers never see the stats and the stored copy disagree. 357func (f * StoredDir )subtractStats (item fs.Item ) { 358itemCount ,size ,usage := item .GetItemCount (),item .GetSize (),item .GetUsage () 359 360cur := f 361for { 362cur .m .Lock () 363cur .ItemCount -= itemCount 364cur .Size -= size 365cur .Usage -= usage 366err := DefaultStorage .StoreDir (cur ) 367cur .m .Unlock () 368 369if err != nil { 370log .err .Error ()) 371 } 372 373parent := cur .GetParent () 374if parent == nil { 375break 376 } 377cur = parent .(* StoredDir ) 378 } 379} 380 381// GetItemStats returns item count, apparent usage and real usage of this dir 382func (f * StoredDir )GetItemStats (linkedItems fs.HardLinkedItems ,filteringFiles bool ) (itemCount ,size ,usage int64 ) { 383f .updateStats (linkedItems ,filteringFiles ) 384return f .GetItemCount (),f .GetSize (),f .GetUsage () 385} 386 387func (f * StoredDir )UpdateStatsWithFileFiltering (linkedItems fs.HardLinkedItems ) { 388f .updateStats (linkedItems ,true ) 389} 390 391// UpdateStats recursively updates size and item count 392func (f * StoredDir )UpdateStats (linkedItems fs.HardLinkedItems ) { 393f .updateStats (linkedItems ,false ) 394} 395 396func (f * StoredDir )updateStats (linkedItems fs.HardLinkedItems ,filteringFiles bool ) { 397closeFn := DefaultStorage .Open () 398defer closeFn () 399 400// Drop the cache and snapshot the fields we accumulate into, so the stats 401// are computed from locals while no lock is held. Recursing into 402// entry.GetItemStats below must not happen under f.m. 403f .invalidateCache () 404 405f .m .RLock () 406curMtime := f .Mtime 407curFlag := f .Flag 408f .m .RUnlock () 409 410files := f .loadFilesLocked () 411totals ,mtime ,flag := aggregateDirEntries (files ,curMtime ,curFlag ,linkedItems ,filteringFiles ) 412 413f .invalidateCache () 414 415// Commit and persist under one write lock so neither a concurrent reader 416// nor the gob encoder can observe a half-updated directory. 417f .m .Lock () 418defer f .m .Unlock () 419 420f .Mtime = mtime 421f .Flag = flag 422f .ItemCount ,f .Size ,f .Usage = resolveDirStats (totals ,filteringFiles ) 423 424if err := DefaultStorage .StoreDir (f );err != nil { 425log .err .Error ()) 426 } 427} 428 429// RemoveFileByName removes file by name from stored directory 430func (f * StoredDir )RemoveFileByName (name string ) { 431closeFn := DefaultStorage .Open () 432defer closeFn () 433 434f .m .Lock () 435idx ,ok := f .Files .FindByName (name ) 436if ! ok { 437f .m .Unlock () 438return 439 } 440item := f .Files [idx ] 441f .Files = append (f .Files [:idx ],f .Files [idx + 1 :]... ) 442f .m .Unlock () 443f .invalidateCache () 444 445f .subtractStats (item ) 446} 447 448// ParentDir represents parent directory of single file 449// It is used to get path to parent directory of a file 450type ParentDir struct { 451Path string 452} 453 454func (p * ParentDir )GetPath ()string { 455return p .Path 456} 457func (p * ParentDir )GetName ()string {panic ("must not be called" ) } 458func (p * ParentDir )GetFlag ()rune {panic ("must not be called" ) } 459func (p * ParentDir )IsDir ()bool {panic ("must not be called" ) } 460func (p * ParentDir )GetSize ()int64 {panic ("must not be called" ) } 461func (p * ParentDir )GetType ()string {panic ("must not be called" ) } 462func (p * ParentDir )GetUsage ()int64 {panic ("must not be called" ) } 463func (p * ParentDir )GetMtime () time.Time {panic ("must not be called" ) } 464func (p * ParentDir )GetItemCount ()int64 {panic ("must not be called" ) } 465func (p * ParentDir )GetParent () fs.Item {panic ("must not be called" ) } 466func (p * ParentDir )SetParent (fs.Item ) {panic ("must not be called" ) } 467func (p * ParentDir )GetMultiLinkedInode ()uint64 {panic ("must not be called" ) } 468func (p * ParentDir )EncodeJSON (io.Writer ,bool , fs.JSONAttributes )error {panic ("must not be called" ) } 469func (p * ParentDir )UpdateStats (linkedItems fs.HardLinkedItems ) {panic ("must not be called" ) } 470func (p * ParentDir )UpdateStatsWithFileFiltering (linkedItems fs.HardLinkedItems ) { 471panic ("must not be called" ) 472} 473func (p * ParentDir )AddFile (fs.Item ) {panic ("must not be called" ) } 474func (p * ParentDir )GetFiles (fs.SortBy , fs.SortOrder ) iter.Seq [fs.Item ] {panic ("must not be called" ) } 475func (p * ParentDir )GetFilesLocked (fs.SortBy , fs.SortOrder ) iter.Seq [fs.Item ] { 476panic ("must not be called" ) 477} 478func (p * ParentDir )RLock ()func () {panic ("must not be called" ) } 479func (p * ParentDir )RemoveFile (item fs.Item ) {panic ("must not be called" ) } 480func (p * ParentDir )RemoveFileByName (name string ) {panic ("must not be called" ) } 481func (p * ParentDir )GetItemStats ( 482linkedItems fs.HardLinkedItems ,filteringFiles bool , 483) (itemCount ,size ,usage int64 ) { 484panic ("must not be called" ) 485}