fork/gdu
git clone https://git.julia.blue/fork/gdu
d062e88
ignore
1package analyze 2 3import ( 4"database/sql" 5"encoding/json" 6"io" 7"iter" 8"os" 9"path/filepath" 10"strconv" 11"sync" 12"time" 13 14"github.com/dundee/gdu/v5/internal/common" 15"github.com/dundee/gdu/v5/pkg/fs" 16"github.com/pkg/errors" 17 log"github.com/sirupsen/logrus" 18) 19 20// SqliteStorage represents SQLite database storage 21type SqliteStorage struct { 22db * sql.DB 23dbPath string 24m sync.RWMutex 25tx * sql.Tx 26insertStmt * sql.Stmt 27updateStmt * sql.Stmt 28hasInodeStmt * sql.Stmt 29} 30 31// NewSqliteStorage creates a new SQLite storage and initializes the schema 32func NewSqliteStorage (dbPath string ) (* SqliteStorage ,error ) { 33parentDir := filepath .Dir (dbPath ) 34if err := os .MkdirAll (parentDir ,0o755 );err != nil { 35return nil ,errors .Wrap (err ,"failed to create parent directory for SQLite database" ) 36 } 37 38db ,err := sql .Open ("sqlite" ,dbPath ) 39if err != nil { 40return nil ,err 41 } 42 43storage := & SqliteStorage { 44db :db , 45dbPath :dbPath , 46 } 47 48if err := storage .createTables ();err != nil { 49db .Close () 50return nil ,err 51 } 52 53return storage ,nil 54} 55 56// createTables creates the database schema if it doesn't exist 57func (s * SqliteStorage )createTables ()error { 58// Optimize for insertion speed 59pragmas := ` 60PRAGMA synchronous = OFF; 61PRAGMA journal_mode = MEMORY; 62PRAGMA cache_size = -64000; 63PRAGMA temp_store = MEMORY; 64` 65if _ ,err := s .db .Exec (pragmas );err != nil { 66return err 67 } 68 69schema := ` 70CREATE TABLE IF NOT EXISTS items ( 71id INTEGER PRIMARY KEY, 72parent_id INTEGER REFERENCES items(id), 73name TEXT NOT NULL, 74is_dir INTEGER NOT NULL, 75size INTEGER NOT NULL, 76usage INTEGER NOT NULL, 77mtime INTEGER NOT NULL, 78item_count INTEGER NOT NULL DEFAULT 1, 79mli INTEGER NOT NULL DEFAULT 0, 80flag TEXT NOT NULL DEFAULT ' ' 81); 82 83CREATE INDEX IF NOT EXISTS idx_items_parent_id ON items(parent_id); 84CREATE INDEX IF NOT EXISTS idx_items_parent_name ON items(parent_id, name); 85CREATE INDEX IF NOT EXISTS idx_items_mli ON items(mli) WHERE mli != 0; 86 87CREATE TABLE IF NOT EXISTS metadata ( 88key TEXT PRIMARY KEY, 89value TEXT 90); 91` 92 93_ ,err := s .db .Exec (schema ) 94return err 95} 96 97// Close closes the database connection 98func (s * SqliteStorage )Close ()error { 99s .m .Lock () 100defer s .m .Unlock () 101if s .db != nil { 102return s .db .Close () 103 } 104return nil 105} 106 107// ClearItems removes all items from the database 108func (s * SqliteStorage )ClearItems ()error { 109_ ,err := s .db .Exec ("DELETE FROM items" ) 110return err 111} 112 113// BeginBulkInsert starts a transaction and prepares statements for bulk insertion 114func (s * SqliteStorage )BeginBulkInsert ()error { 115s .m .Lock () 116defer s .m .Unlock () 117 118tx ,err := s .db .Begin () 119if err != nil { 120return err 121 } 122s .tx = tx 123 124s .insertStmt ,err = tx .Prepare ( 125`INSERT INTO items (parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag) 126VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` , 127 ) 128if err != nil { 129rollbackErr := tx .Rollback () 130if rollbackErr != nil { 131log .Errorf ("failed to rollback transaction: %v" ,rollbackErr ) 132 } 133return err 134 } 135 136s .updateStmt ,err = tx .Prepare ( 137`UPDATE items SET size = ?, usage = ?, item_count = ?, flag = ? WHERE id = ?` , 138 ) 139if err != nil { 140s .insertStmt .Close () 141rollbackErr := tx .Rollback () 142if rollbackErr != nil { 143log .Errorf ("failed to rollback transaction: %v" ,rollbackErr ) 144 } 145return err 146 } 147 148s .hasInodeStmt ,err = tx .Prepare ( 149`SELECT 1 FROM items WHERE mli = ? LIMIT 1` , 150 ) 151if err != nil { 152s .insertStmt .Close () 153s .updateStmt .Close () 154rollbackErr := tx .Rollback () 155if rollbackErr != nil { 156log .Errorf ("failed to rollback transaction: %v" ,rollbackErr ) 157 } 158return err 159 } 160 161return nil 162} 163 164// EndBulkInsert commits the transaction and closes prepared statements 165func (s * SqliteStorage )EndBulkInsert ()error { 166s .m .Lock () 167defer s .m .Unlock () 168 169if s .insertStmt != nil { 170s .insertStmt .Close () 171s .insertStmt = nil 172 } 173if s .updateStmt != nil { 174s .updateStmt .Close () 175s .updateStmt = nil 176 } 177if s .hasInodeStmt != nil { 178s .hasInodeStmt .Close () 179s .hasInodeStmt = nil 180 } 181if s .tx != nil { 182err := s .tx .Commit () 183s .tx = nil 184return err 185 } 186return nil 187} 188 189// HasData returns true if the database contains analysis data 190func (s * SqliteStorage )HasData ()bool { 191s .m .RLock () 192defer s .m .RUnlock () 193 194var rowid int 195err := s .db .QueryRow ("SELECT MAX(rowid) FROM items" ).Scan (& rowid ) 196if err != nil { 197return false 198 } 199return rowid > 0 200} 201 202// HasInode returns true if a file with the given inode already exists in the database 203func (s * SqliteStorage )HasInode (mli uint64 )bool { 204var exists int 205var err error 206 207if s .hasInodeStmt != nil { 208err = s .hasInodeStmt .QueryRow (mli ).Scan (& exists ) 209 }else { 210s .m .RLock () 211err = s .db .QueryRow (`SELECT 1 FROM items WHERE mli = ? LIMIT 1` ,mli ).Scan (& exists ) 212s .m .RUnlock () 213 } 214 215return err == nil 216} 217 218// GetRootItem returns the root item (item with no parent) 219func (s * SqliteStorage )GetRootItem () (* SqliteItem ,error ) { 220s .m .RLock () 221defer s .m .RUnlock () 222 223item := & SqliteItem {storage :s } 224var parentID sql.NullInt64 225var isDirInt int 226var mtimeUnix int64 227var flag string 228 229err := s .db .QueryRow ( 230`SELECT id, parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag 231FROM items WHERE parent_id IS NULL LIMIT 1` , 232 ).Scan ( 233& item .id ,& parentID ,& item .name ,& isDirInt , 234& item .size ,& item .usage ,& mtimeUnix ,& item .itemCount , 235& item .mli ,& flag , 236 ) 237if err != nil { 238return nil ,err 239 } 240 241item .isDir = isDirInt == 1 242item .mtime = time .Unix (mtimeUnix ,0 ) 243if flag != "" { 244item .flag = rune (flag [0 ]) 245 }else { 246item .flag = ' ' 247 } 248 249return item ,nil 250} 251 252// InsertItem inserts a file/directory item into the database 253func (s * SqliteStorage )InsertItem ( 254parentID * int64 ,name string ,isDir bool ,size ,usage int64 ,mtime time.Time ,itemCount int64 ,mli uint64 ,flag rune , 255) (int64 ,error ) { 256isDirInt := 0 257if isDir { 258isDirInt = 1 259 } 260 261var result sql.Result 262var err error 263 264// Use prepared statement if in bulk mode, otherwise use direct exec 265if s .insertStmt != nil { 266result ,err = s .insertStmt .Exec (parentID ,name ,isDirInt ,size ,usage ,mtime .Unix (),itemCount ,mli ,string (flag )) 267 }else { 268s .m .Lock () 269result ,err = s .db .Exec ( 270`INSERT INTO items (parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag) 271VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` , 272parentID ,name ,isDirInt ,size ,usage ,mtime .Unix (),itemCount ,mli ,string (flag ), 273 ) 274s .m .Unlock () 275 } 276if err != nil { 277return 0 ,err 278 } 279 280return result .LastInsertId () 281} 282 283// UpdateItem updates an existing item's stats 284func (s * SqliteStorage )UpdateItem (id ,size ,usage ,itemCount int64 ,flag rune )error { 285var err error 286 287// Use prepared statement if in bulk mode, otherwise use direct exec 288if s .updateStmt != nil { 289_ ,err = s .updateStmt .Exec (size ,usage ,itemCount ,string (flag ),id ) 290 }else { 291s .m .Lock () 292_ ,err = s .db .Exec ( 293`UPDATE items SET size = ?, usage = ?, item_count = ?, flag = ? WHERE id = ?` , 294size ,usage ,itemCount ,string (flag ),id , 295 ) 296s .m .Unlock () 297 } 298return err 299} 300 301// deleteItemTree removes an item and all its descendants from the database within the given transaction. 302func (s * SqliteStorage )deleteItemTree (tx * sql.Tx ,id int64 )error { 303query := ` 304WITH RECURSIVE to_delete(id) AS ( 305SELECT ? 306UNION ALL 307SELECT items.id FROM items JOIN to_delete ON items.parent_id = to_delete.id 308) 309DELETE FROM items WHERE id IN (SELECT id FROM to_delete) 310` 311_ ,err := tx .Exec (query ,id ) 312return err 313} 314 315// RemoveItemAndUpdateStats removes an item and updates all its ancestors' stats in the database 316func (s * SqliteStorage )RemoveItemAndUpdateStats (id ,size ,usage ,itemCount int64 )error { 317s .m .Lock () 318defer s .m .Unlock () 319 320tx ,err := s .db .Begin () 321if err != nil { 322return err 323 } 324 325// 1. Update all ancestors using recursive CTE 326updateAncestorsQuery := ` 327WITH RECURSIVE ancestors(id, parent_id) AS ( 328SELECT parent_id, NULL FROM items WHERE id = ? 329UNION ALL 330SELECT i.parent_id, NULL FROM items i JOIN ancestors a ON i.id = a.id WHERE i.parent_id IS NOT NULL 331) 332UPDATE items 333SET size = size - ?, usage = usage - ?, item_count = item_count - ? 334WHERE id IN (SELECT id FROM ancestors) 335` 336_ ,err = tx .Exec (updateAncestorsQuery ,id ,size ,usage ,itemCount ) 337if err != nil { 338rollbackErr := tx .Rollback () 339if rollbackErr != nil { 340log .Errorf ("failed to rollback transaction: %v" ,rollbackErr ) 341 } 342return err 343 } 344 345// 2. Delete the item and its descendants 346if err = s .deleteItemTree (tx ,id );err != nil { 347rollbackErr := tx .Rollback () 348if rollbackErr != nil { 349log .Errorf ("failed to rollback transaction: %v" ,rollbackErr ) 350 } 351return err 352 } 353 354return tx .Commit () 355} 356 357// GetChildByName returns a child item by its name and parent ID 358func (s * SqliteStorage )GetChildByName (parentID int64 ,name string ) (* SqliteItem ,error ) { 359s .m .RLock () 360defer s .m .RUnlock () 361 362item := & SqliteItem {storage :s } 363var pID sql.NullInt64 364var isDirInt int 365var mtimeUnix int64 366var flag string 367 368err := s .db .QueryRow ( 369`SELECT id, parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag 370FROM items WHERE parent_id = ? AND name = ? LIMIT 1` , 371parentID ,name , 372 ).Scan ( 373& item .id ,& pID ,& item .name ,& isDirInt , 374& item .size ,& item .usage ,& mtimeUnix ,& item .itemCount , 375& item .mli ,& flag , 376 ) 377if err != nil { 378return nil ,err 379 } 380 381if pID .Valid { 382item .parentID = & pID .Int64 383 } 384item .isDir = isDirInt == 1 385item .mtime = time .Unix (mtimeUnix ,0 ) 386if flag != "" { 387item .flag = rune (flag [0 ]) 388 }else { 389item .flag = ' ' 390 } 391 392return item ,nil 393} 394 395// GetChildren returns all children of a given parent ID 396func (s * SqliteStorage )GetChildren (parentID int64 ) ([]* SqliteItem ,error ) { 397s .m .RLock () 398defer s .m .RUnlock () 399 400rows ,err := s .db .Query ( 401`SELECT id, parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag 402FROM items WHERE parent_id = ?` , 403parentID , 404 ) 405if err != nil { 406return nil ,err 407 } 408defer rows .Close () 409 410var items []* SqliteItem 411for rows .Next () { 412item := & SqliteItem {storage :s } 413var parentID sql.NullInt64 414var isDirInt int 415var mtimeUnix int64 416var flag string 417 418err := rows .Scan ( 419& item .id ,& parentID ,& item .name ,& isDirInt , 420& item .size ,& item .usage ,& mtimeUnix ,& item .itemCount , 421& item .mli ,& flag , 422 ) 423if err != nil { 424return nil ,err 425 } 426 427if parentID .Valid { 428item .parentID = & parentID .Int64 429 } 430item .isDir = isDirInt == 1 431item .mtime = time .Unix (mtimeUnix ,0 ) 432if flag != "" { 433item .flag = rune (flag [0 ]) 434 }else { 435item .flag = ' ' 436 } 437items = append (items ,item ) 438 } 439 440return items ,rows .Err () 441} 442 443// GetItemByID returns an item by its ID 444func (s * SqliteStorage )GetItemByID (id int64 ) (* SqliteItem ,error ) { 445s .m .RLock () 446defer s .m .RUnlock () 447 448item := & SqliteItem {storage :s } 449var parentID sql.NullInt64 450var isDirInt int 451var mtimeUnix int64 452var flag string 453 454err := s .db .QueryRow ( 455`SELECT id, parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag 456FROM items WHERE id = ?` , 457id , 458 ).Scan ( 459& item .id ,& parentID ,& item .name ,& isDirInt , 460& item .size ,& item .usage ,& mtimeUnix ,& item .itemCount , 461& item .mli ,& flag , 462 ) 463if err != nil { 464return nil ,err 465 } 466 467if parentID .Valid { 468item .parentID = & parentID .Int64 469 } 470item .isDir = isDirInt == 1 471item .mtime = time .Unix (mtimeUnix ,0 ) 472if flag != "" { 473item .flag = rune (flag [0 ]) 474 }else { 475item .flag = ' ' 476 } 477 478return item ,nil 479} 480 481// SetMetadata stores a metadata key-value pair 482func (s * SqliteStorage )SetMetadata (key ,value string )error { 483s .m .Lock () 484defer s .m .Unlock () 485 486_ ,err := s .db .Exec ( 487`INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)` , 488key ,value , 489 ) 490return err 491} 492 493// GetMetadata retrieves a metadata value by key 494func (s * SqliteStorage )GetMetadata (key string ) (string ,error ) { 495s .m .RLock () 496defer s .m .RUnlock () 497 498var value string 499err := s .db .QueryRow (`SELECT value FROM metadata WHERE key = ?` ,key ).Scan (& value ) 500return value ,err 501} 502 503// SqliteItem represents a file or directory stored in SQLite 504type SqliteItem struct { 505storage * SqliteStorage 506id int64 507parentID * int64 508name string 509isDir bool 510size int64 511usage int64 512mtime time.Time 513itemCount int64 514mli uint64 515flag rune 516parent fs.Item 517m sync.RWMutex 518} 519 520// GetPath returns the full path of the item 521func (i * SqliteItem )GetPath ()string { 522parent := i .GetParent () 523if parent != nil { 524return filepath .Join (parent .GetPath (),i .name ) 525 } 526// For root item, get basePath from metadata 527basePath ,err := i .storage .GetMetadata ("top_dir_path" ) 528if err != nil { 529return i .name 530 } 531return filepath .Join (filepath .Dir (basePath ),i .name ) 532} 533 534// GetName returns the name of the item 535func (i * SqliteItem )GetName ()string { 536return i .name 537} 538 539// GetFlag returns the flag of the item 540func (i * SqliteItem )GetFlag ()rune { 541return i .flag 542} 543 544// IsDir returns true if the item is a directory 545func (i * SqliteItem )IsDir ()bool { 546return i .isDir 547} 548 549// GetSize returns the apparent size 550func (i * SqliteItem )GetSize ()int64 { 551i .m .RLock () 552defer i .m .RUnlock () 553return i .size 554} 555 556// GetType returns the type of the item 557func (i * SqliteItem )GetType ()string { 558if i .isDir { 559return "Directory" 560 } 561if i .flag == '@' { 562return "Other" 563 } 564return "File" 565} 566 567// GetUsage returns the disk usage 568func (i * SqliteItem )GetUsage ()int64 { 569i .m .RLock () 570defer i .m .RUnlock () 571return i .usage 572} 573 574// GetMtime returns the modification time 575func (i * SqliteItem )GetMtime () time.Time { 576i .m .RLock () 577defer i .m .RUnlock () 578return i .mtime 579} 580 581// GetItemCount returns the item count 582func (i * SqliteItem )GetItemCount ()int64 { 583i .m .RLock () 584defer i .m .RUnlock () 585return i .itemCount 586} 587 588// GetParent returns the parent item 589func (i * SqliteItem )GetParent () fs.Item { 590i .m .RLock () 591if i .parent != nil { 592defer i .m .RUnlock () 593return i .parent 594 } 595if i .parentID == nil { 596defer i .m .RUnlock () 597return nil 598 } 599i .m .RUnlock () 600 601parent ,err := i .storage .GetItemByID (* i .parentID ) 602if err != nil { 603log .err .Error ()) 604return nil 605 } 606 607i .m .Lock () 608defer i .m .Unlock () 609if i .parent == nil { 610i .parent = parent 611 } 612return i .parent 613} 614 615// SetParent sets the parent item 616func (i * SqliteItem )SetParent (parent fs.Item ) { 617i .parent = parent 618} 619 620// GetParentLocked returns the in-memory parent without hitting the database. 621// Used inside RemoveFile where storage.m is already write-locked. 622func (i * SqliteItem )GetParentLocked ()* SqliteItem { 623if i .parent != nil { 624return i .parent .(* SqliteItem ) 625 } 626return nil 627} 628 629// GetMultiLinkedInode returns the multi-linked inode number 630func (i * SqliteItem )GetMultiLinkedInode ()uint64 { 631return i .mli 632} 633 634// EncodeJSON encodes the item to JSON 635func (i * SqliteItem )EncodeJSON (writer io.Writer ,topLevel bool ,attributes fs.JSONAttributes )error { 636if i .isDir { 637return i .encodeDirJSON (writer ,topLevel ,attributes ) 638 } 639return i .encodeFileJSON (writer ,attributes ) 640} 641 642func (i * SqliteItem )encodeDirJSON (writer io.Writer ,topLevel bool ,attributes fs.JSONAttributes )error { 643buff := make ([]byte ,0 ,128 ) 644buff = append (buff , []byte (`[{"name":` )... ) 645 646name := i .GetName () 647if topLevel { 648name = i .GetPath () 649 } 650if err := addSqliteString (& buff ,name );err != nil { 651return err 652 } 653 654// Directory summary stats (asize, dsize, items) are written by default so 655// they can be preserved on import. When --output-attrs is supplied, each is 656// emitted only if explicitly selected. 657if attributes .Includes ("asize" ) { 658buff = append (buff , []byte (`,"asize":` )... ) 659buff = append (buff , []byte (strconv .FormatInt (i .GetSize (),10 ))... ) 660 } 661if attributes .Includes ("dsize" ) { 662buff = append (buff , []byte (`,"dsize":` )... ) 663buff = append (buff , []byte (strconv .FormatInt (i .GetUsage (),10 ))... ) 664 } 665if attributes .Includes ("items" ) { 666buff = append (buff , []byte (`,"items":` )... ) 667buff = append (buff , []byte (strconv .FormatInt (i .GetItemCount (),10 ))... ) 668 } 669if attributes .Includes ("mtime" )&& ! i .GetMtime ().IsZero () { 670buff = append (buff , []byte (`,"mtime":` )... ) 671buff = append (buff , []byte (strconv .FormatInt (i .GetMtime ().Unix (),10 ))... ) 672 } 673 674buff = append (buff ,'}' ) 675 676children ,err := i .storage .GetChildren (i .id ) 677if err != nil { 678return err 679 } 680 681if len (children )> 0 { 682buff = append (buff ,',' ) 683 } 684buff = append (buff ,'\n' ) 685 686if _ ,err := writer .Write (buff );err != nil { 687return err 688 } 689 690for idx ,child := range children { 691if idx > 0 { 692if _ ,err := writer .Write ([]byte (",\n" ));err != nil { 693return err 694 } 695 } 696child .parent = i 697if err := child .EncodeJSON (writer ,false ,attributes );err != nil { 698return err 699 } 700 } 701 702if _ ,err := writer .Write ([]byte ("]" ));err != nil { 703return err 704 } 705return nil 706} 707 708func (i * SqliteItem )encodeFileJSON (writer io.Writer ,attributes fs.JSONAttributes )error { 709buff := make ([]byte ,0 ,128 ) 710buff = append (buff , []byte (`{"name":` )... ) 711if err := addSqliteString (& buff ,i .GetName ());err != nil { 712return err 713 } 714if attributes .Includes ("asize" )&& i .GetSize ()> 0 { 715buff = append (buff , []byte (`,"asize":` )... ) 716buff = append (buff , []byte (strconv .FormatInt (i .GetSize (),10 ))... ) 717 } 718if attributes .Includes ("dsize" )&& i .GetUsage ()> 0 { 719buff = append (buff , []byte (`,"dsize":` )... ) 720buff = append (buff , []byte (strconv .FormatInt (i .GetUsage (),10 ))... ) 721 } 722if attributes .Includes ("mtime" )&& ! i .GetMtime ().IsZero () { 723buff = append (buff , []byte (`,"mtime":` )... ) 724buff = append (buff , []byte (strconv .FormatInt (i .GetMtime ().Unix (),10 ))... ) 725 } 726 727if attributes .Includes ("notreg" )&& i .flag == '@' { 728buff = append (buff , []byte (`,"notreg":true` )... ) 729 } 730if attributes == nil && i .flag == 'H' { 731buff = append (buff , []byte (`,"ino":` + strconv .FormatUint (i .mli ,10 )+ `,"hlnkc":true` )... ) 732 } 733 734buff = append (buff ,'}' ) 735 736if _ ,err := writer .Write (buff );err != nil { 737return err 738 } 739return nil 740} 741 742func addSqliteString (buff * []byte ,val string )error { 743b ,err := json .Marshal (val ) 744if err != nil { 745return err 746 } 747* buff = append (* buff ,b ... ) 748return nil 749} 750 751// GetItemStats returns item statistics - hard links already handled during scan 752func (i * SqliteItem )GetItemStats (linkedItems fs.HardLinkedItems ,filteringFiles bool ) (itemCount ,size ,usage int64 ) { 753return i .itemCount ,i .size ,i .usage 754} 755 756// UpdateStats is a no-op for SqliteItem - hard links are handled during scan 757func (i * SqliteItem )UpdateStats (linkedItems fs.HardLinkedItems ) {} 758func (i * SqliteItem )UpdateStatsWithFileFiltering (linkedItems fs.HardLinkedItems ) {} 759 760// AddFile adds a child file (no-op for SQLite items - children are in DB) 761func (i * SqliteItem )AddFile (item fs.Item ) { 762// Children are stored in database via parent_id relationship 763} 764 765// GetFiles returns children as a sorted iterator 766func (i * SqliteItem )GetFiles (sortBy fs.SortBy ,order fs.SortOrder ) iter.Seq [fs.Item ] { 767return func (yield func (fs.Item )bool ) { 768children ,err := i .storage .GetChildren (i .id ) 769if err != nil { 770log .err .Error ()) 771return 772 } 773 774// Convert to fs.Files for sorting 775files := make (fs.Files ,len (children )) 776for idx ,child := range children { 777child .parent = i 778files [idx ]= child 779 } 780 781sortFiles (files ,sortBy ,order ) 782 783for _ ,item := range files { 784if ! yield (item ) { 785return 786 } 787 } 788 } 789} 790 791// GetFilesLocked returns children with locking 792func (i * SqliteItem )GetFilesLocked (sortBy fs.SortBy ,order fs.SortOrder ) iter.Seq [fs.Item ] { 793return i .GetFiles (sortBy ,order ) 794} 795 796// RemoveFile removes a child file, updating both in-memory stats and the database in one pass. 797func (i * SqliteItem )RemoveFile (item fs.Item ) { 798sqliteItem := item .(* SqliteItem ) 799size := item .GetSize () 800usage := item .GetUsage () 801itemCount := item .GetItemCount () 802 803// Apply in-memory stats updates for loaded ancestors first, 804// to avoid double-counting if ancestors are loaded from DB after it's updated. 805cur := i 806for { 807cur .m .Lock () 808cur .size -= size 809cur .usage -= usage 810cur .itemCount -= itemCount 811cur .m .Unlock () 812 813parent := cur .GetParentLocked () 814if parent == nil { 815break 816 } 817cur = parent 818 } 819 820if err := i .storage .RemoveItemAndUpdateStats (sqliteItem .id ,size ,usage ,itemCount );err != nil { 821log .Errorf ("Error removing item and updating stats: %v" ,err ) 822return 823 } 824} 825 826// RemoveFileByName removes a child by name 827func (i * SqliteItem )RemoveFileByName (name string ) { 828child ,err := i .storage .GetChildByName (i .id ,name ) 829if err != nil { 830log .Errorf ("Error getting child from database: %v" ,err ) 831return 832 } 833 834i .RemoveFile (child ) 835} 836 837// RLock returns a no-op unlock function 838func (i * SqliteItem )RLock ()func () { 839i .m .RLock () 840return i .m .RUnlock 841} 842 843// SqliteAnalyzer implements Analyzer using SQLite storage 844type SqliteAnalyzer struct { 845BaseAnalyzer 846storage * SqliteStorage 847dbWriteMu sync.Mutex 848} 849 850// insertItemLocked is a serialized wrapper around storage.InsertItem. 851// All concurrent goroutines spawned during AnalyzeDir must use this helper 852// because the underlying *sql.Tx and prepared statements are not safe for 853// concurrent use. 854func (a * SqliteAnalyzer )insertItemLocked ( 855parentID * int64 ,name string ,isDir bool ,size ,usage int64 ,mtime time.Time , 856itemCount int64 ,mli uint64 ,flag rune , 857) (int64 ,error ) { 858a .dbWriteMu .Lock () 859defer a .dbWriteMu .Unlock () 860return a .storage .InsertItem (parentID ,name ,isDir ,size ,usage ,mtime ,itemCount ,mli ,flag ) 861} 862 863func (a * SqliteAnalyzer )updateDirLocked (id ,size ,usage ,itemCount int64 ,flag rune )error { 864a .dbWriteMu .Lock () 865defer a .dbWriteMu .Unlock () 866return a .storage .UpdateItem (id ,size ,usage ,itemCount ,flag ) 867} 868 869// hasInodeLocked is a serialized wrapper around storage.HasInode. 870func (a * SqliteAnalyzer )hasInodeLocked (mli uint64 )bool { 871a .dbWriteMu .Lock () 872defer a .dbWriteMu .Unlock () 873return a .storage .HasInode (mli ) 874} 875 876// CreateSqliteAnalyzer creates a new SQLite analyzer 877func CreateSqliteAnalyzer (dbPath string ) (* SqliteAnalyzer ,error ) { 878if err := checkAvailable ();err != nil { 879return nil ,err 880 } 881 882storage ,err := NewSqliteStorage (dbPath ) 883if err != nil { 884return nil ,err 885 } 886 887a := & SqliteAnalyzer { 888storage :storage , 889 } 890a .Init () 891return a ,nil 892} 893 894// AnalyzeDir analyzes the given path and stores results in SQLite. 895// If the database already contains data, it loads from the database instead of re-scanning. 896func (a * SqliteAnalyzer )AnalyzeDir ( 897path string ,ignore common.ShouldDirBeIgnored ,fileTypeFilter common.ShouldFileBeIgnored , 898) fs.Item { 899// Check if database already has data 900if a .storage .HasData () { 901log .Printf ("Loading analysis from existing SQLite database" ) 902rootItem ,err := a .storage .GetRootItem () 903if err != nil { 904log .Printf ("Error loading from database, will re-scan: %v" ,err ) 905 }else { 906// Signal that we're done immediately 907a .doneChan .Broadcast () 908return rootItem 909 } 910 } 911 912a .ignoreDir = ignore 913a .ignoreFileType = fileTypeFilter 914 915// Clear existing data and store metadata 916err := a .storage .ClearItems () 917if err != nil { 918log .Printf ("Error clearing items: %v" ,err ) 919 } 920err = a .storage .SetMetadata ("top_dir_path" ,path ) 921if err != nil { 922log .Printf ("Error setting metadata: %v" ,err ) 923 } 924 925// Start bulk insert transaction 926if err := a .storage .BeginBulkInsert ();err != nil { 927log .Printf ("Error starting bulk insert: %v" ,err ) 928 } 929 930go a .UpdateProgress () 931 932// Process directory and get the root item 933rootItem := a .processDir (path ,nil ) 934 935a .wait .Wait () 936 937// Commit bulk insert transaction 938if err := a .storage .EndBulkInsert ();err != nil { 939log .Printf ("Error committing bulk insert: %v" ,err ) 940 } 941 942a .progressDoneChan <- struct {}{} 943a .doneChan .Broadcast () 944 945return rootItem 946} 947 948// fileStat holds stats for a single file entry computed by processFile. 949type fileStat struct { 950size int64 951usage int64 952itemCount int64 953mli uint64 954flag rune 955// archiveDir is non-nil when the file is an archive that was expanded. 956archiveDir * Dir 957} 958 959// processArchiveEntry tries to expand name/entryPath as a zip or tar archive. 960// Returns the archive *Dir on success, or nil on failure (in which case err is set). 961func (a * SqliteAnalyzer )processArchiveEntry (entryPath ,name string ,info os.FileInfo ) (* Dir ,error ) { 962if isZipFile (name ) { 963archiveDirZip ,errZip := processZipFile (entryPath ,info ) 964if errZip != nil { 965return nil ,errZip 966 } 967uncompressedSize ,compressedSize ,errSize := getZipFileSize (entryPath ) 968if errSize == nil { 969archiveDirZip .Size = uncompressedSize 970archiveDirZip .Usage = compressedSize 971 } 972return archiveDirZip .Dir ,nil 973 } 974archiveDirTar ,errTar := processTarFile (entryPath ,info ) 975if errTar != nil { 976return nil ,errTar 977 } 978return archiveDirTar .Dir ,nil 979} 980 981// processFile resolves a single non-directory entry and returns its stats. 982// It returns nil when the file should be skipped entirely. 983func (a * SqliteAnalyzer )processFile (entryPath ,name string ,f os.DirEntry ) (stat fileStat ) { 984if a .ignoreFileType != nil && a .ignoreFileType (name ) { 985return stat 986 } 987 988info ,err := f .Info () 989if err != nil { 990log .err .Error ()) 991return stat 992 } 993 994if a .followSymlinks && info .Mode ()& os .ModeSymlink != 0 { 995infoF ,err := followSymlink (entryPath ,a .gitAnnexedSize ) 996if err != nil { 997log .err .Error ()) 998return stat 999 } 1000if infoF != nil { 1001info = infoF 1002 } 1003 } 1004 1005if a .matchesTimeFilterFn != nil && ! a .matchesTimeFilterFn (info .ModTime ()) { 1006return stat 1007 } 1008 1009if a .archiveBrowsing && (isZipFile (name )|| isTarFile (name )) { 1010archiveDir ,err := a .processArchiveEntry (entryPath ,name ,info ) 1011if err != nil { 1012log .Printf ("Failed to process archive %s: %v" ,entryPath ,err ) 1013 }else { 1014return fileStat { 1015size :archiveDir .Size , 1016usage :archiveDir .Usage , 1017itemCount :archiveDir .ItemCount , 1018flag :archiveDir .Flag , 1019archiveDir :archiveDir , 1020 } 1021 } 1022 } 1023 1024fileSize := info .Size () 1025fileUsage ,fileMli := getSyscallStats (info ) 1026fileFlag := getFlag (info ) 1027 1028if fileMli != 0 && a .hasInodeLocked (fileMli ) { 1029fileSize = 0 1030fileUsage = 0 1031fileFlag = 'H' 1032 } 1033 1034return fileStat { 1035size :fileSize , 1036usage :fileUsage , 1037mli :fileMli , 1038flag :fileFlag , 1039 } 1040} 1041 1042// nolint:funlen 1043func (a * SqliteAnalyzer )processDir (path string ,parentID * int64 )* SqliteItem { 1044// Start with 4096 for directory's own size/usage, matching Dir.UpdateStats behavior 1045var ( 1046totalSize int64 = 4096 1047totalUsage int64 = 4096 1048filesSize int64 // only files in this directory, for progress reporting 1049itemCount int64 = 1 1050subDirChan = make (chan * SqliteItem ) 1051dirCount int 1052 ) 1053 1054a .wait .Add (1 ) 1055defer a .wait .Done () 1056 1057// Hold a concurrency slot only during the scan/insert phase. We must 1058// release it before draining subDirChan, otherwise child goroutines 1059// (which need to acquire a slot themselves) would deadlock once the 1060// global concurrencyLimit is saturated by ancestors waiting on their 1061// own children. 1062concurrencyLimit <- struct {}{} 1063slotReleased := false 1064releaseSlot := func () { 1065if ! slotReleased { 1066<- concurrencyLimit 1067slotReleased = true 1068 } 1069 } 1070defer releaseSlot () 1071 1072if parentID != nil && a .IsCancelled () { 1073return nil 1074 } 1075 1076files ,err := os .ReadDir (path ) 1077if err != nil { 1078log .err .Error ()) 1079 } 1080 1081// Get directory info for mtime 1082dirInfo ,statErr := os .Stat (path ) 1083var dirMtime time.Time 1084if statErr == nil { 1085dirMtime = dirInfo .ModTime () 1086 } 1087 1088dirFlag := getDirFlag (err ,len (files )) 1089 1090// Insert directory into database (size/usage will be updated later) 1091dirID ,err := a .insertItemLocked ( 1092parentID , 1093filepath .Base (path ), 1094true , 10950 ,// size will be updated later 10960 ,// usage will be updated later 1097dirMtime , 10981 ,// item_count will be updated later 10990 , 1100dirFlag , 1101 ) 1102if err != nil { 1103log .err .Error ()) 1104return nil 1105 } 1106 1107// Spawn subdirectory scans in parallel; each goroutine fully completes its 1108// subtree (including DB row finalization) before sending the result back. 1109for _ ,f := range files { 1110if a .IsCancelled () { 1111break 1112 } 1113name := f .Name () 1114entryPath := filepath .Join (path ,name ) 1115 1116if f .IsDir () { 1117if a .shouldSkipDir (name ,entryPath ) { 1118continue 1119 } 1120dirCount ++ 1121 1122go func (entryPath string ) { 1123sub := a .processDir (entryPath ,& dirID ) 1124subDirChan <- sub 1125 }(entryPath ) 1126continue 1127 } 1128 1129info ,err := f .Info () 1130if err != nil { 1131log .err .Error ()) 1132continue 1133 } 1134 1135stat := a .processFile (entryPath ,name ,f ) 1136if stat == (fileStat {}) { 1137continue 1138 } 1139 1140if stat .archiveDir != nil { 1141archiveID ,err := a .insertItemLocked ( 1142& dirID , 1143name , 1144true , 1145stat .archiveDir .Size , 1146stat .archiveDir .Usage , 1147info .ModTime (), 1148stat .archiveDir .ItemCount , 11490 , 1150stat .archiveDir .Flag , 1151 ) 1152if err != nil { 1153log .err .Error ()) 1154continue 1155 } 1156a .persistArchive (stat .archiveDir ,archiveID ) 1157 1158totalSize += stat .size 1159totalUsage += stat .usage 1160filesSize += stat .usage 1161itemCount += stat .itemCount 1162continue 1163 } 1164 1165_ ,err = a .insertItemLocked ( 1166& dirID , 1167name , 1168false , 1169stat .size , 1170stat .usage , 1171info .ModTime (), 11721 , 1173stat .mli , 1174stat .flag , 1175 ) 1176if err != nil { 1177log .err .Error ()) 1178continue 1179 } 1180 1181totalSize += stat .size 1182totalUsage += stat .usage 1183filesSize += stat .usage 1184itemCount ++ 1185 } 1186 1187// Release the concurrency slot before draining subDirChan so children 1188// blocked on concurrencyLimit can make progress. 1189releaseSlot () 1190 1191// Aggregate subdirectory results. Each sub is fully finalized when received. 1192for i := 0 ;i < dirCount ;i ++ { 1193sub := <- subDirChan 1194if sub != nil { 1195totalSize += sub .size 1196totalUsage += sub .usage 1197itemCount += sub .itemCount 1198switch sub .flag { 1199case '!' ,'.' : 1200if dirFlag != '!' { 1201dirFlag = '.' 1202 } 1203 } 1204 } 1205 } 1206 1207// Update directory with computed stats 1208if err := a .updateDirLocked (dirID ,totalSize ,totalUsage ,itemCount ,dirFlag );err != nil { 1209log .Printf ("Error updating item: %v" ,err ) 1210 } 1211 1212// Report progress (only files in this dir, subdirs already reported themselves) 1213a .progressCurrentItemName .Store (path ) 1214a .progressItemCount .Add (int64 (len (files ))) 1215a .progressTotalUsage .Add (filesSize ) 1216 1217// Return SqliteItem for the directory 1218return & SqliteItem { 1219storage :a .storage , 1220id :dirID , 1221parentID :parentID , 1222name :filepath .Base (path ), 1223isDir :true , 1224size :totalSize , 1225usage :totalUsage , 1226mtime :dirMtime , 1227itemCount :itemCount , 1228flag :dirFlag , 1229 } 1230} 1231 1232func (a * SqliteAnalyzer )persistArchive (archiveDir * Dir ,parentID int64 ) { 1233if archiveDir == nil { 1234return 1235 } 1236for _ ,f := range archiveDir .Files { 1237if a .IsCancelled () { 1238return 1239 } 1240if f .IsDir () { 1241var subDir * Dir 1242switch v := f .(type ) { 1243case * ZipDir : 1244subDir = v .Dir 1245case * TarDir : 1246subDir = v .Dir 1247 } 1248 1249if subDir == nil { 1250continue 1251 } 1252 1253id ,err := a .insertItemLocked ( 1254& parentID , 1255f .GetName (), 1256true , 1257f .GetSize (), 1258f .GetUsage (), 1259f .GetMtime (), 1260f .GetItemCount (), 12610 , 1262f .GetFlag (), 1263 ) 1264if err != nil { 1265log .err .Error ()) 1266continue 1267 } 1268a .persistArchive (subDir ,id ) 1269 }else { 1270_ ,err := a .insertItemLocked ( 1271& parentID , 1272f .GetName (), 1273false , 1274f .GetSize (), 1275f .GetUsage (), 1276f .GetMtime (), 12771 , 1278f .GetMultiLinkedInode (), 1279f .GetFlag (), 1280 ) 1281if err != nil { 1282log .err .Error ()) 1283continue 1284 } 1285 } 1286 } 1287}