fork/gdu

git clone https://git.julia.blue/fork/gdu

ShivamB25feat: customize JSON export attributes (#605)d062e88

subvolume
30.2 KiB1287 linesraw
1package analyze
2
3import (
4	"database/sql"
5	"encoding/json"
6	"io"
7	"iter"
8	"os"
9	"path/filepath"
10	"strconv"
11	"sync"
12	"time"
13
14	"github.com/dundee/gdu/v5/internal/common"
15	"github.com/dundee/gdu/v5/pkg/fs"
16	"github.com/pkg/errors"
17	log "github.com/sirupsen/logrus"
18)
19
20// SqliteStorage represents SQLite database storage
21type SqliteStorage struct {
22	db           *sql.DB
23	dbPath       string
24	m            sync.RWMutex
25	tx           *sql.Tx
26	insertStmt   *sql.Stmt
27	updateStmt   *sql.Stmt
28	hasInodeStmt *sql.Stmt
29}
30
31// NewSqliteStorage creates a new SQLite storage and initializes the schema
32func NewSqliteStorage(dbPath string) (*SqliteStorage, error) {
33	parentDir := filepath.Dir(dbPath)
34	if err := os.MkdirAll(parentDir, 0o755); err != nil {
35		return nil, errors.Wrap(err, "failed to create parent directory for SQLite database")
36	}
37
38	db, err := sql.Open("sqlite", dbPath)
39	if err != nil {
40		return nil, err
41	}
42
43	storage := &SqliteStorage{
44		db:     db,
45		dbPath: dbPath,
46	}
47
48	if err := storage.createTables(); err != nil {
49		db.Close()
50		return nil, err
51	}
52
53	return storage, nil
54}
55
56// createTables creates the database schema if it doesn't exist
57func (s *SqliteStorage) createTables() error {
58	// Optimize for insertion speed
59	pragmas := `
60	PRAGMA synchronous = OFF;
61	PRAGMA journal_mode = MEMORY;
62	PRAGMA cache_size = -64000;
63	PRAGMA temp_store = MEMORY;
64	`
65	if _, err := s.db.Exec(pragmas); err != nil {
66		return err
67	}
68
69	schema := `
70	CREATE TABLE IF NOT EXISTS items (
71		id          INTEGER PRIMARY KEY,
72		parent_id   INTEGER REFERENCES items(id),
73		name        TEXT NOT NULL,
74		is_dir      INTEGER NOT NULL,
75		size        INTEGER NOT NULL,
76		usage       INTEGER NOT NULL,
77		mtime       INTEGER NOT NULL,
78		item_count  INTEGER NOT NULL DEFAULT 1,
79		mli         INTEGER NOT NULL DEFAULT 0,
80		flag        TEXT NOT NULL DEFAULT ' '
81	);
82
83	CREATE INDEX IF NOT EXISTS idx_items_parent_id ON items(parent_id);
84	CREATE INDEX IF NOT EXISTS idx_items_parent_name ON items(parent_id, name);
85	CREATE INDEX IF NOT EXISTS idx_items_mli ON items(mli) WHERE mli != 0;
86
87	CREATE TABLE IF NOT EXISTS metadata (
88		key   TEXT PRIMARY KEY,
89		value TEXT
90	);
91	`
92
93	_, err := s.db.Exec(schema)
94	return err
95}
96
97// Close closes the database connection
98func (s *SqliteStorage) Close() error {
99	s.m.Lock()
100	defer s.m.Unlock()
101	if s.db != nil {
102		return s.db.Close()
103	}
104	return nil
105}
106
107// ClearItems removes all items from the database
108func (s *SqliteStorage) ClearItems() error {
109	_, err := s.db.Exec("DELETE FROM items")
110	return err
111}
112
113// BeginBulkInsert starts a transaction and prepares statements for bulk insertion
114func (s *SqliteStorage) BeginBulkInsert() error {
115	s.m.Lock()
116	defer s.m.Unlock()
117
118	tx, err := s.db.Begin()
119	if err != nil {
120		return err
121	}
122	s.tx = tx
123
124	s.insertStmt, err = tx.Prepare(
125		`INSERT INTO items (parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag)
126		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
127	)
128	if err != nil {
129		rollbackErr := tx.Rollback()
130		if rollbackErr != nil {
131			log.Errorf("failed to rollback transaction: %v", rollbackErr)
132		}
133		return err
134	}
135
136	s.updateStmt, err = tx.Prepare(
137		`UPDATE items SET size = ?, usage = ?, item_count = ?, flag = ? WHERE id = ?`,
138	)
139	if err != nil {
140		s.insertStmt.Close()
141		rollbackErr := tx.Rollback()
142		if rollbackErr != nil {
143			log.Errorf("failed to rollback transaction: %v", rollbackErr)
144		}
145		return err
146	}
147
148	s.hasInodeStmt, err = tx.Prepare(
149		`SELECT 1 FROM items WHERE mli = ? LIMIT 1`,
150	)
151	if err != nil {
152		s.insertStmt.Close()
153		s.updateStmt.Close()
154		rollbackErr := tx.Rollback()
155		if rollbackErr != nil {
156			log.Errorf("failed to rollback transaction: %v", rollbackErr)
157		}
158		return err
159	}
160
161	return nil
162}
163
164// EndBulkInsert commits the transaction and closes prepared statements
165func (s *SqliteStorage) EndBulkInsert() error {
166	s.m.Lock()
167	defer s.m.Unlock()
168
169	if s.insertStmt != nil {
170		s.insertStmt.Close()
171		s.insertStmt = nil
172	}
173	if s.updateStmt != nil {
174		s.updateStmt.Close()
175		s.updateStmt = nil
176	}
177	if s.hasInodeStmt != nil {
178		s.hasInodeStmt.Close()
179		s.hasInodeStmt = nil
180	}
181	if s.tx != nil {
182		err := s.tx.Commit()
183		s.tx = nil
184		return err
185	}
186	return nil
187}
188
189// HasData returns true if the database contains analysis data
190func (s *SqliteStorage) HasData() bool {
191	s.m.RLock()
192	defer s.m.RUnlock()
193
194	var rowid int
195	err := s.db.QueryRow("SELECT MAX(rowid) FROM items").Scan(&rowid)
196	if err != nil {
197		return false
198	}
199	return rowid > 0
200}
201
202// HasInode returns true if a file with the given inode already exists in the database
203func (s *SqliteStorage) HasInode(mli uint64) bool {
204	var exists int
205	var err error
206
207	if s.hasInodeStmt != nil {
208		err = s.hasInodeStmt.QueryRow(mli).Scan(&exists)
209	} else {
210		s.m.RLock()
211		err = s.db.QueryRow(`SELECT 1 FROM items WHERE mli = ? LIMIT 1`, mli).Scan(&exists)
212		s.m.RUnlock()
213	}
214
215	return err == nil
216}
217
218// GetRootItem returns the root item (item with no parent)
219func (s *SqliteStorage) GetRootItem() (*SqliteItem, error) {
220	s.m.RLock()
221	defer s.m.RUnlock()
222
223	item := &SqliteItem{storage: s}
224	var parentID sql.NullInt64
225	var isDirInt int
226	var mtimeUnix int64
227	var flag string
228
229	err := s.db.QueryRow(
230		`SELECT id, parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag
231		 FROM items WHERE parent_id IS NULL LIMIT 1`,
232	).Scan(
233		&item.id, &parentID, &item.name, &isDirInt,
234		&item.size, &item.usage, &mtimeUnix, &item.itemCount,
235		&item.mli, &flag,
236	)
237	if err != nil {
238		return nil, err
239	}
240
241	item.isDir = isDirInt == 1
242	item.mtime = time.Unix(mtimeUnix, 0)
243	if flag != "" {
244		item.flag = rune(flag[0])
245	} else {
246		item.flag = ' '
247	}
248
249	return item, nil
250}
251
252// InsertItem inserts a file/directory item into the database
253func (s *SqliteStorage) InsertItem(
254	parentID *int64, name string, isDir bool, size, usage int64, mtime time.Time, itemCount int64, mli uint64, flag rune,
255) (int64, error) {
256	isDirInt := 0
257	if isDir {
258		isDirInt = 1
259	}
260
261	var result sql.Result
262	var err error
263
264	// Use prepared statement if in bulk mode, otherwise use direct exec
265	if s.insertStmt != nil {
266		result, err = s.insertStmt.Exec(parentID, name, isDirInt, size, usage, mtime.Unix(), itemCount, mli, string(flag))
267	} else {
268		s.m.Lock()
269		result, err = s.db.Exec(
270			`INSERT INTO items (parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag)
271			 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
272			parentID, name, isDirInt, size, usage, mtime.Unix(), itemCount, mli, string(flag),
273		)
274		s.m.Unlock()
275	}
276	if err != nil {
277		return 0, err
278	}
279
280	return result.LastInsertId()
281}
282
283// UpdateItem updates an existing item's stats
284func (s *SqliteStorage) UpdateItem(id, size, usage, itemCount int64, flag rune) error {
285	var err error
286
287	// Use prepared statement if in bulk mode, otherwise use direct exec
288	if s.updateStmt != nil {
289		_, err = s.updateStmt.Exec(size, usage, itemCount, string(flag), id)
290	} else {
291		s.m.Lock()
292		_, err = s.db.Exec(
293			`UPDATE items SET size = ?, usage = ?, item_count = ?, flag = ? WHERE id = ?`,
294			size, usage, itemCount, string(flag), id,
295		)
296		s.m.Unlock()
297	}
298	return err
299}
300
301// deleteItemTree removes an item and all its descendants from the database within the given transaction.
302func (s *SqliteStorage) deleteItemTree(tx *sql.Tx, id int64) error {
303	query := `
304	WITH RECURSIVE to_delete(id) AS (
305		SELECT ?
306		UNION ALL
307		SELECT items.id FROM items JOIN to_delete ON items.parent_id = to_delete.id
308	)
309	DELETE FROM items WHERE id IN (SELECT id FROM to_delete)
310	`
311	_, err := tx.Exec(query, id)
312	return err
313}
314
315// RemoveItemAndUpdateStats removes an item and updates all its ancestors' stats in the database
316func (s *SqliteStorage) RemoveItemAndUpdateStats(id, size, usage, itemCount int64) error {
317	s.m.Lock()
318	defer s.m.Unlock()
319
320	tx, err := s.db.Begin()
321	if err != nil {
322		return err
323	}
324
325	// 1. Update all ancestors using recursive CTE
326	updateAncestorsQuery := `
327	WITH RECURSIVE ancestors(id, parent_id) AS (
328		SELECT parent_id, NULL FROM items WHERE id = ?
329		UNION ALL
330		SELECT i.parent_id, NULL FROM items i JOIN ancestors a ON i.id = a.id WHERE i.parent_id IS NOT NULL
331	)
332	UPDATE items
333	SET size = size - ?, usage = usage - ?, item_count = item_count - ?
334	WHERE id IN (SELECT id FROM ancestors)
335	`
336	_, err = tx.Exec(updateAncestorsQuery, id, size, usage, itemCount)
337	if err != nil {
338		rollbackErr := tx.Rollback()
339		if rollbackErr != nil {
340			log.Errorf("failed to rollback transaction: %v", rollbackErr)
341		}
342		return err
343	}
344
345	// 2. Delete the item and its descendants
346	if err = s.deleteItemTree(tx, id); err != nil {
347		rollbackErr := tx.Rollback()
348		if rollbackErr != nil {
349			log.Errorf("failed to rollback transaction: %v", rollbackErr)
350		}
351		return err
352	}
353
354	return tx.Commit()
355}
356
357// GetChildByName returns a child item by its name and parent ID
358func (s *SqliteStorage) GetChildByName(parentID int64, name string) (*SqliteItem, error) {
359	s.m.RLock()
360	defer s.m.RUnlock()
361
362	item := &SqliteItem{storage: s}
363	var pID sql.NullInt64
364	var isDirInt int
365	var mtimeUnix int64
366	var flag string
367
368	err := s.db.QueryRow(
369		`SELECT id, parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag
370		 FROM items WHERE parent_id = ? AND name = ? LIMIT 1`,
371		parentID, name,
372	).Scan(
373		&item.id, &pID, &item.name, &isDirInt,
374		&item.size, &item.usage, &mtimeUnix, &item.itemCount,
375		&item.mli, &flag,
376	)
377	if err != nil {
378		return nil, err
379	}
380
381	if pID.Valid {
382		item.parentID = &pID.Int64
383	}
384	item.isDir = isDirInt == 1
385	item.mtime = time.Unix(mtimeUnix, 0)
386	if flag != "" {
387		item.flag = rune(flag[0])
388	} else {
389		item.flag = ' '
390	}
391
392	return item, nil
393}
394
395// GetChildren returns all children of a given parent ID
396func (s *SqliteStorage) GetChildren(parentID int64) ([]*SqliteItem, error) {
397	s.m.RLock()
398	defer s.m.RUnlock()
399
400	rows, err := s.db.Query(
401		`SELECT id, parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag
402		 FROM items WHERE parent_id = ?`,
403		parentID,
404	)
405	if err != nil {
406		return nil, err
407	}
408	defer rows.Close()
409
410	var items []*SqliteItem
411	for rows.Next() {
412		item := &SqliteItem{storage: s}
413		var parentID sql.NullInt64
414		var isDirInt int
415		var mtimeUnix int64
416		var flag string
417
418		err := rows.Scan(
419			&item.id, &parentID, &item.name, &isDirInt,
420			&item.size, &item.usage, &mtimeUnix, &item.itemCount,
421			&item.mli, &flag,
422		)
423		if err != nil {
424			return nil, err
425		}
426
427		if parentID.Valid {
428			item.parentID = &parentID.Int64
429		}
430		item.isDir = isDirInt == 1
431		item.mtime = time.Unix(mtimeUnix, 0)
432		if flag != "" {
433			item.flag = rune(flag[0])
434		} else {
435			item.flag = ' '
436		}
437		items = append(items, item)
438	}
439
440	return items, rows.Err()
441}
442
443// GetItemByID returns an item by its ID
444func (s *SqliteStorage) GetItemByID(id int64) (*SqliteItem, error) {
445	s.m.RLock()
446	defer s.m.RUnlock()
447
448	item := &SqliteItem{storage: s}
449	var parentID sql.NullInt64
450	var isDirInt int
451	var mtimeUnix int64
452	var flag string
453
454	err := s.db.QueryRow(
455		`SELECT id, parent_id, name, is_dir, size, usage, mtime, item_count, mli, flag
456		 FROM items WHERE id = ?`,
457		id,
458	).Scan(
459		&item.id, &parentID, &item.name, &isDirInt,
460		&item.size, &item.usage, &mtimeUnix, &item.itemCount,
461		&item.mli, &flag,
462	)
463	if err != nil {
464		return nil, err
465	}
466
467	if parentID.Valid {
468		item.parentID = &parentID.Int64
469	}
470	item.isDir = isDirInt == 1
471	item.mtime = time.Unix(mtimeUnix, 0)
472	if flag != "" {
473		item.flag = rune(flag[0])
474	} else {
475		item.flag = ' '
476	}
477
478	return item, nil
479}
480
481// SetMetadata stores a metadata key-value pair
482func (s *SqliteStorage) SetMetadata(key, value string) error {
483	s.m.Lock()
484	defer s.m.Unlock()
485
486	_, err := s.db.Exec(
487		`INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)`,
488		key, value,
489	)
490	return err
491}
492
493// GetMetadata retrieves a metadata value by key
494func (s *SqliteStorage) GetMetadata(key string) (string, error) {
495	s.m.RLock()
496	defer s.m.RUnlock()
497
498	var value string
499	err := s.db.QueryRow(`SELECT value FROM metadata WHERE key = ?`, key).Scan(&value)
500	return value, err
501}
502
503// SqliteItem represents a file or directory stored in SQLite
504type SqliteItem struct {
505	storage   *SqliteStorage
506	id        int64
507	parentID  *int64
508	name      string
509	isDir     bool
510	size      int64
511	usage     int64
512	mtime     time.Time
513	itemCount int64
514	mli       uint64
515	flag      rune
516	parent    fs.Item
517	m         sync.RWMutex
518}
519
520// GetPath returns the full path of the item
521func (i *SqliteItem) GetPath() string {
522	parent := i.GetParent()
523	if parent != nil {
524		return filepath.Join(parent.GetPath(), i.name)
525	}
526	// For root item, get basePath from metadata
527	basePath, err := i.storage.GetMetadata("top_dir_path")
528	if err != nil {
529		return i.name
530	}
531	return filepath.Join(filepath.Dir(basePath), i.name)
532}
533
534// GetName returns the name of the item
535func (i *SqliteItem) GetName() string {
536	return i.name
537}
538
539// GetFlag returns the flag of the item
540func (i *SqliteItem) GetFlag() rune {
541	return i.flag
542}
543
544// IsDir returns true if the item is a directory
545func (i *SqliteItem) IsDir() bool {
546	return i.isDir
547}
548
549// GetSize returns the apparent size
550func (i *SqliteItem) GetSize() int64 {
551	i.m.RLock()
552	defer i.m.RUnlock()
553	return i.size
554}
555
556// GetType returns the type of the item
557func (i *SqliteItem) GetType() string {
558	if i.isDir {
559		return "Directory"
560	}
561	if i.flag == '@' {
562		return "Other"
563	}
564	return "File"
565}
566
567// GetUsage returns the disk usage
568func (i *SqliteItem) GetUsage() int64 {
569	i.m.RLock()
570	defer i.m.RUnlock()
571	return i.usage
572}
573
574// GetMtime returns the modification time
575func (i *SqliteItem) GetMtime() time.Time {
576	i.m.RLock()
577	defer i.m.RUnlock()
578	return i.mtime
579}
580
581// GetItemCount returns the item count
582func (i *SqliteItem) GetItemCount() int64 {
583	i.m.RLock()
584	defer i.m.RUnlock()
585	return i.itemCount
586}
587
588// GetParent returns the parent item
589func (i *SqliteItem) GetParent() fs.Item {
590	i.m.RLock()
591	if i.parent != nil {
592		defer i.m.RUnlock()
593		return i.parent
594	}
595	if i.parentID == nil {
596		defer i.m.RUnlock()
597		return nil
598	}
599	i.m.RUnlock()
600
601	parent, err := i.storage.GetItemByID(*i.parentID)
602	if err != nil {
603		log.Print(err.Error())
604		return nil
605	}
606
607	i.m.Lock()
608	defer i.m.Unlock()
609	if i.parent == nil {
610		i.parent = parent
611	}
612	return i.parent
613}
614
615// SetParent sets the parent item
616func (i *SqliteItem) SetParent(parent fs.Item) {
617	i.parent = parent
618}
619
620// GetParentLocked returns the in-memory parent without hitting the database.
621// Used inside RemoveFile where storage.m is already write-locked.
622func (i *SqliteItem) GetParentLocked() *SqliteItem {
623	if i.parent != nil {
624		return i.parent.(*SqliteItem)
625	}
626	return nil
627}
628
629// GetMultiLinkedInode returns the multi-linked inode number
630func (i *SqliteItem) GetMultiLinkedInode() uint64 {
631	return i.mli
632}
633
634// EncodeJSON encodes the item to JSON
635func (i *SqliteItem) EncodeJSON(writer io.Writer, topLevel bool, attributes fs.JSONAttributes) error {
636	if i.isDir {
637		return i.encodeDirJSON(writer, topLevel, attributes)
638	}
639	return i.encodeFileJSON(writer, attributes)
640}
641
642func (i *SqliteItem) encodeDirJSON(writer io.Writer, topLevel bool, attributes fs.JSONAttributes) error {
643	buff := make([]byte, 0, 128)
644	buff = append(buff, []byte(`[{"name":`)...)
645
646	name := i.GetName()
647	if topLevel {
648		name = i.GetPath()
649	}
650	if err := addSqliteString(&buff, name); err != nil {
651		return err
652	}
653
654	// Directory summary stats (asize, dsize, items) are written by default so
655	// they can be preserved on import. When --output-attrs is supplied, each is
656	// emitted only if explicitly selected.
657	if attributes.Includes("asize") {
658		buff = append(buff, []byte(`,"asize":`)...)
659		buff = append(buff, []byte(strconv.FormatInt(i.GetSize(), 10))...)
660	}
661	if attributes.Includes("dsize") {
662		buff = append(buff, []byte(`,"dsize":`)...)
663		buff = append(buff, []byte(strconv.FormatInt(i.GetUsage(), 10))...)
664	}
665	if attributes.Includes("items") {
666		buff = append(buff, []byte(`,"items":`)...)
667		buff = append(buff, []byte(strconv.FormatInt(i.GetItemCount(), 10))...)
668	}
669	if attributes.Includes("mtime") && !i.GetMtime().IsZero() {
670		buff = append(buff, []byte(`,"mtime":`)...)
671		buff = append(buff, []byte(strconv.FormatInt(i.GetMtime().Unix(), 10))...)
672	}
673
674	buff = append(buff, '}')
675
676	children, err := i.storage.GetChildren(i.id)
677	if err != nil {
678		return err
679	}
680
681	if len(children) > 0 {
682		buff = append(buff, ',')
683	}
684	buff = append(buff, '\n')
685
686	if _, err := writer.Write(buff); err != nil {
687		return err
688	}
689
690	for idx, child := range children {
691		if idx > 0 {
692			if _, err := writer.Write([]byte(",\n")); err != nil {
693				return err
694			}
695		}
696		child.parent = i
697		if err := child.EncodeJSON(writer, false, attributes); err != nil {
698			return err
699		}
700	}
701
702	if _, err := writer.Write([]byte("]")); err != nil {
703		return err
704	}
705	return nil
706}
707
708func (i *SqliteItem) encodeFileJSON(writer io.Writer, attributes fs.JSONAttributes) error {
709	buff := make([]byte, 0, 128)
710	buff = append(buff, []byte(`{"name":`)...)
711	if err := addSqliteString(&buff, i.GetName()); err != nil {
712		return err
713	}
714	if attributes.Includes("asize") && i.GetSize() > 0 {
715		buff = append(buff, []byte(`,"asize":`)...)
716		buff = append(buff, []byte(strconv.FormatInt(i.GetSize(), 10))...)
717	}
718	if attributes.Includes("dsize") && i.GetUsage() > 0 {
719		buff = append(buff, []byte(`,"dsize":`)...)
720		buff = append(buff, []byte(strconv.FormatInt(i.GetUsage(), 10))...)
721	}
722	if attributes.Includes("mtime") && !i.GetMtime().IsZero() {
723		buff = append(buff, []byte(`,"mtime":`)...)
724		buff = append(buff, []byte(strconv.FormatInt(i.GetMtime().Unix(), 10))...)
725	}
726
727	if attributes.Includes("notreg") && i.flag == '@' {
728		buff = append(buff, []byte(`,"notreg":true`)...)
729	}
730	if attributes == nil && i.flag == 'H' {
731		buff = append(buff, []byte(`,"ino":`+strconv.FormatUint(i.mli, 10)+`,"hlnkc":true`)...)
732	}
733
734	buff = append(buff, '}')
735
736	if _, err := writer.Write(buff); err != nil {
737		return err
738	}
739	return nil
740}
741
742func addSqliteString(buff *[]byte, val string) error {
743	b, err := json.Marshal(val)
744	if err != nil {
745		return err
746	}
747	*buff = append(*buff, b...)
748	return nil
749}
750
751// GetItemStats returns item statistics - hard links already handled during scan
752func (i *SqliteItem) GetItemStats(linkedItems fs.HardLinkedItems, filteringFiles bool) (itemCount, size, usage int64) {
753	return i.itemCount, i.size, i.usage
754}
755
756// UpdateStats is a no-op for SqliteItem - hard links are handled during scan
757func (i *SqliteItem) UpdateStats(linkedItems fs.HardLinkedItems)                  {}
758func (i *SqliteItem) UpdateStatsWithFileFiltering(linkedItems fs.HardLinkedItems) {}
759
760// AddFile adds a child file (no-op for SQLite items - children are in DB)
761func (i *SqliteItem) AddFile(item fs.Item) {
762	// Children are stored in database via parent_id relationship
763}
764
765// GetFiles returns children as a sorted iterator
766func (i *SqliteItem) GetFiles(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
767	return func(yield func(fs.Item) bool) {
768		children, err := i.storage.GetChildren(i.id)
769		if err != nil {
770			log.Print(err.Error())
771			return
772		}
773
774		// Convert to fs.Files for sorting
775		files := make(fs.Files, len(children))
776		for idx, child := range children {
777			child.parent = i
778			files[idx] = child
779		}
780
781		sortFiles(files, sortBy, order)
782
783		for _, item := range files {
784			if !yield(item) {
785				return
786			}
787		}
788	}
789}
790
791// GetFilesLocked returns children with locking
792func (i *SqliteItem) GetFilesLocked(sortBy fs.SortBy, order fs.SortOrder) iter.Seq[fs.Item] {
793	return i.GetFiles(sortBy, order)
794}
795
796// RemoveFile removes a child file, updating both in-memory stats and the database in one pass.
797func (i *SqliteItem) RemoveFile(item fs.Item) {
798	sqliteItem := item.(*SqliteItem)
799	size := item.GetSize()
800	usage := item.GetUsage()
801	itemCount := item.GetItemCount()
802
803	// Apply in-memory stats updates for loaded ancestors first,
804	// to avoid double-counting if ancestors are loaded from DB after it's updated.
805	cur := i
806	for {
807		cur.m.Lock()
808		cur.size -= size
809		cur.usage -= usage
810		cur.itemCount -= itemCount
811		cur.m.Unlock()
812
813		parent := cur.GetParentLocked()
814		if parent == nil {
815			break
816		}
817		cur = parent
818	}
819
820	if err := i.storage.RemoveItemAndUpdateStats(sqliteItem.id, size, usage, itemCount); err != nil {
821		log.Errorf("Error removing item and updating stats: %v", err)
822		return
823	}
824}
825
826// RemoveFileByName removes a child by name
827func (i *SqliteItem) RemoveFileByName(name string) {
828	child, err := i.storage.GetChildByName(i.id, name)
829	if err != nil {
830		log.Errorf("Error getting child from database: %v", err)
831		return
832	}
833
834	i.RemoveFile(child)
835}
836
837// RLock returns a no-op unlock function
838func (i *SqliteItem) RLock() func() {
839	i.m.RLock()
840	return i.m.RUnlock
841}
842
843// SqliteAnalyzer implements Analyzer using SQLite storage
844type SqliteAnalyzer struct {
845	BaseAnalyzer
846	storage   *SqliteStorage
847	dbWriteMu sync.Mutex
848}
849
850// insertItemLocked is a serialized wrapper around storage.InsertItem.
851// All concurrent goroutines spawned during AnalyzeDir must use this helper
852// because the underlying *sql.Tx and prepared statements are not safe for
853// concurrent use.
854func (a *SqliteAnalyzer) insertItemLocked(
855	parentID *int64, name string, isDir bool, size, usage int64, mtime time.Time,
856	itemCount int64, mli uint64, flag rune,
857) (int64, error) {
858	a.dbWriteMu.Lock()
859	defer a.dbWriteMu.Unlock()
860	return a.storage.InsertItem(parentID, name, isDir, size, usage, mtime, itemCount, mli, flag)
861}
862
863func (a *SqliteAnalyzer) updateDirLocked(id, size, usage, itemCount int64, flag rune) error {
864	a.dbWriteMu.Lock()
865	defer a.dbWriteMu.Unlock()
866	return a.storage.UpdateItem(id, size, usage, itemCount, flag)
867}
868
869// hasInodeLocked is a serialized wrapper around storage.HasInode.
870func (a *SqliteAnalyzer) hasInodeLocked(mli uint64) bool {
871	a.dbWriteMu.Lock()
872	defer a.dbWriteMu.Unlock()
873	return a.storage.HasInode(mli)
874}
875
876// CreateSqliteAnalyzer creates a new SQLite analyzer
877func CreateSqliteAnalyzer(dbPath string) (*SqliteAnalyzer, error) {
878	if err := checkAvailable(); err != nil {
879		return nil, err
880	}
881
882	storage, err := NewSqliteStorage(dbPath)
883	if err != nil {
884		return nil, err
885	}
886
887	a := &SqliteAnalyzer{
888		storage: storage,
889	}
890	a.Init()
891	return a, nil
892}
893
894// AnalyzeDir analyzes the given path and stores results in SQLite.
895// If the database already contains data, it loads from the database instead of re-scanning.
896func (a *SqliteAnalyzer) AnalyzeDir(
897	path string, ignore common.ShouldDirBeIgnored, fileTypeFilter common.ShouldFileBeIgnored,
898) fs.Item {
899	// Check if database already has data
900	if a.storage.HasData() {
901		log.Printf("Loading analysis from existing SQLite database")
902		rootItem, err := a.storage.GetRootItem()
903		if err != nil {
904			log.Printf("Error loading from database, will re-scan: %v", err)
905		} else {
906			// Signal that we're done immediately
907			a.doneChan.Broadcast()
908			return rootItem
909		}
910	}
911
912	a.ignoreDir = ignore
913	a.ignoreFileType = fileTypeFilter
914
915	// Clear existing data and store metadata
916	err := a.storage.ClearItems()
917	if err != nil {
918		log.Printf("Error clearing items: %v", err)
919	}
920	err = a.storage.SetMetadata("top_dir_path", path)
921	if err != nil {
922		log.Printf("Error setting metadata: %v", err)
923	}
924
925	// Start bulk insert transaction
926	if err := a.storage.BeginBulkInsert(); err != nil {
927		log.Printf("Error starting bulk insert: %v", err)
928	}
929
930	go a.UpdateProgress()
931
932	// Process directory and get the root item
933	rootItem := a.processDir(path, nil)
934
935	a.wait.Wait()
936
937	// Commit bulk insert transaction
938	if err := a.storage.EndBulkInsert(); err != nil {
939		log.Printf("Error committing bulk insert: %v", err)
940	}
941
942	a.progressDoneChan <- struct{}{}
943	a.doneChan.Broadcast()
944
945	return rootItem
946}
947
948// fileStat holds stats for a single file entry computed by processFile.
949type fileStat struct {
950	size      int64
951	usage     int64
952	itemCount int64
953	mli       uint64
954	flag      rune
955	// archiveDir is non-nil when the file is an archive that was expanded.
956	archiveDir *Dir
957}
958
959// processArchiveEntry tries to expand name/entryPath as a zip or tar archive.
960// Returns the archive *Dir on success, or nil on failure (in which case err is set).
961func (a *SqliteAnalyzer) processArchiveEntry(entryPath, name string, info os.FileInfo) (*Dir, error) {
962	if isZipFile(name) {
963		archiveDirZip, errZip := processZipFile(entryPath, info)
964		if errZip != nil {
965			return nil, errZip
966		}
967		uncompressedSize, compressedSize, errSize := getZipFileSize(entryPath)
968		if errSize == nil {
969			archiveDirZip.Size = uncompressedSize
970			archiveDirZip.Usage = compressedSize
971		}
972		return archiveDirZip.Dir, nil
973	}
974	archiveDirTar, errTar := processTarFile(entryPath, info)
975	if errTar != nil {
976		return nil, errTar
977	}
978	return archiveDirTar.Dir, nil
979}
980
981// processFile resolves a single non-directory entry and returns its stats.
982// It returns nil when the file should be skipped entirely.
983func (a *SqliteAnalyzer) processFile(entryPath, name string, f os.DirEntry) (stat fileStat) {
984	if a.ignoreFileType != nil && a.ignoreFileType(name) {
985		return stat
986	}
987
988	info, err := f.Info()
989	if err != nil {
990		log.Print(err.Error())
991		return stat
992	}
993
994	if a.followSymlinks && info.Mode()&os.ModeSymlink != 0 {
995		infoF, err := followSymlink(entryPath, a.gitAnnexedSize)
996		if err != nil {
997			log.Print(err.Error())
998			return stat
999		}
1000		if infoF != nil {
1001			info = infoF
1002		}
1003	}
1004
1005	if a.matchesTimeFilterFn != nil && !a.matchesTimeFilterFn(info.ModTime()) {
1006		return stat
1007	}
1008
1009	if a.archiveBrowsing && (isZipFile(name) || isTarFile(name)) {
1010		archiveDir, err := a.processArchiveEntry(entryPath, name, info)
1011		if err != nil {
1012			log.Printf("Failed to process archive %s: %v", entryPath, err)
1013		} else {
1014			return fileStat{
1015				size:       archiveDir.Size,
1016				usage:      archiveDir.Usage,
1017				itemCount:  archiveDir.ItemCount,
1018				flag:       archiveDir.Flag,
1019				archiveDir: archiveDir,
1020			}
1021		}
1022	}
1023
1024	fileSize := info.Size()
1025	fileUsage, fileMli := getSyscallStats(info)
1026	fileFlag := getFlag(info)
1027
1028	if fileMli != 0 && a.hasInodeLocked(fileMli) {
1029		fileSize = 0
1030		fileUsage = 0
1031		fileFlag = 'H'
1032	}
1033
1034	return fileStat{
1035		size:  fileSize,
1036		usage: fileUsage,
1037		mli:   fileMli,
1038		flag:  fileFlag,
1039	}
1040}
1041
1042// nolint:funlen
1043func (a *SqliteAnalyzer) processDir(path string, parentID *int64) *SqliteItem {
1044	// Start with 4096 for directory's own size/usage, matching Dir.UpdateStats behavior
1045	var (
1046		totalSize  int64 = 4096
1047		totalUsage int64 = 4096
1048		filesSize  int64 // only files in this directory, for progress reporting
1049		itemCount  int64 = 1
1050		subDirChan       = make(chan *SqliteItem)
1051		dirCount   int
1052	)
1053
1054	a.wait.Add(1)
1055	defer a.wait.Done()
1056
1057	// Hold a concurrency slot only during the scan/insert phase. We must
1058	// release it before draining subDirChan, otherwise child goroutines
1059	// (which need to acquire a slot themselves) would deadlock once the
1060	// global concurrencyLimit is saturated by ancestors waiting on their
1061	// own children.
1062	concurrencyLimit <- struct{}{}
1063	slotReleased := false
1064	releaseSlot := func() {
1065		if !slotReleased {
1066			<-concurrencyLimit
1067			slotReleased = true
1068		}
1069	}
1070	defer releaseSlot()
1071
1072	if parentID != nil && a.IsCancelled() {
1073		return nil
1074	}
1075
1076	files, err := os.ReadDir(path)
1077	if err != nil {
1078		log.Print(err.Error())
1079	}
1080
1081	// Get directory info for mtime
1082	dirInfo, statErr := os.Stat(path)
1083	var dirMtime time.Time
1084	if statErr == nil {
1085		dirMtime = dirInfo.ModTime()
1086	}
1087
1088	dirFlag := getDirFlag(err, len(files))
1089
1090	// Insert directory into database (size/usage will be updated later)
1091	dirID, err := a.insertItemLocked(
1092		parentID,
1093		filepath.Base(path),
1094		true,
1095		0, // size will be updated later
1096		0, // usage will be updated later
1097		dirMtime,
1098		1, // item_count will be updated later
1099		0,
1100		dirFlag,
1101	)
1102	if err != nil {
1103		log.Print(err.Error())
1104		return nil
1105	}
1106
1107	// Spawn subdirectory scans in parallel; each goroutine fully completes its
1108	// subtree (including DB row finalization) before sending the result back.
1109	for _, f := range files {
1110		if a.IsCancelled() {
1111			break
1112		}
1113		name := f.Name()
1114		entryPath := filepath.Join(path, name)
1115
1116		if f.IsDir() {
1117			if a.shouldSkipDir(name, entryPath) {
1118				continue
1119			}
1120			dirCount++
1121
1122			go func(entryPath string) {
1123				sub := a.processDir(entryPath, &dirID)
1124				subDirChan <- sub
1125			}(entryPath)
1126			continue
1127		}
1128
1129		info, err := f.Info()
1130		if err != nil {
1131			log.Print(err.Error())
1132			continue
1133		}
1134
1135		stat := a.processFile(entryPath, name, f)
1136		if stat == (fileStat{}) {
1137			continue
1138		}
1139
1140		if stat.archiveDir != nil {
1141			archiveID, err := a.insertItemLocked(
1142				&dirID,
1143				name,
1144				true,
1145				stat.archiveDir.Size,
1146				stat.archiveDir.Usage,
1147				info.ModTime(),
1148				stat.archiveDir.ItemCount,
1149				0,
1150				stat.archiveDir.Flag,
1151			)
1152			if err != nil {
1153				log.Print(err.Error())
1154				continue
1155			}
1156			a.persistArchive(stat.archiveDir, archiveID)
1157
1158			totalSize += stat.size
1159			totalUsage += stat.usage
1160			filesSize += stat.usage
1161			itemCount += stat.itemCount
1162			continue
1163		}
1164
1165		_, err = a.insertItemLocked(
1166			&dirID,
1167			name,
1168			false,
1169			stat.size,
1170			stat.usage,
1171			info.ModTime(),
1172			1,
1173			stat.mli,
1174			stat.flag,
1175		)
1176		if err != nil {
1177			log.Print(err.Error())
1178			continue
1179		}
1180
1181		totalSize += stat.size
1182		totalUsage += stat.usage
1183		filesSize += stat.usage
1184		itemCount++
1185	}
1186
1187	// Release the concurrency slot before draining subDirChan so children
1188	// blocked on concurrencyLimit can make progress.
1189	releaseSlot()
1190
1191	// Aggregate subdirectory results. Each sub is fully finalized when received.
1192	for i := 0; i < dirCount; i++ {
1193		sub := <-subDirChan
1194		if sub != nil {
1195			totalSize += sub.size
1196			totalUsage += sub.usage
1197			itemCount += sub.itemCount
1198			switch sub.flag {
1199			case '!', '.':
1200				if dirFlag != '!' {
1201					dirFlag = '.'
1202				}
1203			}
1204		}
1205	}
1206
1207	// Update directory with computed stats
1208	if err := a.updateDirLocked(dirID, totalSize, totalUsage, itemCount, dirFlag); err != nil {
1209		log.Printf("Error updating item: %v", err)
1210	}
1211
1212	// Report progress (only files in this dir, subdirs already reported themselves)
1213	a.progressCurrentItemName.Store(path)
1214	a.progressItemCount.Add(int64(len(files)))
1215	a.progressTotalUsage.Add(filesSize)
1216
1217	// Return SqliteItem for the directory
1218	return &SqliteItem{
1219		storage:   a.storage,
1220		id:        dirID,
1221		parentID:  parentID,
1222		name:      filepath.Base(path),
1223		isDir:     true,
1224		size:      totalSize,
1225		usage:     totalUsage,
1226		mtime:     dirMtime,
1227		itemCount: itemCount,
1228		flag:      dirFlag,
1229	}
1230}
1231
1232func (a *SqliteAnalyzer) persistArchive(archiveDir *Dir, parentID int64) {
1233	if archiveDir == nil {
1234		return
1235	}
1236	for _, f := range archiveDir.Files {
1237		if a.IsCancelled() {
1238			return
1239		}
1240		if f.IsDir() {
1241			var subDir *Dir
1242			switch v := f.(type) {
1243			case *ZipDir:
1244				subDir = v.Dir
1245			case *TarDir:
1246				subDir = v.Dir
1247			}
1248
1249			if subDir == nil {
1250				continue
1251			}
1252
1253			id, err := a.insertItemLocked(
1254				&parentID,
1255				f.GetName(),
1256				true,
1257				f.GetSize(),
1258				f.GetUsage(),
1259				f.GetMtime(),
1260				f.GetItemCount(),
1261				0,
1262				f.GetFlag(),
1263			)
1264			if err != nil {
1265				log.Print(err.Error())
1266				continue
1267			}
1268			a.persistArchive(subDir, id)
1269		} else {
1270			_, err := a.insertItemLocked(
1271				&parentID,
1272				f.GetName(),
1273				false,
1274				f.GetSize(),
1275				f.GetUsage(),
1276				f.GetMtime(),
1277				1,
1278				f.GetMultiLinkedInode(),
1279				f.GetFlag(),
1280			)
1281			if err != nil {
1282				log.Print(err.Error())
1283				continue
1284			}
1285		}
1286	}
1287}