fork/gdu

git clone https://git.julia.blue/fork/gdu

Daniel Mildefix: StoreDir fixes (#644)35cba00

master
4.1 KiB190 linesraw
1package analyze
2
3import (
4	"bytes"
5	"encoding/gob"
6	"path/filepath"
7	"sync"
8	"sync/atomic"
9
10	"github.com/dgraph-io/badger/v4"
11	"github.com/dundee/gdu/v5/internal/common"
12	"github.com/dundee/gdu/v5/pkg/fs"
13	"github.com/pkg/errors"
14)
15
16func init() {
17	gob.RegisterName("analyze.StoredDir", &StoredDir{})
18	gob.RegisterName("analyze.Dir", &Dir{})
19	gob.RegisterName("analyze.File", &File{})
20	gob.RegisterName("analyze.ParentDir", &ParentDir{})
21}
22
23// DefaultStorage is a default instance of badger storage
24var DefaultStorage *Storage
25
26// Storage represents a badger storage
27//
28// All access to db is serialized by m. Open is reference counted: badger takes
29// an exclusive lock on the storage directory, so a second concurrent
30// badger.Open on the same path would panic, and an inner caller's closer must
31// not tear the database out from under an outer caller that is still using it.
32type Storage struct {
33	db          *badger.DB
34	storagePath string
35	topDir      string
36	m           sync.RWMutex
37	counter     atomic.Int64
38	openCount   int
39}
40
41// NewStorage returns new instance of badger storage
42func NewStorage(storagePath, topDir string) *Storage {
43	st := &Storage{
44		storagePath: storagePath,
45		topDir:      topDir,
46	}
47	DefaultStorage = st
48	return st
49}
50
51// GetTopDir returns top directory
52func (s *Storage) GetTopDir() string {
53	return s.topDir
54}
55
56// IsOpen returns true if badger DB is open
57func (s *Storage) IsOpen() bool {
58	s.m.RLock()
59	defer s.m.RUnlock()
60	return s.db != nil
61}
62
63// Open opens badger DB and returns a function closing it again.
64//
65// Opening is reference counted, so nested and concurrent callers all get a
66// usable database and only the last returned closer actually closes it. Each
67// closer is idempotent; calling it more than once is a no-op.
68func (s *Storage) Open() func() {
69	s.m.Lock()
70	defer s.m.Unlock()
71
72	if s.db == nil {
73		s.openLocked()
74	}
75	s.openCount++
76
77	var once sync.Once
78	return func() {
79		once.Do(s.release)
80	}
81}
82
83// release drops one reference taken by Open, closing the database once the
84// last one is gone.
85func (s *Storage) release() {
86	s.m.Lock()
87	defer s.m.Unlock()
88
89	if s.openCount == 0 {
90		return
91	}
92	s.openCount--
93	if s.openCount > 0 || s.db == nil {
94		return
95	}
96	s.db.Close()
97	s.db = nil
98}
99
100// openLocked opens the badger DB. The caller must hold m for writing.
101func (s *Storage) openLocked() {
102	options := badger.DefaultOptions(s.storagePath)
103	options.Logger = nil
104
105	if !common.Is64Bit {
106		// For 32-bit systems, we need to set ValueLogFileSize to a smaller value to
107		// avoid "cannot allocate memory while mmapping" error
108		options.ValueLogFileSize = (1<<30 - 1) / 2
109	}
110
111	db, err := badger.Open(options)
112	if err != nil {
113		panic(err)
114	}
115	s.db = db
116}
117
118// StoreDir saves item info into badger DB
119func (s *Storage) StoreDir(dir fs.Item) error {
120	s.checkCount()
121	s.m.RLock()
122	defer s.m.RUnlock()
123
124	return s.db.Update(func(txn *badger.Txn) error {
125		b := &bytes.Buffer{}
126		enc := gob.NewEncoder(b)
127		err := enc.Encode(dir)
128		if err != nil {
129			return errors.Wrap(err, "encoding dir value")
130		}
131
132		return txn.Set([]byte(dir.GetPath()), b.Bytes())
133	})
134}
135
136// LoadDir saves item info into badger DB
137func (s *Storage) LoadDir(dir fs.Item) error {
138	s.checkCount()
139	s.m.RLock()
140	defer s.m.RUnlock()
141
142	return s.db.View(func(txn *badger.Txn) error {
143		path := dir.GetPath()
144		item, err := txn.Get([]byte(path))
145		if err != nil {
146			return errors.Wrap(err, "reading stored value for path: "+path)
147		}
148		return item.Value(func(val []byte) error {
149			b := bytes.NewBuffer(val)
150			dec := gob.NewDecoder(b)
151			return dec.Decode(dir)
152		})
153	})
154}
155
156// GetDirForPath returns Dir for given path
157func (s *Storage) GetDirForPath(path string) (item fs.Item, err error) {
158	dirPath := filepath.Dir(path)
159	name := filepath.Base(path)
160	dir := &StoredDir{
161		Dir: &Dir{
162			File: &File{
163				Name: name,
164			},
165			BasePath: dirPath,
166		},
167	}
168	err = s.LoadDir(dir)
169	if err != nil {
170		return nil, err
171	}
172	return dir, nil
173}
174
175func (s *Storage) checkCount() {
176	s.counter.Add(1)
177	if s.counter.Load() >= 10000 {
178		s.m.Lock()
179		defer s.m.Unlock()
180		if s.counter.Load() < 10000 {
181			// another goroutine already recycled the database
182			return
183		}
184		s.counter.Store(0)
185		if s.db != nil {
186			s.db.Close()
187		}
188		s.openLocked()
189	}
190}