1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
|
// This file is part of gnunet-go, a GNUnet-implementation in Golang.
// Copyright (C) 2019-2022 Bernd Fix >Y<
//
// gnunet-go is free software: you can redistribute it and/or modify it
// under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License,
// or (at your option) any later version.
//
// gnunet-go is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// SPDX-License-Identifier: AGPL3.0-or-later
package store
import (
"database/sql"
_ "embed"
"gnunet/crypto"
"gnunet/enums"
"gnunet/service/dht/blocks"
"gnunet/util"
"os"
)
//============================================================
// Metadata handling for file storage
//============================================================
// FileMetadata holds information about a file (raw block data)
// and is stored in a SQL database for faster access.
type FileMetadata struct {
key *crypto.HashCode // storage key
size uint64 // size of file
btype enums.BlockType // block type
bhash *crypto.HashCode // block hash
stored util.AbsoluteTime // time added to store
expires util.AbsoluteTime // expiration time
lastUsed util.AbsoluteTime // time last used
usedCount uint64 // usage count
}
// NewFileMetadata creates a new file metadata instance
func NewFileMetadata() *FileMetadata {
return &FileMetadata{
key: crypto.NewHashCode(nil),
bhash: crypto.NewHashCode(nil),
}
}
//------------------------------------------------------------
// Metadata database: A SQLite3 database to hold metadata about
// blocks in file storage
//------------------------------------------------------------
//go:embed store_dht_meta.sql
var initScript []byte
// FileMetaDB is a SQLite3 database for block metadata
type FileMetaDB struct {
conn *DBConn // database connection
}
// OpenMetaDB opens a metadata database in the given path. The name of the
// database is "access.db".
func OpenMetaDB(path string) (db *FileMetaDB, err error) {
// connect to database
dbFile := path + "/access.db"
if _, err = os.Stat(path + "/access.db"); err != nil {
var file *os.File
if file, err = os.Create(dbFile); err != nil {
return
}
file.Close()
}
db = new(FileMetaDB)
if db.conn, err = DBPool.Connect("sqlite3:" + dbFile); err != nil {
return
}
// check for initialized database
res := db.conn.QueryRow("select name from sqlite_master where type='table' and name='meta'")
var s string
if res.Scan(&s) != nil {
// initialize database
if _, err = db.conn.Exec(string(initScript)); err != nil {
return
}
}
return
}
// Store metadata in database: creates or updates a record for the metadata
// in the database; primary key is the query key
func (db *FileMetaDB) Store(md *FileMetadata) (err error) {
// work around a SQLite3 bug when storing uint64 with high bit set
var exp *uint64
if !md.expires.IsNever() {
exp = new(uint64)
*exp = md.expires.Val
}
sql := "replace into meta(qkey,btype,bhash,size,stored,expires,lastUsed,usedCount) values(?,?,?,?,?,?,?,?)"
_, err = db.conn.Exec(sql,
md.key.Data, md.btype, md.bhash.Data, md.size, md.stored.Epoch(),
exp, md.lastUsed.Epoch(), md.usedCount)
return
}
// Get block metadata from database
func (db *FileMetaDB) Get(query blocks.Query) (mds []*FileMetadata, err error) {
// select rows in database matching the query
stmt := "select size,bhash,stored,expires,lastUsed,usedCount from meta where qkey=?"
btype := query.Type()
var rows *sql.Rows
if btype == enums.BLOCK_TYPE_ANY {
rows, err = db.conn.Query(stmt, query.Key().Data)
} else {
rows, err = db.conn.Query(stmt+" and btype=?", query.Key().Data, btype)
}
if err != nil {
return
}
// process results
for rows.Next() {
md := NewFileMetadata()
md.key = query.Key()
md.btype = btype
var st, lu uint64
var exp *uint64
if err = rows.Scan(&md.size, &md.bhash.Data, &st, &exp, &lu, &md.usedCount); err != nil {
if err == sql.ErrNoRows {
md = nil
err = nil
}
return
}
if exp != nil {
md.expires.Val = *exp
} else {
md.expires = util.AbsoluteTimeNever()
}
md.stored.Val = st * 1000000
md.lastUsed.Val = lu * 1000000
mds = append(mds, md)
}
return
}
// Drop metadata for block from database
func (db *FileMetaDB) Drop(key []byte, btype enums.BlockType) (err error) {
if btype != enums.BLOCK_TYPE_ANY {
_, err = db.conn.Exec("delete from meta where qkey=? and btype=?", key, btype)
} else {
_, err = db.conn.Exec("delete from meta where qkey=?", key)
}
return
}
// Used a block from store: increment usage count and lastUsed time.
func (db *FileMetaDB) Used(key []byte, btype enums.BlockType) (err error) {
stmt := "update meta set usedCount=usedCount+1,lastUsed=unixepoch() where qkey=?"
if btype != enums.BLOCK_TYPE_ANY {
_, err = db.conn.Exec(stmt+" and btype=?", key, btype)
} else {
_, err = db.conn.Exec(stmt, key)
}
return
}
// Obsolete collects records from the meta database that are considered
// "removable". Entries are rated by the value of "(lifetime * size) / usedCount"
func (db *FileMetaDB) Obsolete(n int) (removable []*FileMetadata, err error) {
// get obsolete records from database
rate := "(unixepoch()-unixepoch(stored))*size/usedCount"
stmt := "select qkey,btype from meta order by " + rate + " limit ?"
var rows *sql.Rows
if rows, err = db.conn.Query(stmt, n); err != nil {
return
}
var md *FileMetadata
for rows.Next() {
var st, lu uint64
if err = rows.Scan(&md.key, &md.btype, &md.size, &st, &md.expires.Val, &lu, &md.usedCount); err != nil {
return
}
md.stored.Val = st * 1000000
md.lastUsed.Val = lu * 1000000
removable = append(removable, md)
}
return
}
// Traverse metadata records and call function on each record
func (db *FileMetaDB) Traverse(f func(*FileMetadata)) error {
sql := "select qkey,btype,bhash,size,stored,expires,lastUsed,usedCount from meta"
rows, err := db.conn.Query(sql)
if err != nil {
return err
}
md := NewFileMetadata()
for rows.Next() {
var st, lu uint64
var exp *uint64
err = rows.Scan(&md.key.Data, &md.btype, &md.bhash.Data, &md.size, &st, &exp, &lu, &md.usedCount)
if err != nil {
return err
}
if exp != nil {
md.expires.Val = *exp
} else {
md.expires = util.AbsoluteTimeNever()
}
md.stored.Val = st * 1000000
md.lastUsed.Val = lu * 1000000
// call process function
f(md)
}
return nil
}
// Close metadata database
func (db *FileMetaDB) Close() error {
return db.conn.Close()
}
|
