Page snapshot browser data
This commit is contained in:
Mikei386
+189
-20
@@ -10,6 +10,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
@@ -32,17 +33,49 @@ type SecretStore interface {
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
store *store.Store
|
||||
secrets SecretStore
|
||||
restic *restic.Runner
|
||||
rsync *rsync.Runner
|
||||
mounts *platform.MountManager
|
||||
workloads *platform.WorkloadManager
|
||||
notifier *notify.Sender
|
||||
log *slog.Logger
|
||||
queue *queue.Queue
|
||||
mu sync.RWMutex
|
||||
config model.Config
|
||||
store *store.Store
|
||||
secrets SecretStore
|
||||
restic *restic.Runner
|
||||
rsync *rsync.Runner
|
||||
mounts *platform.MountManager
|
||||
workloads *platform.WorkloadManager
|
||||
notifier *notify.Sender
|
||||
log *slog.Logger
|
||||
queue *queue.Queue
|
||||
mu sync.RWMutex
|
||||
config model.Config
|
||||
snapshotMu sync.Mutex
|
||||
snapshotTree map[string]snapshotTreeCache
|
||||
}
|
||||
|
||||
type SnapshotBrowserNode struct {
|
||||
Path string `json:"path"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
HasChildren bool `json:"hasChildren"`
|
||||
}
|
||||
|
||||
type SnapshotBrowserPage struct {
|
||||
Path string `json:"path"`
|
||||
Items []SnapshotBrowserNode `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Cached bool `json:"cached"`
|
||||
Generated time.Time `json:"generated"`
|
||||
}
|
||||
|
||||
type snapshotTreeCache struct {
|
||||
generated time.Time
|
||||
total int
|
||||
children map[string][]SnapshotBrowserNode
|
||||
}
|
||||
|
||||
type snapshotTreeNode struct {
|
||||
path string
|
||||
name string
|
||||
nodeType string
|
||||
size int64
|
||||
children map[string]*snapshotTreeNode
|
||||
}
|
||||
|
||||
func New(st *store.Store, secrets SecretStore, rr *restic.Runner, rs *rsync.Runner, mounts *platform.MountManager, workloads *platform.WorkloadManager, notifier *notify.Sender, log *slog.Logger) (*Service, error) {
|
||||
@@ -384,26 +417,162 @@ func (s *Service) RepositoryStats(ctx context.Context) []model.RepositoryStats {
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *Service) SnapshotFiles(ctx context.Context, repoID, snapshot, path string) ([]map[string]any, error) {
|
||||
func (s *Service) SnapshotFiles(ctx context.Context, repoID, snapshot, path string) (SnapshotBrowserPage, error) {
|
||||
started := time.Now()
|
||||
s.log.Info("snapshot file listing started", "repoID", repoID, "snapshot", snapshot, "path", path)
|
||||
repo, ok := s.repository(repoID)
|
||||
if !ok {
|
||||
return nil, errors.New("validation: unknown repository")
|
||||
return SnapshotBrowserPage{}, errors.New("validation: unknown repository")
|
||||
}
|
||||
|
||||
normalizedPath := normalizeSnapshotPath(path)
|
||||
key := snapshotCacheKey(repoID, snapshot)
|
||||
if page, ok := s.snapshotBrowserPage(key, normalizedPath); ok {
|
||||
s.log.Info("snapshot file listing loaded from cache", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "items", len(page.Items), "total", page.Total, "durationMs", time.Since(started).Milliseconds())
|
||||
return page, nil
|
||||
}
|
||||
|
||||
mounted, err := s.mounts.Prepare(ctx, repo)
|
||||
if err != nil {
|
||||
s.log.Warn("snapshot file listing failed", "repoID", repoID, "snapshot", snapshot, "path", path, "durationMs", time.Since(started).Milliseconds(), "error", err)
|
||||
return nil, err
|
||||
s.log.Warn("snapshot file listing failed", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "durationMs", time.Since(started).Milliseconds(), "error", err)
|
||||
return SnapshotBrowserPage{}, err
|
||||
}
|
||||
defer s.mounts.Cleanup(context.Background(), mounted)
|
||||
items, err := s.restic.List(ctx, mounted.Repository, snapshot, path)
|
||||
|
||||
items, err := s.restic.List(ctx, mounted.Repository, snapshot, "")
|
||||
if err != nil {
|
||||
s.log.Warn("snapshot file listing failed", "repoID", repoID, "snapshot", snapshot, "path", path, "durationMs", time.Since(started).Milliseconds(), "error", err)
|
||||
return nil, err
|
||||
s.log.Warn("snapshot file listing failed", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "durationMs", time.Since(started).Milliseconds(), "error", err)
|
||||
return SnapshotBrowserPage{}, err
|
||||
}
|
||||
s.log.Info("snapshot file listing loaded", "repoID", repoID, "snapshot", snapshot, "path", path, "items", len(items), "durationMs", time.Since(started).Milliseconds())
|
||||
return items, nil
|
||||
cache := buildSnapshotTreeCache(items)
|
||||
s.storeSnapshotTree(key, cache)
|
||||
page, _ := s.snapshotBrowserPage(key, normalizedPath)
|
||||
s.log.Info("snapshot file listing loaded", "repoID", repoID, "snapshot", snapshot, "path", normalizedPath, "items", len(page.Items), "total", cache.total, "durationMs", time.Since(started).Milliseconds())
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func snapshotCacheKey(repoID, snapshot string) string {
|
||||
return repoID + "\x00" + snapshot
|
||||
}
|
||||
|
||||
func (s *Service) snapshotBrowserPage(key, path string) (SnapshotBrowserPage, bool) {
|
||||
s.snapshotMu.Lock()
|
||||
defer s.snapshotMu.Unlock()
|
||||
cache, ok := s.snapshotTree[key]
|
||||
if !ok || time.Since(cache.generated) > 30*time.Minute {
|
||||
return SnapshotBrowserPage{}, false
|
||||
}
|
||||
items, ok := cache.children[path]
|
||||
if !ok {
|
||||
items = []SnapshotBrowserNode{}
|
||||
}
|
||||
return SnapshotBrowserPage{Path: path, Items: items, Total: cache.total, Cached: true, Generated: cache.generated}, true
|
||||
}
|
||||
|
||||
func (s *Service) storeSnapshotTree(key string, cache snapshotTreeCache) {
|
||||
s.snapshotMu.Lock()
|
||||
defer s.snapshotMu.Unlock()
|
||||
if s.snapshotTree == nil {
|
||||
s.snapshotTree = map[string]snapshotTreeCache{}
|
||||
}
|
||||
for cacheKey, value := range s.snapshotTree {
|
||||
if time.Since(value.generated) > 30*time.Minute {
|
||||
delete(s.snapshotTree, cacheKey)
|
||||
}
|
||||
}
|
||||
s.snapshotTree[key] = cache
|
||||
}
|
||||
|
||||
func buildSnapshotTreeCache(items []map[string]any) snapshotTreeCache {
|
||||
root := &snapshotTreeNode{path: "/", name: "/", nodeType: "dir", children: map[string]*snapshotTreeNode{}}
|
||||
nodes := map[string]*snapshotTreeNode{"/": root}
|
||||
ensure := func(path string, nodeType string, size int64) *snapshotTreeNode {
|
||||
return root
|
||||
}
|
||||
ensure = func(path string, nodeType string, size int64) *snapshotTreeNode {
|
||||
path = normalizeSnapshotPath(path)
|
||||
if node, ok := nodes[path]; ok {
|
||||
if nodeType != "" && nodeType != "dir" {
|
||||
node.nodeType = nodeType
|
||||
node.size = size
|
||||
}
|
||||
return node
|
||||
}
|
||||
parentPath := "/"
|
||||
if path != "/" {
|
||||
parentPath = filepath.Dir(path)
|
||||
if parentPath == "." {
|
||||
parentPath = "/"
|
||||
}
|
||||
}
|
||||
parent := ensure(parentPath, "dir", 0)
|
||||
node := &snapshotTreeNode{path: path, name: filepath.Base(path), nodeType: nodeType, size: size, children: map[string]*snapshotTreeNode{}}
|
||||
if path == "/" {
|
||||
node.name = "/"
|
||||
}
|
||||
nodes[path] = node
|
||||
parent.children[path] = node
|
||||
return node
|
||||
}
|
||||
for _, item := range items {
|
||||
path := snapshotItemPath(item)
|
||||
if path == "/" {
|
||||
continue
|
||||
}
|
||||
nodeType := "file"
|
||||
if value, _ := item["type"].(string); value == "dir" {
|
||||
nodeType = "dir"
|
||||
}
|
||||
ensure(path, nodeType, snapshotItemSize(item))
|
||||
}
|
||||
children := make(map[string][]SnapshotBrowserNode, len(nodes))
|
||||
for _, node := range nodes {
|
||||
if len(node.children) == 0 {
|
||||
continue
|
||||
}
|
||||
list := make([]SnapshotBrowserNode, 0, len(node.children))
|
||||
for _, child := range node.children {
|
||||
list = append(list, SnapshotBrowserNode{Path: child.path, Name: child.name, Type: child.nodeType, Size: child.size, HasChildren: len(child.children) > 0})
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool {
|
||||
if list[i].Type != list[j].Type {
|
||||
return list[i].Type == "dir"
|
||||
}
|
||||
return strings.ToLower(list[i].Name) < strings.ToLower(list[j].Name)
|
||||
})
|
||||
children[node.path] = list
|
||||
}
|
||||
return snapshotTreeCache{generated: time.Now().UTC(), total: len(items), children: children}
|
||||
}
|
||||
|
||||
func snapshotItemPath(item map[string]any) string {
|
||||
for _, key := range []string{"path", "name"} {
|
||||
if value, ok := item[key].(string); ok && value != "" {
|
||||
return normalizeSnapshotPath(value)
|
||||
}
|
||||
}
|
||||
return "/"
|
||||
}
|
||||
|
||||
func snapshotItemSize(item map[string]any) int64 {
|
||||
switch value := item["size"].(type) {
|
||||
case float64:
|
||||
return int64(value)
|
||||
case int64:
|
||||
return value
|
||||
case int:
|
||||
return int64(value)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeSnapshotPath(path string) string {
|
||||
path = filepath.Clean("/" + strings.TrimSpace(path))
|
||||
if path == "." {
|
||||
return "/"
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func (s *Service) TestRepository(ctx context.Context, repoID string, initialize bool) error {
|
||||
|
||||
Reference in New Issue
Block a user