feat: storage migration to backuppath,snap can read filepath from storage config

This commit is contained in:
pggiroro
2025-12-14 17:50:31 +08:00
parent 855511a8ff
commit 3adcb6211c
15 changed files with 1129 additions and 392 deletions

View File

@@ -0,0 +1,177 @@
# Local Storage 主备存储配置说明
## 配置格式
### 1. 旧格式(字符串)- 兼容
```yaml
onpub:
record:
"(.*)":
filepath: "/data/record/$0"
storage:
local: "/data/record" # 直接字符串路径
```
### 2. 新格式(只有主存储)
```yaml
onpub:
record:
"(.*)":
filepath: "/data/record/$0"
storage:
local:
path: "/data/ssd"
diskthreshold: 90
```
### 3. 新格式(主备两级存储)
```yaml
onpub:
record:
"(.*)":
filepath: "/data/record/$0"
storage:
local:
path: "/data/ssd" # 主存储路径
backuppath: "/data/hdd" # 备用存储路径
diskthreshold: 80 # 主存储阈值
backupdiskthreshold: 95 # 备用存储阈值
```
## 配置字段说明
### LocalStorageConfig 字段
| 字段 | 类型 | 必填 | 默认值 | 说明 |
|------|------|------|--------|------|
| `path` | string | 是 | - | 主存储路径(绝对路径或相对路径) |
| `backuppath` | string | 否 | "" | 备用存储路径(可选) |
| `diskthreshold` | int | 否 | 0 | 主存储磁盘使用率阈值0-1000表示不检查 |
| `backupdiskthreshold` | int | 否 | 0 | 备用存储磁盘使用率阈值0-100 |
## 配置示例
### 示例1单级存储旧格式
```yaml
storage:
local: "/data/record"
```
**说明**:兼容旧配置,等价于:
```yaml
storage:
local:
path: "/data/record"
diskthreshold: 0
```
### 示例2只有主存储新格式
```yaml
storage:
local:
path: "/data/ssd"
diskthreshold: 90
```
**说明**
- 只配置主存储
- 当磁盘使用率达到 90% 时,删除最旧文件
### 示例3主备两级存储
```yaml
storage:
local:
path: "/data/ssd"
backuppath: "/data/hdd"
diskthreshold: 80
backupdiskthreshold: 95
```
**说明**
- 主存储SSD`/data/ssd`
- 备用存储HDD`/data/hdd`
- 当 SSD 使用率达到 80% 时,迁移最旧文件到 HDD
- 当 HDD 使用率达到 95% 时,删除 HDD 中最旧文件
## 实现逻辑
### 路径选择算法selectStoragePath
当前实现:简单返回主存储路径
```go
func (s *LocalStorage) selectStoragePath() (string, error) {
// 当前简单返回主存储路径
// 后续可根据磁盘使用率动态选择主存储或备用存储
return s.config.Path, nil
}
```
### 存储管理策略(待实现)
#### 单级存储
```
磁盘使用率 < diskthreshold → 正常写入
磁盘使用率 ≥ diskthreshold → 删除最旧文件
```
#### 主备两级存储
```
主存储使用率 < diskthreshold → 正常写入主存储
主存储使用率 ≥ diskthreshold → 迁移最旧文件到备用存储
备用存储使用率 < backupdiskthreshold → 正常接收迁移文件
备用存储使用率 ≥ backupdiskthreshold → 删除备用存储中最旧文件
```
### 与全局配置的关系
```yaml
mp4:
autooverwritediskpercent: 90 # 全局阈值(兜底)
onpub:
record:
^mp4/.+:
storage:
local:
path: "/data/ssd"
diskthreshold: 80 # 优先使用这个阈值
```
**优先级规则**
1. 如果 `storage.local.diskthreshold` 配置了(> 0使用它
2. 如果 `storage.local.diskthreshold` 未配置(= 0使用 `mp4.autooverwritediskpercent`
## 当前实现状态
### ✅ 已实现
- [x] 配置解析(支持字符串、对象格式)
- [x] 配置验证(路径必填、磁盘使用率范围检查)
- [x] 兼容旧配置(字符串路径)
- [x] 结构体定义LocalStorageConfig主备两级
- [x] 路径选择逻辑(当前返回主存储路径)
- [x] 单元测试(配置解析、路径选择、配置验证)
### 🚧 待实现
- [ ] 磁盘使用率检查(使用 `syscall.Statfs``github.com/shirou/gopsutil/v4/disk`
- [ ] 文件迁移逻辑(主存储 → 备用存储)
- [ ] 文件删除逻辑(达到阈值时删除最旧文件)
- [ ] 与全局阈值的集成(优先级判断)
- [ ] 存储管理任务(定期检查和执行)
- [ ] 存储切换日志(记录迁移和删除操作)
## 测试
运行单元测试:
```bash
cd /Volumes/extend/go/src/m7s/monibucav5/pkg/storage
go test -v -run TestParseLocalStorageConfig
go test -v -run TestLocalStorageConfigValidate
```
## 注意事项
1. **路径格式**:建议使用绝对路径,相对路径会被转换为绝对路径
2. **优先级**数字越小优先级越高1 > 2 > 3
3. **磁盘使用率**0 表示不检查1-100 表示阈值百分比
4. **兼容性**:旧配置(字符串)仍然完全兼容
5. **降级路径**`record.filepath` 始终作为最后的降级选项

View File

@@ -87,6 +87,9 @@ func NewCOSStorage(config *COSStorageConfig) (*COSStorage, error) {
}, nil
}
func (s *COSStorage) GetKey() string {
return "cos"
}
func (s *COSStorage) CreateFile(ctx context.Context, path string) (File, error) {
objectKey := s.getObjectKey(path)
return &COSFile{

View File

@@ -3,59 +3,198 @@ package storage
import (
"context"
"fmt"
"io"
"log"
"os"
"path/filepath"
"time"
"github.com/shirou/gopsutil/v4/disk"
"gorm.io/gorm"
)
// LocalStorageConfig 本地存储配置
type LocalStorageConfig string
// RecordFile 录像文件记录(避免循环引用,只包含必要字段)
type RecordFile struct {
ID uint `gorm:"primarykey"`
StartTime time.Time `gorm:"column:start_time"`
EndTime time.Time `gorm:"column:end_time"`
FilePath string `gorm:"column:file_path"`
StreamPath string `gorm:"column:stream_path"`
StorageLevel int `gorm:"column:storage_level"` // 1=主存储2=备用存储
StorageType string `gorm:"column:storage_type"` // 存储类型local/s3/oss/cos
RecordLevel string `gorm:"column:record_level"` // 'high'=重要录像,其他=普通录像
CreatedAt time.Time `gorm:"column:created_at"`
DeletedAt gorm.DeletedAt `gorm:"index"` // 软删除支持
}
func (c LocalStorageConfig) GetType() StorageType {
// TableName 指定表名
func (RecordFile) TableName() string {
return "record_streams"
}
// LocalStorageConfig 本地存储配置(主备两级)
type LocalStorageConfig struct {
Path string `json:"path" yaml:"path"` // 主存储路径
BackupPath string `json:"backuppath" yaml:"backuppath"` // 备用存储路径(可选)
OverwritePercent int `json:"overwritepercent" yaml:"overwritepercent"` // 主存储磁盘使用率阈值0-1000表示不检查
BackupOverwritePercent int `json:"backupoverwritepercent" yaml:"backupoverwritepercent"` // 备用存储磁盘使用率阈值0-100
FilePathPattern string `json:"-" yaml:"-"` // 文件路径模式(从 record.filepath 传入,用于数据库查询)
}
func (c *LocalStorageConfig) GetType() StorageType {
return StorageTypeLocal
}
func (c LocalStorageConfig) Validate() error {
if c == "" {
return fmt.Errorf("base_path is required for local storage")
func (c *LocalStorageConfig) Validate() error {
if c.OverwritePercent < 0 || c.OverwritePercent > 100 {
return fmt.Errorf("overwritepercent must be between 0 and 100")
}
if c.BackupOverwritePercent < 0 || c.BackupOverwritePercent > 100 {
return fmt.Errorf("backupoverwritepercent must be between 0 and 100")
}
// 如果配置了备用路径,必须配置备用阈值
return nil
}
// parseLocalStorageConfig 解析配置,支持字符串和对象格式
func parseLocalStorageConfig(config any) (*LocalStorageConfig, error) {
switch v := config.(type) {
case string:
// 兼容旧配置:字符串路径
return &LocalStorageConfig{
Path: v,
BackupPath: "",
OverwritePercent: 0, // 0 表示不检查磁盘使用率
BackupOverwritePercent: 0,
}, nil
case map[string]any:
// 新配置:对象格式
cfg := &LocalStorageConfig{}
// 解析 path必填
if path, ok := v["path"].(string); ok {
cfg.Path = path
} else {
return nil, fmt.Errorf("path is required")
}
// 解析 backuppath可选
if backupPath, ok := v["backuppath"].(string); ok {
cfg.BackupPath = backupPath
}
// 解析 overwritepercent可选
if percent, ok := v["overwritepercent"].(int); ok {
cfg.OverwritePercent = percent
} else if percent, ok := v["overwritepercent"].(float64); ok {
cfg.OverwritePercent = int(percent)
}
// 解析 backupoverwritepercent可选
if percent, ok := v["backupoverwritepercent"].(int); ok {
cfg.BackupOverwritePercent = percent
} else if percent, ok := v["backupoverwritepercent"].(float64); ok {
cfg.BackupOverwritePercent = int(percent)
}
return cfg, nil
default:
return nil, fmt.Errorf("invalid config type for local storage: %T, expected string or map", config)
}
}
// LocalStorage 本地存储实现
type LocalStorage struct {
basePath string
config *LocalStorageConfig // 存储配置
db *gorm.DB // 数据库连接(用于查询和更新记录)
globalThreshold float64 // 全局磁盘使用率阈值(来自 mp4.autooverwritediskpercent
}
// NewLocalStorage 创建本地存储实例
func NewLocalStorage(config LocalStorageConfig) (*LocalStorage, error) {
func NewLocalStorage(configAny any) (*LocalStorage, error) {
config, err := parseLocalStorageConfig(configAny)
if err != nil {
return nil, err
}
if err := config.Validate(); err != nil {
return nil, err
}
basePath, err := filepath.Abs(string(config))
// 验证并创建主存储路径
absPath, err := filepath.Abs(config.Path)
if err != nil {
return nil, fmt.Errorf("invalid base path: %w", err)
return nil, fmt.Errorf("invalid path: %w", err)
}
if err := os.MkdirAll(absPath, 0755); err != nil {
return nil, fmt.Errorf("failed to create path: %w", err)
}
// 确保基础路径存在
if err := os.MkdirAll(basePath, 0755); err != nil {
return nil, fmt.Errorf("failed to create base path: %w", err)
// 如果配置了备用路径,验证并创建
if config.BackupPath != "" {
backupAbsPath, err := filepath.Abs(config.BackupPath)
if err != nil {
return nil, fmt.Errorf("invalid backup path: %w", err)
}
if err := os.MkdirAll(backupAbsPath, 0755); err != nil {
return nil, fmt.Errorf("failed to create backup path: %w", err)
}
}
return &LocalStorage{
basePath: basePath,
config: config,
}, nil
}
// selectStoragePath 选择存储路径(主存储或备用存储)
// TODO: 后续可在此处实现磁盘使用率检查和自动降级
func (s *LocalStorage) selectStoragePath() (string, error) {
// 当前简单返回主存储路径
// 后续可根据磁盘使用率动态选择主存储或备用存储
return s.config.Path, nil
}
func (s *LocalStorage) GetKey() string {
return string(s.config.GetType())
}
// GetStoragePath 根据存储级别返回对应的存储路径
// storageLevel: 1=主存储, 2=备用存储
func (s *LocalStorage) GetStoragePath(storageLevel int) string {
if storageLevel == 2 && s.config.BackupPath != "" {
return s.config.BackupPath
}
return s.config.Path
}
// GetFullPath 根据存储级别和相对路径返回完整路径
// storageLevel: 1=主存储, 2=备用存储
// relativePath: 相对路径(如数据库中的 FilePath
func (s *LocalStorage) GetFullPath(relativePath string, storageLevel int) string {
basePath := s.GetStoragePath(storageLevel)
return filepath.Join(basePath, relativePath)
}
func (s *LocalStorage) CreateFile(ctx context.Context, path string) (File, error) {
// 选择存储路径
basePath, err := s.selectStoragePath()
if err != nil {
return nil, err
}
// 构建完整路径
fullPath := filepath.Join(basePath, path)
dir := filepath.Dir(fullPath)
// 确保目录存在
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("failed to create directory: %w", err)
}
// 使用 O_RDWR 而不是 O_WRONLY,因为某些场景(如 MP4 writeTrailer)需要读取文件内容
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
file, err := os.OpenFile(fullPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
if err != nil {
return nil, fmt.Errorf("failed to create file: %w", err)
}
@@ -64,8 +203,17 @@ func (s *LocalStorage) CreateFile(ctx context.Context, path string) (File, error
}
func (s *LocalStorage) OpenFile(ctx context.Context, path string) (File, error) {
// 选择存储路径
basePath, err := s.selectStoragePath()
if err != nil {
return nil, err
}
// 构建完整路径
fullPath := filepath.Join(basePath, path)
// 只读模式打开文件
file, err := os.Open(path)
file, err := os.Open(fullPath)
if err != nil {
return nil, fmt.Errorf("failed to open file: %w", err)
}
@@ -73,11 +221,21 @@ func (s *LocalStorage) OpenFile(ctx context.Context, path string) (File, error)
}
func (s *LocalStorage) Delete(ctx context.Context, path string) error {
return os.Remove(path)
basePath, err := s.selectStoragePath()
if err != nil {
return err
}
fullPath := filepath.Join(basePath, path)
return os.Remove(fullPath)
}
func (s *LocalStorage) Exists(ctx context.Context, path string) (bool, error) {
_, err := os.Stat(path)
basePath, err := s.selectStoragePath()
if err != nil {
return false, err
}
fullPath := filepath.Join(basePath, path)
_, err = os.Stat(fullPath)
if err != nil {
if os.IsNotExist(err) {
return false, nil
@@ -88,7 +246,12 @@ func (s *LocalStorage) Exists(ctx context.Context, path string) (bool, error) {
}
func (s *LocalStorage) GetSize(ctx context.Context, path string) (int64, error) {
info, err := os.Stat(path)
basePath, err := s.selectStoragePath()
if err != nil {
return 0, err
}
fullPath := filepath.Join(basePath, path)
info, err := os.Stat(fullPath)
if err != nil {
if os.IsNotExist(err) {
return 0, ErrFileNotFound
@@ -99,21 +262,29 @@ func (s *LocalStorage) GetSize(ctx context.Context, path string) (int64, error)
}
func (s *LocalStorage) GetURL(ctx context.Context, path string) (string, error) {
// 本地存储返回文件路径
return path, nil
basePath, err := s.selectStoragePath()
if err != nil {
return "", err
}
// 本地存储返回完整文件路径
return filepath.Join(basePath, path), nil
}
func (s *LocalStorage) List(ctx context.Context, prefix string) ([]FileInfo, error) {
searchPath := filepath.Join(prefix)
basePath, err := s.selectStoragePath()
if err != nil {
return nil, err
}
searchPath := filepath.Join(basePath, prefix)
var files []FileInfo
err := filepath.Walk(searchPath, func(path string, info os.FileInfo, err error) error {
err = filepath.Walk(searchPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
relPath, err := filepath.Rel(prefix, path)
relPath, err := filepath.Rel(searchPath, path)
if err != nil {
return err
}
@@ -135,12 +306,317 @@ func (s *LocalStorage) Close() error {
return nil
}
// SetDB 设置数据库连接
func (s *LocalStorage) SetDB(db *gorm.DB) {
s.db = db
}
// SetGlobalThreshold 设置全局磁盘使用率阈值
func (s *LocalStorage) SetGlobalThreshold(threshold float64) {
s.globalThreshold = threshold
}
// SetFilePathPattern 设置文件路径模式(用于数据库查询)
func (s *LocalStorage) SetFilePathPattern(pattern string) {
s.config.FilePathPattern = pattern
}
// getDiskUsagePercent 获取指定路径的磁盘使用率百分比
func (s *LocalStorage) getDiskUsagePercent(path string) float64 {
d, err := disk.Usage(path)
if err != nil || d == nil {
return 0
}
return d.UsedPercent
}
// getEffectiveThreshold 获取有效的磁盘使用率阈值本地配置优先0 则使用全局)
func (s *LocalStorage) getEffectiveThreshold(localPercent int) float64 {
if localPercent > 0 {
return float64(localPercent)
}
return s.globalThreshold
}
// CheckAndManageStorage 检查并管理存储空间(迁移或删除文件)
func (s *LocalStorage) CheckAndManageStorage() error {
if s.db == nil {
return fmt.Errorf("database connection not set")
}
// 获取有效阈值
primaryThreshold := s.getEffectiveThreshold(s.config.OverwritePercent)
backupThreshold := s.getEffectiveThreshold(s.config.BackupOverwritePercent)
// 检查主存储使用率
primaryUsage := s.getDiskUsagePercent(s.config.Path)
// 打印当前存储配置和使用情况
log.Printf("[LocalStorage] CheckAndManageStorage - Config: path=%s, backupPath=%s, overwritePercent=%d, backupOverwritePercent=%d, globalThreshold=%.2f",
s.config.Path, s.config.BackupPath, s.config.OverwritePercent, s.config.BackupOverwritePercent, s.globalThreshold)
log.Printf("[LocalStorage] CheckAndManageStorage - Primary: usage=%.2f%%, threshold=%.2f%%",
primaryUsage, primaryThreshold)
// 主存储管理:循环处理直到低于阈值
if primaryThreshold > 0 {
for primaryUsage >= primaryThreshold {
log.Printf("[LocalStorage] Primary storage exceeded threshold: %.2f%% >= %.2f%%", primaryUsage, primaryThreshold)
if s.config.BackupPath != "" {
// 有备用存储:迁移一个文件
log.Printf("[LocalStorage] Action: Migrating one file to backup storage")
if err := s.migrateOneFile(); err != nil {
if err.Error() == "query record failed: record not found" {
log.Printf("[LocalStorage] No more files to migrate, stopping")
break
}
log.Printf("[LocalStorage] Migrate file failed: %v, continuing to next file", err)
// 继续处理下一个文件(已在 migrateOneFile 中软删除失败的记录)
}
} else {
// 无备用存储:删除一个文件
log.Printf("[LocalStorage] Action: Deleting one file from primary storage")
if err := s.deleteOldestFiles(s.config.Path); err != nil {
if err.Error() == "query oldest record failed: record not found" {
log.Printf("[LocalStorage] No more files to delete, stopping")
break
}
log.Printf("[LocalStorage] Delete file failed: %v, continuing to next file", err)
// 继续处理下一个文件(已在 deleteOldestFiles 中软删除失败的记录)
}
}
// 重新检查磁盘使用率
primaryUsage = s.getDiskUsagePercent(s.config.Path)
log.Printf("[LocalStorage] Primary storage after operation: %.2f%%", primaryUsage)
// 避免无限循环
time.Sleep(100 * time.Millisecond)
}
log.Printf("[LocalStorage] Primary storage OK: %.2f%% < %.2f%%", primaryUsage, primaryThreshold)
}
// 备用存储管理:循环处理直到低于阈值
if s.config.BackupPath != "" && backupThreshold > 0 {
backupUsage := s.getDiskUsagePercent(s.config.BackupPath)
log.Printf("[LocalStorage] CheckAndManageStorage - Backup: usage=%.2f%%, threshold=%.2f%%",
backupUsage, backupThreshold)
for backupUsage >= backupThreshold {
log.Printf("[LocalStorage] Backup storage exceeded threshold: %.2f%% >= %.2f%%", backupUsage, backupThreshold)
log.Printf("[LocalStorage] Action: Deleting one file from backup storage")
if err := s.deleteOldestFiles(s.config.BackupPath); err != nil {
if err.Error() == "query oldest record failed: record not found" {
log.Printf("[LocalStorage] No more files to delete, stopping")
break
}
log.Printf("[LocalStorage] Delete file failed: %v, continuing to next file", err)
// 继续处理下一个文件(已在 deleteOldestFiles 中软删除失败的记录)
}
// 重新检查磁盘使用率
backupUsage = s.getDiskUsagePercent(s.config.BackupPath)
log.Printf("[LocalStorage] Backup storage after operation: %.2f%%", backupUsage)
// 避免无限循环
time.Sleep(100 * time.Millisecond)
}
log.Printf("[LocalStorage] Backup storage OK: %.2f%% < %.2f%%", backupUsage, backupThreshold)
}
return nil
}
// migrateOneFile 迁移一个最旧的文件到备用存储
func (s *LocalStorage) migrateOneFile() error {
if s.config.BackupPath == "" {
return fmt.Errorf("backup path not configured")
}
// 查询主存储中最旧的一个文件storage_level=1 表示主存储)
var record RecordFile
err := s.db.Where("storage_level = ?", 1).
Where("storage_type = ?", "local").
Where("type = ?", "mp4").
Where("end_time IS NOT NULL").
Order("end_time ASC").
First(&record).Error
if err != nil {
return fmt.Errorf("query record failed: %w", err)
}
// 迁移文件
err = s.migrateFile(&record)
if err != nil {
// 迁移失败,软删除数据库记录,避免永远卡在这个文件上
log.Printf("[LocalStorage] migrateOneFile - migration failed, soft deleting record: %s (ID=%d), error: %v", record.FilePath, record.ID, err)
if deleteErr := s.db.Delete(&record).Error; deleteErr != nil {
log.Printf("[LocalStorage] migrateOneFile - failed to soft delete record: %v", deleteErr)
}
return err
}
return nil
}
// migrateFile 迁移单个文件
func (s *LocalStorage) migrateFile(record *RecordFile) error {
log.Printf("[LocalStorage] migrateFile - migrating file: %s (ID=%d, StorageLevel=%d -> 2)", record.FilePath, record.ID, record.StorageLevel)
// 构建源文件和目标文件的绝对路径
var srcPath, destPath string
if filepath.IsAbs(record.FilePath) {
// 已经是绝对路径(不应该出现这种情况,但做兼容处理)
log.Printf("[LocalStorage] migrateFile - WARNING: file_path is absolute, this should not happen")
srcPath = record.FilePath
// 尝试提取相对路径部分用于目标路径
relPath, err := filepath.Rel(s.config.Path, record.FilePath)
if err != nil {
return fmt.Errorf("cannot extract relative path: %w", err)
}
destPath = filepath.Join(s.config.BackupPath, relPath)
} else {
// 相对路径(正常情况)
srcPath = filepath.Join(s.config.Path, record.FilePath)
destPath = filepath.Join(s.config.BackupPath, record.FilePath)
}
destDir := filepath.Dir(destPath)
log.Printf("[LocalStorage] migrateFile - source: %s, destination: %s", srcPath, destPath)
// 确保目标目录存在
if err := os.MkdirAll(destDir, 0755); err != nil {
return fmt.Errorf("create dest directory failed: %w", err)
}
// 尝试使用 os.Rename同磁盘快速移动
err := os.Rename(srcPath, destPath)
if err != nil {
// 跨磁盘移动,需要复制后删除
log.Printf("[LocalStorage] migrateFile - cross-disk migration, using copy and remove")
if err := s.copyAndRemove(srcPath, destPath); err != nil {
return fmt.Errorf("copy and remove failed: %w", err)
}
}
// 更新数据库记录file_path 保持相对路径不变,只更新 storage_level
err = s.db.Model(record).
Updates(map[string]interface{}{
"storage_level": 2, // 2 表示备用存储
}).Error
if err != nil {
return fmt.Errorf("update database failed: %w", err)
}
log.Printf("[LocalStorage] migrateFile - successfully migrated and updated database (ID=%d)", record.ID)
return nil
}
// copyAndRemove 复制文件并删除源文件(用于跨磁盘迁移)
func (s *LocalStorage) copyAndRemove(src, dst string) error {
// 打开源文件
srcFile, err := os.Open(src)
if err != nil {
return fmt.Errorf("open source file failed: %w", err)
}
defer srcFile.Close()
// 创建目标文件
dstFile, err := os.Create(dst)
if err != nil {
return fmt.Errorf("create dest file failed: %w", err)
}
defer dstFile.Close()
// 复制内容
if _, err := io.Copy(dstFile, srcFile); err != nil {
return fmt.Errorf("copy file failed: %w", err)
}
// 同步到磁盘
if err := dstFile.Sync(); err != nil {
return fmt.Errorf("sync file failed: %w", err)
}
// 删除源文件
if err := os.Remove(src); err != nil {
return fmt.Errorf("remove source file failed: %w", err)
}
return nil
}
// deleteOldestFiles 删除指定路径下最旧的文件(优先删除非重要录像)
func (s *LocalStorage) deleteOldestFiles(path string) error {
// 判断是主存储还是备用存储
storageLevel := 1 // 默认主存储
if path == s.config.BackupPath {
storageLevel = 2 // 备用存储
}
log.Printf("[LocalStorage] deleteOldestFiles - path=%s, storageLevel=%d", path, storageLevel)
// 查询该存储级别下最旧的文件record_level != 'high' 表示非重要录像)
var record RecordFile
err := s.db.Where("storage_type = ?", "local").
Where("type = ?", "mp4").
Where("storage_level = ?", storageLevel).
Where("record_level != ?", "high").
Where("end_time IS NOT NULL").
Order("end_time ASC").
First(&record).Error
if err != nil {
return fmt.Errorf("query oldest record failed: %w", err)
}
log.Printf("[LocalStorage] deleteOldestFiles - deleting file: %s (ID=%d, StorageLevel=%d)", record.FilePath, record.ID, record.StorageLevel)
// 构建绝对路径
var absolutePath string
if filepath.IsAbs(record.FilePath) {
// 已经是绝对路径,直接使用
absolutePath = record.FilePath
log.Printf("[LocalStorage] deleteOldestFiles - file_path is absolute, using directly")
} else {
// 相对路径,根据 storageLevel 拼接
if storageLevel == 1 {
// 主存储
absolutePath = filepath.Join(s.config.Path, record.FilePath)
} else {
// 备用存储
absolutePath = filepath.Join(s.config.BackupPath, record.FilePath)
}
log.Printf("[LocalStorage] deleteOldestFiles - file_path is relative, joined with storage path")
}
log.Printf("[LocalStorage] deleteOldestFiles - absolute path: %s", absolutePath)
// 删除文件
fileDeleteErr := os.Remove(absolutePath)
if fileDeleteErr != nil && !os.IsNotExist(err) {
// 文件删除失败,记录错误日志
log.Printf("[LocalStorage] deleteOldestFiles - remove file failed: %v, will soft delete record anyway", fileDeleteErr)
}
// 删除数据库记录(软删除)
// 即使文件删除失败,也要删除数据库记录,避免永远卡在这个文件上
if err := s.db.Delete(&record).Error; err != nil {
log.Printf("[LocalStorage] deleteOldestFiles - soft delete record failed: %v", err)
return fmt.Errorf("delete database record failed: %w", err)
}
log.Printf("[LocalStorage] deleteOldestFiles - successfully deleted file and record (ID=%d)", record.ID)
return nil
}
func init() {
Factory["local"] = func(config any) (Storage, error) {
localConfig, ok := config.(string)
if !ok {
return nil, fmt.Errorf("invalid config type for local storage")
}
return NewLocalStorage(LocalStorageConfig(localConfig))
return NewLocalStorage(config)
}
}

View File

@@ -84,6 +84,10 @@ func NewOSSStorage(config *OSSStorageConfig) (*OSSStorage, error) {
}, nil
}
func (s *OSSStorage) GetKey() string {
return "oss"
}
func (s *OSSStorage) CreateFile(ctx context.Context, path string) (File, error) {
objectKey := s.getObjectKey(path)
return &OSSFile{

View File

@@ -103,7 +103,9 @@ func NewS3Storage(config *S3StorageConfig) (*S3Storage, error) {
downloader: s3manager.NewDownloader(sess),
}, nil
}
func (s *S3Storage) GetKey() string {
return "s3"
}
func (s *S3Storage) CreateFile(ctx context.Context, path string) (File, error) {
objectKey := s.getObjectKey(path)
return &S3File{

View File

@@ -46,6 +46,8 @@ type Storage interface {
// Close 关闭存储连接
Close() error
GetKey() string
}
// Writer 写入器接口

View File

@@ -128,13 +128,13 @@ func writeMetaTag(file storage.File, suber *m7s.Subscriber, filepositions []uint
}
amf.GetBuffer().Reset()
marshals := amf.Marshals("onMetaData", metaData)
task := &writeMetaTagTask{
wrTask := &writeMetaTagTask{
file: file,
flags: flags,
metaData: marshals,
}
task.Logger = suber.Logger.With("file", file.Name())
writeMetaTagQueueTask.AddTask(task)
wrTask.Logger = suber.Logger.With("file", file.Name())
writeMetaTagQueueTask.AddTask(wrTask)
}
func NewRecorder(conf config.Record) m7s.IRecorder {
@@ -162,22 +162,17 @@ func (r *Recorder) createStream(start time.Time) (err error) {
}
// 获取存储实例
storage := r.RecordJob.GetStorage()
st := r.RecordJob.GetStorage()
if storage != nil {
// 使用存储抽象层
r.file, err = storage.CreateFile(context.Background(), r.Event.FilePath)
if err != nil {
return
}
r.writer = NewFlvWriter(r.file)
} else {
// 默认本地文件行为
if r.file, err = os.OpenFile(r.Event.FilePath, os.O_CREATE|os.O_RDWR, 0666); err != nil {
return
}
r.writer = NewFlvWriter(r.file)
if st == nil {
return fmt.Errorf("global storage is nil")
}
// 使用存储抽象层
r.file, err = st.CreateFile(context.Background(), r.Event.FilePath)
if err != nil {
return
}
r.writer = NewFlvWriter(r.file)
_, err = r.writer.Write(FLVHead)
if err != nil {

View File

@@ -43,7 +43,11 @@ func (r *Recorder) writeTailer(end time.Time) {
if !r.RecordJob.RecConf.RealTime {
defer r.TsInMemory.Recycle()
var err error
r.file, err = r.RecordJob.GetStorage().CreateFile(context.Background(), r.Event.FilePath)
st := r.RecordJob.GetStorage()
if st == nil {
return
}
r.file, err = st.CreateFile(context.Background(), r.Event.FilePath)
if err != nil {
return
}
@@ -59,7 +63,11 @@ func (r *Recorder) Dispose() {
func (r *Recorder) createNewTs() (err error) {
if r.RecordJob.RecConf.RealTime {
r.file, err = r.RecordJob.GetStorage().CreateFile(context.Background(), r.Event.FilePath)
st := r.RecordJob.GetStorage()
if st == nil {
return
}
r.file, err = st.CreateFile(context.Background(), r.Event.FilePath)
}
return
}

View File

@@ -51,72 +51,128 @@ func (p *MP4Plugin) downloadSingleFile(stream *m7s.RecordStream, flag mp4.Flag,
var file storage.File
var err error
if stream.StorageType != "" && stream.StorageType != "local" {
// 远程存储:从 S3/OSS/COS 获取文件
var storageConfig map[string]any
// 最高优先级:如果 FilePath 是绝对路径,直接使用,跳过所有 storage 处理
if filepath.IsAbs(stream.FilePath) {
if flag == 0 {
// 普通 MP4直接 ServeFile
http.ServeFile(w, r, stream.FilePath)
return
}
// fMP4直接打开文件
file, err = os.Open(stream.FilePath)
if err != nil {
http.Error(w, fmt.Sprintf("failed to open file: %v", err), http.StatusInternalServerError)
p.Error("failed to open file", "err", err, "path", stream.FilePath)
return
}
defer file.Close()
p.Info("reading file for fmp4 conversion from absolute path", "path", stream.FilePath)
// 继续执行 fMP4 转换处理
} else {
// 相对路径:使用 storage 处理
// 检查全局存储是否存在且类型匹配
st := p.Server.Storage
var globalStorageType string
if st != nil {
globalStorageType = st.GetKey()
}
useGlobalStorage := st != nil && globalStorageType == stream.StorageType
isLocalStorage := stream.StorageType == string(storage.StorageTypeLocal) || stream.StorageType == ""
// 遍历所有录像配置规则,查找包含该 storage 类型的配置
commonConf := p.GetCommonConf()
for _, recConf := range commonConf.OnPub.Record {
if recConf.Storage != nil {
if cfg, ok := recConf.Storage[stream.StorageType]; ok {
if cfgMap, ok := cfg.(map[string]any); ok {
storageConfig = cfgMap
break
// 对于普通 MP4优先直接获取存储URL或路径
if flag == 0 {
if useGlobalStorage {
if isLocalStorage {
// 本地存储:根据存储级别获取完整路径
if localStorage, ok := st.(*storage.LocalStorage); ok {
fullPath := localStorage.GetFullPath(stream.FilePath, stream.StorageLevel)
http.ServeFile(w, r, fullPath)
} else {
// 类型不匹配,使用 GetURL 作为兜底
url, err := st.GetURL(context.Background(), stream.FilePath)
if err != nil {
http.Error(w, fmt.Sprintf("failed to get URL: %v", err), http.StatusInternalServerError)
p.Error("failed to get URL", "err", err)
return
}
http.ServeFile(w, r, url)
}
} else {
// 其他存储类型,使用 GetURL 并重定向
url, err := st.GetURL(context.Background(), stream.FilePath)
if err != nil {
http.Error(w, fmt.Sprintf("failed to get URL: %v", err), http.StatusInternalServerError)
p.Error("failed to get URL", "err", err)
return
}
p.Info("redirect to storage URL", "storageType", stream.StorageType, "url", url)
http.Redirect(w, r, url, http.StatusFound)
}
} else {
// 兜底逻辑:直接使用 stream.FilePath
if isLocalStorage {
http.ServeFile(w, r, stream.FilePath)
} else {
http.Error(w, "storage type mismatch, cannot serve file", http.StatusInternalServerError)
p.Error("storage type mismatch", "streamType", stream.StorageType, "globalType", globalStorageType)
}
}
}
if storageConfig == nil {
http.Error(w, "storage config not found", http.StatusInternalServerError)
p.Error("storage config not found", "storageType", stream.StorageType)
return
}
// 创建 storage 实例
storageInstance, err := storage.CreateStorage(stream.StorageType, storageConfig)
if err != nil {
http.Error(w, fmt.Sprintf("failed to create storage: %v", err), http.StatusInternalServerError)
p.Error("failed to create storage", "err", err)
return
}
// 对于普通 MP4直接重定向到预签名 URL
if flag == 0 {
url, err := storageInstance.GetURL(context.Background(), stream.FilePath)
if err != nil {
http.Error(w, fmt.Sprintf("failed to get URL: %v", err), http.StatusInternalServerError)
p.Error("failed to get URL", "err", err)
return
}
p.Info("redirect to storage URL", "storageType", stream.StorageType, "url", url)
http.Redirect(w, r, url, http.StatusFound)
return
}
// 对于 fmp4需要读取文件进行转换只读模式不会上传
file, err = storageInstance.OpenFile(context.Background(), stream.FilePath)
if err != nil {
http.Error(w, fmt.Sprintf("failed to open remote file: %v", err), http.StatusInternalServerError)
p.Error("failed to open remote file", "err", err)
return
if useGlobalStorage {
if isLocalStorage {
// 本地存储:根据存储级别获取完整路径后打开文件
if localStorage, ok := st.(*storage.LocalStorage); ok {
fullPath := localStorage.GetFullPath(stream.FilePath, stream.StorageLevel)
file, err = os.Open(fullPath)
if err != nil {
http.Error(w, fmt.Sprintf("failed to open local file: %v", err), http.StatusInternalServerError)
p.Error("failed to open local file", "err", err, "path", fullPath)
return
}
defer file.Close()
p.Info("reading file for fmp4 conversion from local storage", "storageLevel", stream.StorageLevel, "path", fullPath)
} else {
// 类型不匹配,使用 OpenFile 作为兜底
file, err = st.OpenFile(context.Background(), stream.FilePath)
if err != nil {
http.Error(w, fmt.Sprintf("failed to open file: %v", err), http.StatusInternalServerError)
p.Error("failed to open file", "err", err)
return
}
defer file.Close()
p.Info("reading file for fmp4 conversion from global storage", "storageType", stream.StorageType, "path", stream.FilePath)
}
} else {
// 其他存储类型,使用 OpenFile
file, err = st.OpenFile(context.Background(), stream.FilePath)
if err != nil {
http.Error(w, fmt.Sprintf("failed to open file: %v", err), http.StatusInternalServerError)
p.Error("failed to open file", "err", err)
return
}
defer file.Close()
p.Info("reading file for fmp4 conversion from global storage", "storageType", stream.StorageType, "path", stream.FilePath)
}
} else {
// 兜底逻辑:直接使用 stream.FilePath 作为本地文件
if isLocalStorage {
file, err = os.Open(stream.FilePath)
if err != nil {
http.Error(w, fmt.Sprintf("failed to open local file: %v", err), http.StatusInternalServerError)
p.Error("failed to open local file", "err", err)
return
}
defer file.Close()
p.Info("reading file for fmp4 conversion from local path", "path", stream.FilePath)
} else {
http.Error(w, "storage type mismatch, cannot open file", http.StatusInternalServerError)
p.Error("storage type mismatch", "streamType", stream.StorageType, "globalType", globalStorageType)
return
}
}
defer file.Close()
p.Info("reading remote file for fmp4 conversion", "storageType", stream.StorageType, "path", stream.FilePath)
} else {
// 本地存储
if flag == 0 {
http.ServeFile(w, r, stream.FilePath)
return
}
// 本地文件用于 fmp4 转换
file, err = os.Open(stream.FilePath)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer file.Close()
}
// fmp4 转换处理(本地和远程文件统一处理)

View File

@@ -1,8 +1,6 @@
package plugin_mp4
import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
@@ -12,6 +10,7 @@ import (
"github.com/shirou/gopsutil/v4/disk"
"gorm.io/gorm"
"m7s.live/v5"
"m7s.live/v5/pkg/storage"
)
// mysql数据库里Exception 定义异常结构体
@@ -69,12 +68,12 @@ func (p *DeleteRecordTask) getDiskOutOfSpace(filePath string) bool {
return false
}
p.plugin.Debug("getDiskOutOfSpace", "current path", exePath, "disk UsedPercent", d.UsedPercent, "total disk space", d.Total,
"disk free", d.Free, "disk usage", d.Used, "AutoOverWriteDiskPercent", p.AutoOverWriteDiskPercent, "DiskMaxPercent", p.DiskMaxPercent)
return d.UsedPercent >= p.AutoOverWriteDiskPercent
"disk free", d.Free, "disk usage", d.Used, "OverwritePercent", p.OverwritePercent, "DiskMaxPercent", p.DiskMaxPercent)
return d.UsedPercent >= p.OverwritePercent
}
func (p *DeleteRecordTask) deleteOldestFile() {
//当当前磁盘使用量大于AutoOverWriteDiskPercent自动覆盖磁盘使用量配置时自动删除最旧的文件
//当当前磁盘使用量大于OverwritePercent自动覆盖磁盘使用量配置时自动删除最旧的文件
//连续录像删除最旧的文件
// 使用 map 去重,存储所有的 conf.FilePath
pathMap := make(map[string]bool)
@@ -86,9 +85,9 @@ func (p *DeleteRecordTask) deleteOldestFile() {
dirPath := filepath.Dir(conf.FilePath)
if _, exists := pathMap[dirPath]; !exists {
pathMap[dirPath] = true
p.Info("deleteOldestFile", "original filepath", conf.FilePath, "processed filepath", dirPath)
p.Info("deleteOldestFile", "action", "add path", "original", conf.FilePath, "processed", dirPath)
} else {
p.Debug("deleteOldestFile", "duplicate path ignored", "path", dirPath)
p.Debug("deleteOldestFile", "status", "duplicate path ignored", "path", dirPath)
}
}
}
@@ -101,13 +100,13 @@ func (p *DeleteRecordTask) deleteOldestFile() {
for path := range pathMap {
filePaths = append(filePaths, path)
}
p.Debug("deleteOldestFile", "after get onpub.record,filePaths.length", len(filePaths))
p.Debug("deleteOldestFile", "stage", "after onpub.record", "count", len(filePaths))
if p.plugin.EventRecordFilePath != "" {
// 同样处理EventRecordFilePath
dirPath := filepath.Dir(p.plugin.EventRecordFilePath)
filePaths = append(filePaths, dirPath)
}
p.Debug("deleteOldestFile", "after get eventrecordfilepath,filePaths.length", len(filePaths))
p.Debug("deleteOldestFile", "stage", "after get eventrecordfilepath", "count", len(filePaths))
for _, filePath := range filePaths {
for p.getDiskOutOfSpace(filePath) {
var recordStreams []m7s.RecordStream
@@ -117,43 +116,43 @@ func (p *DeleteRecordTask) deleteOldestFile() {
// 直接替换所有反斜杠,不需要判断是否包含
basePath = strings.Replace(basePath, "\\", "\\\\", -1)
searchPattern := basePath + "%"
p.Info("deleteOldestFile", "searching with path pattern", searchPattern)
p.Info("deleteOldestFile", "action", "searching", "pattern", searchPattern)
err := p.DB.Where(" record_level!='high' AND end_time IS NOT NULL").
Where("file_path LIKE ?", searchPattern).
Order("end_time ASC").Limit(1).Find(&recordStreams).Error
if err == nil {
if len(recordStreams) > 0 {
p.Info("deleteOldestFile", "found %d records", len(recordStreams))
p.Info("deleteOldestFile", "found", len(recordStreams), "unit", "records")
for _, record := range recordStreams {
p.Info("deleteOldestFile", "ready to delete oldestfile,ID", record.ID, "create time", record.EndTime, "filepath", record.FilePath)
p.Info("deleteOldestFile", "action", "deleting", "ID", record.ID, "endTime", record.EndTime, "filepath", record.FilePath)
err = os.Remove(record.FilePath)
if err != nil {
// 检查是否为文件不存在的错误
if os.IsNotExist(err) {
// 文件不存在,记录日志但视为删除成功
p.Warn("deleteOldestFile", "file does not exist, continuing with database deletion", record.FilePath)
p.Warn("deleteOldestFile", "status", "file not exist, continuing", "filepath", record.FilePath)
// 继续删除数据库记录
err = p.DB.Delete(&record).Error
if err != nil {
p.Error("deleteOldestFile", "delete record from db error", err)
p.Error("deleteOldestFile", "error", "delete record from db", "err", err)
}
} else {
// 其他错误,记录并跳过此记录
p.Error("deleteOldestFile", "delete file from disk error", err)
p.Error("deleteOldestFile", "error", "delete file from disk", "err", err)
continue
}
} else {
// 文件删除成功,继续删除数据库记录
err = p.DB.Delete(&record).Error
if err != nil {
p.Error("deleteOldestFile", "delete record from db error", err)
p.Error("deleteOldestFile", "error", "delete record from db", "err", err)
}
}
}
}
} else {
p.Error("deleteOldestFile", "search record from db error", err)
p.Error("deleteOldestFile", "error", "search record from db", "err", err)
}
time.Sleep(time.Second * 3)
}
@@ -162,12 +161,11 @@ func (p *DeleteRecordTask) deleteOldestFile() {
type StorageManagementTask struct {
task.TickTask
DiskMaxPercent float64
AutoOverWriteDiskPercent float64
MigrationThresholdPercent float64
RecordFileExpireDays int
DB *gorm.DB
plugin *MP4Plugin
DiskMaxPercent float64
OverwritePercent float64
RecordFileExpireDays int
DB *gorm.DB
plugin *MP4Plugin
}
// 为了兼容性,保留 DeleteRecordTask 作为别名
@@ -178,248 +176,169 @@ func (t *DeleteRecordTask) GetTickInterval() time.Duration {
}
func (t *StorageManagementTask) Tick(any) {
t.Debug("StorageManagementTask", "tick started")
t.Debug("StorageManagementTask", "status", "tick started")
// 阶段1文件迁移(优先级最高,释放主存储空间
t.Debug("StorageManagementTask", "phase 1: file migration")
t.migrateFiles()
// 阶段1LocalStorage 存储管理(迁移或删除
t.Debug("StorageManagementTask", "phase", "1", "action", "local storage management")
t.manageLocalStorage()
// 阶段2删除过期文件
t.Debug("StorageManagementTask", "phase 2: delete expired files")
t.Debug("StorageManagementTask", "phase", "2", "action", "delete expired files")
t.deleteExpiredFiles()
// 阶段3删除最旧文件兜底机制
t.Debug("StorageManagementTask", "phase 3: delete oldest files")
t.deleteOldestFile()
// 注意阶段3 deleteOldestFile 已被移除因为阶段1的 manageLocalStorage 已经处理了磁盘空间管理
t.Debug("StorageManagementTask", "tick completed")
t.Debug("StorageManagementTask", "status", "tick completed")
}
// migrateFiles 将主存储中的文件迁移到次级存储
func (t *StorageManagementTask) migrateFiles() {
// 只有配置了迁移阈值才执行迁移
if t.MigrationThresholdPercent <= 0 {
t.Debug("migrateFiles", "migration disabled", "threshold not configured or set to 0")
// manageLocalStorage 管理本地存储(通过 LocalStorage 进行迁移或删除)
func (t *StorageManagementTask) manageLocalStorage() {
t.Debug("manageLocalStorage", "status", "starting")
// 检查全局存储是否存在且为 LocalStorage 类型
st := t.plugin.Server.Storage
if st == nil {
t.Debug("manageLocalStorage", "status", "global storage not initialized, using fallback logic")
t.manageFallbackStorage()
return
}
t.Debug("migrateFiles", "starting migration check,threshold", t.MigrationThresholdPercent)
// 收集所有需要检查的路径(使用 map 去重)
pathMap := make(map[string]string) // primary path -> secondary path
if t.plugin.Server.Plugins.Length > 0 {
t.plugin.Server.Plugins.Range(func(plugin *m7s.Plugin) bool {
if len(plugin.GetCommonConf().OnPub.Record) > 0 {
for _, conf := range plugin.GetCommonConf().OnPub.Record {
// 只处理配置了次级路径的录像配置
if conf.SecondaryFilePath == "" {
t.Debug("migrateFiles", "skipping path without secondary storage,path", conf.FilePath)
continue
}
primaryPath := filepath.Dir(conf.FilePath)
secondaryPath := filepath.Dir(conf.SecondaryFilePath)
// 检查是否已存在
if existingSecondary, exists := pathMap[primaryPath]; exists {
if existingSecondary != secondaryPath {
t.Warn("migrateFiles", "duplicate primary path with different secondary paths",
"primary", primaryPath,
"existing secondary", existingSecondary,
"new secondary", secondaryPath)
} else {
t.Debug("migrateFiles", "duplicate path ignored,primary", primaryPath)
}
continue
}
pathMap[primaryPath] = secondaryPath
t.Debug("migrateFiles", "added path for migration check,primary", primaryPath, "secondary", secondaryPath)
}
}
return true
})
}
if len(pathMap) == 0 {
t.Debug("migrateFiles", "no secondary paths configured", "skipping migration")
localStorage, ok := st.(*storage.LocalStorage)
if !ok {
t.Debug("manageLocalStorage", "status", "global storage is not LocalStorage, using fallback logic")
t.manageFallbackStorage()
return
}
t.Debug("migrateFiles", "checking paths count", len(pathMap))
// 设置数据库连接和全局阈值
localStorage.SetDB(t.DB)
localStorage.SetGlobalThreshold(t.OverwritePercent)
// 遍历每个主存储路径
for primaryPath, secondaryPath := range pathMap {
usage := t.getDiskUsagePercent(primaryPath)
t.Debug("migrateFiles", "checking disk usage,path", primaryPath, "usage", usage, "threshold", t.MigrationThresholdPercent)
// 执行存储管理
if err := localStorage.CheckAndManageStorage(); err != nil {
t.Error("manageLocalStorage", "error", "check and manage storage failed", "err", err)
} else {
t.Debug("manageLocalStorage", "status", "success")
}
if usage < t.MigrationThresholdPercent {
t.Debug("migrateFiles", "usage below threshold,path", primaryPath, "skipping")
continue // 未达到迁移阈值,跳过
t.Debug("manageLocalStorage", "status", "completed")
}
// manageFallbackStorage 兜底逻辑:当全局存储不是 LocalStorage 时,使用全局配置管理磁盘空间
func (t *StorageManagementTask) manageFallbackStorage() {
t.Debug("manageFallbackStorage", "status", "starting")
// 尝试从全局存储获取路径
var storagePath string
if st := t.plugin.Server.Storage; st != nil {
if localStorage, ok := st.(*storage.LocalStorage); ok {
// 使用主存储路径
storagePath = localStorage.GetStoragePath(1)
} else {
// 非本地存储,尝试使用 GetURL 获取路径(可能不适用)
t.Debug("manageFallbackStorage", "status", "global storage is not LocalStorage, cannot get path")
}
}
t.Info("migrateFiles", "migration triggered", "primary path", primaryPath, "secondary path", secondaryPath, "usage", usage, "threshold", t.MigrationThresholdPercent)
// 如果无法从全局存储获取路径,使用第一个录像配置的 filepath 目录
if storagePath == "" {
recordConfigs := t.plugin.GetCommonConf().OnPub.Record
if len(recordConfigs) > 0 {
// 遍历 map 获取第一个配置的 filepath 目录
for _, conf := range recordConfigs {
storagePath = filepath.Dir(conf.FilePath)
t.Debug("manageFallbackStorage", "action", "using first record config path", "path", storagePath)
break
}
} else {
t.Debug("manageFallbackStorage", "status", "no storage path found")
return
}
}
// 查找主存储中最旧的已完成录像storage_level=1
var recordStreams []m7s.RecordStream
basePath := strings.Replace(primaryPath, "\\", "\\\\", -1)
searchPattern := basePath + "%"
// 检查磁盘使用率(使用存储路径
diskUsage := t.getDiskUsagePercent(storagePath)
t.Debug("manageFallbackStorage", "storagePath", storagePath, "diskUsage", diskUsage, "threshold", t.OverwritePercent)
// 每次迁移多个文件,提高效率
err := t.DB.Where("record_level!='high' AND end_time IS NOT NULL AND storage_level=1").
Where("file_path LIKE ?", searchPattern).
// 如果超过阈值,删除最旧的文件
for diskUsage >= t.OverwritePercent {
t.Info("manageFallbackStorage", "action", "disk usage exceeded", "storagePath", storagePath, "usage", diskUsage, "threshold", t.OverwritePercent)
// 查询最旧的文件
var record m7s.RecordStream
err := t.DB.Where("storage_type = ?", "local").
Where("type = ?", "mp4").
Where("storage_level = ?", 1).
Where("record_level != ?", "high").
Where("end_time IS NOT NULL").
Order("end_time ASC").
Limit(10). // 批量迁移10个文件
Find(&recordStreams).Error
First(&record).Error
if err != nil {
t.Error("migrateFiles", "query records error", err)
continue
}
if len(recordStreams) == 0 {
t.Debug("migrateFiles", "no files to migrate", "path", primaryPath)
continue
}
t.Info("migrateFiles", "found files to migrate", "count", len(recordStreams), "path", primaryPath)
for _, record := range recordStreams {
t.Debug("migrateFiles", "migrating file", "ID", record.ID, "filepath", record.FilePath, "endTime", record.EndTime)
if err := t.migrateFile(&record, primaryPath); err != nil {
t.Error("migrateFiles", "migrate file error", err, "ID", record.ID, "filepath", record.FilePath)
} else {
t.Info("migrateFiles", "file migrated successfully", "ID", record.ID, "from", record.FilePath, "to", record.FilePath)
if err == gorm.ErrRecordNotFound {
// 没有非重要录像,查询所有录像
err = t.DB.Where("storage_type = ?", "local").
Where("type = ?", "mp4").
Where("storage_level = ?", 1).
Where("end_time IS NOT NULL").
Order("end_time ASC").
First(&record).Error
}
if err != nil {
t.Error("manageFallbackStorage", "error", "query oldest record failed", "err", err, "storagePath", storagePath)
break
}
}
}
t.Debug("migrateFiles", "migration check completed")
}
t.Info("manageFallbackStorage", "action", "deleting file", "ID", record.ID, "filePath", record.FilePath)
// migrateFile 迁移单个文件到次级存储
func (t *StorageManagementTask) migrateFile(record *m7s.RecordStream, primaryPath string) error {
// 获取次级存储路径
secondaryPath := t.getSecondaryPath(primaryPath)
if secondaryPath == "" {
t.Debug("migrateFile", "no secondary path found", "primaryPath", primaryPath)
return fmt.Errorf("no secondary path configured for %s", primaryPath)
}
// 构建目标路径(保持相对路径结构)
relativePath := strings.TrimPrefix(record.FilePath, primaryPath)
relativePath = strings.TrimPrefix(relativePath, string(filepath.Separator))
targetPath := filepath.Join(secondaryPath, relativePath)
t.Debug("migrateFile", "preparing migration", "from", record.FilePath, "to", targetPath)
// 创建目标目录
targetDir := filepath.Dir(targetPath)
if err := os.MkdirAll(targetDir, 0755); err != nil {
t.Error("migrateFile", "create target directory failed", err, "dir", targetDir)
return fmt.Errorf("create target directory failed: %w", err)
}
t.Debug("migrateFile", "target directory created", "dir", targetDir)
// 移动文件
if err := os.Rename(record.FilePath, targetPath); err != nil {
t.Debug("migrateFile", "rename failed, trying copy", "error", err)
// 如果跨磁盘移动失败,尝试复制后删除
if err := t.copyAndRemove(record.FilePath, targetPath); err != nil {
t.Error("migrateFile", "copy and remove failed", err)
return fmt.Errorf("move file failed: %w", err)
}
t.Debug("migrateFile", "file copied and removed")
} else {
t.Debug("migrateFile", "file renamed successfully")
}
// 更新数据库记录
oldPath := record.FilePath
record.FilePath = targetPath
record.StorageLevel = 2
if err := t.DB.Save(record).Error; err != nil {
t.Error("migrateFile", "database update failed, rolling back", err)
// 如果数据库更新失败,尝试回滚文件移动
if rollbackErr := os.Rename(targetPath, oldPath); rollbackErr != nil {
t.Error("migrateFile", "rollback failed", rollbackErr, "file may be in inconsistent state")
// 判断 file_path 是相对路径还是绝对路径
var absolutePath string
if filepath.IsAbs(record.FilePath) {
// 绝对路径,直接使用
absolutePath = record.FilePath
} else {
t.Debug("migrateFile", "rollback successful")
}
return fmt.Errorf("update database failed: %w", err)
}
t.Debug("migrateFile", "database updated", "storageLevel", 2, "newPath", targetPath)
return nil
}
// copyAndRemove 复制文件后删除原文件(用于跨磁盘移动)
func (t *StorageManagementTask) copyAndRemove(src, dst string) error {
t.Debug("copyAndRemove", "starting cross-disk copy", "from", src, "to", dst)
// 获取源文件信息
srcInfo, err := os.Stat(src)
if err != nil {
return err
}
fileSize := srcInfo.Size()
t.Debug("copyAndRemove", "source file info", "size", fileSize)
// 打开源文件
srcFile, err := os.Open(src)
if err != nil {
return err
}
defer srcFile.Close()
// 创建目标文件
dstFile, err := os.Create(dst)
if err != nil {
return err
}
defer dstFile.Close()
// 复制文件内容
t.Debug("copyAndRemove", "copying file content")
copiedBytes, err := io.Copy(dstFile, srcFile)
if err != nil {
os.Remove(dst) // 复制失败,删除目标文件
t.Error("copyAndRemove", "copy failed", err, "copiedBytes", copiedBytes)
return err
}
t.Debug("copyAndRemove", "file copied", "bytes", copiedBytes)
// 同步到磁盘
if err := dstFile.Sync(); err != nil {
t.Error("copyAndRemove", "sync failed", err)
return err
}
t.Debug("copyAndRemove", "synced to disk, removing source file")
// 删除源文件
if err := os.Remove(src); err != nil {
t.Error("copyAndRemove", "remove source file failed", err)
return err
}
t.Debug("copyAndRemove", "source file removed successfully")
return nil
}
// getSecondaryPath 获取主路径对应的次级存储路径
func (t *StorageManagementTask) getSecondaryPath(primaryPath string) string {
if len(t.plugin.GetCommonConf().OnPub.Record) > 0 {
for _, conf := range t.plugin.GetCommonConf().OnPub.Record {
dirPath := filepath.Dir(conf.FilePath)
if dirPath == primaryPath && conf.SecondaryFilePath != "" {
return filepath.Dir(conf.SecondaryFilePath)
// 相对路径,使用全局存储的 GetFullPath 方法
if st := t.plugin.Server.Storage; st != nil {
if localStorage, ok := st.(*storage.LocalStorage); ok {
absolutePath = localStorage.GetFullPath(record.FilePath, record.StorageLevel)
} else {
absolutePath = record.FilePath
t.Warn("manageFallbackStorage", "warning", "file_path is relative and storage is not LocalStorage", "filePath", record.FilePath)
}
} else {
absolutePath = record.FilePath
t.Warn("manageFallbackStorage", "warning", "file_path is relative and no global storage", "filePath", record.FilePath)
}
}
t.Debug("manageFallbackStorage", "action", "removing file", "absolutePath", absolutePath)
// 删除文件
fileDeleteErr := os.Remove(absolutePath)
if fileDeleteErr != nil && !os.IsNotExist(fileDeleteErr) {
// 文件删除失败,记录错误日志
t.Error("manageFallbackStorage", "error", "remove file failed", "err", fileDeleteErr, "filePath", absolutePath)
}
// 删除数据库记录(软删除)
// 即使文件删除失败,也要删除数据库记录,避免永远卡在这个文件上
if err := t.DB.Delete(&record).Error; err != nil {
t.Error("manageFallbackStorage", "error", "delete database record failed", "err", err, "ID", record.ID)
break
}
t.Info("manageFallbackStorage", "status", "file deleted successfully", "ID", record.ID)
// 重新检查磁盘使用率
diskUsage = t.getDiskUsagePercent(storagePath)
t.Debug("manageFallbackStorage", "status", "rechecking disk usage", "storagePath", storagePath, "usage", diskUsage)
// 避免无限循环,休眠一下
time.Sleep(time.Second)
}
return ""
t.Debug("manageFallbackStorage", "status", "completed")
}
// getDiskUsagePercent 获取磁盘使用率百分比
@@ -449,15 +368,15 @@ func (t *StorageManagementTask) deleteExpiredFiles() {
err = os.Remove(record.FilePath)
if err != nil {
if os.IsNotExist(err) {
t.Warn("deleteExpiredFiles", "file does not exist", record.FilePath)
t.Warn("deleteExpiredFiles", "status", "file not exist", "filepath", record.FilePath)
} else {
t.Error("deleteExpiredFiles", "delete file error", err)
t.Error("deleteExpiredFiles", "error", "delete file", "err", err)
}
}
// 无论文件是否存在,都删除数据库记录
err = t.DB.Delete(&record).Error
if err != nil {
t.Error("deleteExpiredFiles", "delete record from db error", err)
t.Error("deleteExpiredFiles", "error", "delete record from db", "err", err)
}
}
}

View File

@@ -17,15 +17,14 @@ import (
type MP4Plugin struct {
pb.UnimplementedApiServer
m7s.Plugin
BeforeDuration time.Duration `default:"30s" desc:"事件录像提前时长不配置则默认30s"`
AfterDuration time.Duration `default:"30s" desc:"事件录像结束时长不配置则默认30s"`
RecordFileExpireDays int `desc:"录像自动删除的天数,0或未设置表示不自动删除"`
DiskMaxPercent float64 `default:"90" desc:"硬盘使用百分之上限值,超上限后触发报警,并停止当前所有磁盘写入动作。"`
AutoOverWriteDiskPercent float64 `default:"0" desc:"自动覆盖功能磁盘占用上限值,超过上限时连续录像自动删除日有录像,事件录像自动删除非重要事件录像,删除规则为删除距离当日最久日期的连续录像或非重要事件录像。"`
MigrationThresholdPercent float64 `default:"60" desc:"开始迁移到次级存储的磁盘使用率阈值,当主存储达到此阈值时自动将文件迁移到次级存储"`
AutoRecovery bool `default:"false" desc:"是否自动恢复"`
ExceptionPostUrl string `desc:"第三方异常上报地址"`
EventRecordFilePath string `desc:"事件录像存放地址"`
BeforeDuration time.Duration `default:"30s" desc:"事件录像提前时长不配置则默认30s"`
AfterDuration time.Duration `default:"30s" desc:"事件录像结束时长不配置则默认30s"`
RecordFileExpireDays int `desc:"录像自动删除的天数,0或未设置表示不自动删除"`
DiskMaxPercent float64 `default:"90" desc:"硬盘使用百分之上限值,超上限后触发报警,并停止当前所有磁盘写入动作。"`
OverwritePercent float64 `default:"80" desc:"全局磁盘使用率阈值,当 storage.local 的 overwritepercent 为 0 时使用此值作为全局兼底配置。超过阈值时自动迁移或删除最旧文件。"`
AutoRecovery bool `default:"false" desc:"是否自动恢复"`
ExceptionPostUrl string `desc:"第三方异常上报地址"`
EventRecordFilePath string `desc:"事件录像存放地址"`
}
const defaultConfig m7s.DefaultYaml = `publish:
@@ -60,12 +59,11 @@ func (p *MP4Plugin) Start() (err error) {
if err != nil {
return
}
if p.AutoOverWriteDiskPercent > 0 {
if p.OverwritePercent > 0 {
var storageTask StorageManagementTask
storageTask.DB = p.DB
storageTask.DiskMaxPercent = p.DiskMaxPercent
storageTask.AutoOverWriteDiskPercent = p.AutoOverWriteDiskPercent
storageTask.MigrationThresholdPercent = p.MigrationThresholdPercent
storageTask.OverwritePercent = p.OverwritePercent
storageTask.RecordFileExpireDays = p.RecordFileExpireDays
storageTask.plugin = p
p.AddTask(&storageTask)

View File

@@ -157,21 +157,15 @@ func (r *Recorder) createStream(start time.Time) (err error) {
// 直接创建新文件并覆盖 r.file
// 获取存储实例
storage := r.RecordJob.GetStorage()
st := r.RecordJob.GetStorage()
if storage != nil {
// 使用存储抽象层
r.file, err = storage.CreateFile(context.Background(), r.Event.FilePath)
if err != nil {
return
}
} else {
// 默认本地文件行为
// 使用 OpenFile 以读写模式打开,因为 writeTrailerTask.Run() 需要读取文件内容
r.file, err = os.OpenFile(r.Event.FilePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
if err != nil {
return
}
if st == nil {
return fmt.Errorf("global storage is nil")
}
// 使用存储抽象层
r.file, err = st.CreateFile(context.Background(), r.Event.FilePath)
if err != nil {
return
}
if r.Event.Type == "fmp4" {

View File

@@ -2,11 +2,13 @@ package plugin_snap
import (
"bytes"
"context"
"encoding/json"
"fmt"
"image"
"image/color"
_ "image/jpeg"
"io"
"net/http"
"os"
"os/exec"
@@ -20,6 +22,7 @@ import (
"github.com/disintegration/imaging"
m7s "m7s.live/v5"
"m7s.live/v5/pkg"
"m7s.live/v5/pkg/storage"
"m7s.live/v5/pkg/util"
snap_pkg "m7s.live/v5/plugin/snap/pkg"
"m7s.live/v5/plugin/snap/pkg/watermark"
@@ -238,6 +241,7 @@ func (p *SnapPlugin) querySnap(rw http.ResponseWriter, r *http.Request) {
http.Error(rw, "database not initialized", http.StatusInternalServerError)
return
}
var err error
streamPath := r.PathValue("streamPath")
if streamPath == "" {
@@ -277,7 +281,26 @@ func (p *SnapPlugin) querySnap(rw http.ResponseWriter, r *http.Request) {
}
// 读取图片文件
imgData, err := os.ReadFile(record.SnapPath)
var imgData []byte
if st := p.Server.Storage; st != nil {
// 优先通过全局存储读取
if localStorage, ok := st.(*storage.LocalStorage); ok {
path := record.SnapPath
if !filepath.IsAbs(path) {
path = localStorage.GetFullPath(path, 1)
}
imgData, err = os.ReadFile(path)
} else {
var f storage.File
f, err = st.OpenFile(context.Background(), record.SnapPath)
if err == nil {
defer f.Close()
imgData, err = io.ReadAll(f)
}
}
} else {
imgData, err = os.ReadFile(record.SnapPath)
}
if err != nil {
http.Error(rw, "failed to read snapshot file", http.StatusNotFound)
return
@@ -641,11 +664,9 @@ func (p *SnapPlugin) batchPlayBack(rw http.ResponseWriter, r *http.Request) {
granularity = granularityVal
}
// 创建保存目录
// 保存路径前缀(相对路径),实际写入时使用全局存储解析
savePath := filepath.Join("snap", "playback", streamPath)
os.MkdirAll(savePath, 0755)
savePath = strings.ReplaceAll(savePath, "/", "_")
os.MkdirAll(savePath, 0755)
// 立即返回成功响应,表示任务已接收
response := BatchSnapResponse{
@@ -666,6 +687,14 @@ func (p *SnapPlugin) executePlayBackSnapTask(streamPath string, startTime, endTi
taskStartTime := time.Now()
p.Info("playback snap task started", "streamPath", streamPath, "startTime", startTime, "endTime", endTime)
// 必须有全局存储且为本地存储,便于 ffmpeg 输出到文件系统
st := p.Server.Storage
localStorage, ok := st.(*storage.LocalStorage)
if !ok || st == nil {
p.Error("playback snap aborted: storage is not LocalStorage")
return
}
// 从数据库中查询指定时间范围内的MP4录像文件
var streams []m7s.RecordStream
queryRecord := m7s.RecordStream{
@@ -688,17 +717,39 @@ func (p *SnapPlugin) executePlayBackSnapTask(streamPath string, startTime, endTi
return streams[i].StartTime.Before(streams[j].StartTime)
})
// 检查全局存储是否可用
if p.Server.Storage == nil {
p.Error("global storage not initialized")
return
}
// 全局截图时间点列表
var allSnapTimes []time.Time
// 如果颜粒度小于等于0则对每个文件提取关键帧
if granularity <= 0 {
// 对每个文件分别提取关键帧
for _, stream := range streams {
// 检查文件是否存在
if _, err := os.Stat(stream.FilePath); os.IsNotExist(err) {
p.Warn("mp4 file not found", "path", stream.FilePath)
continue
// 如果文件不存在,尝试从全局存储获取完整路径
var absFilePath string
if localStorage, ok := p.Server.Storage.(*storage.LocalStorage); ok {
// 使用本地存储的方法根据存储级别获取完整路径
absFilePath = localStorage.GetFullPath(stream.FilePath, stream.StorageLevel)
} else {
// 非本地存储,尝试使用 GetURL 获取路径
url, err := p.Server.Storage.GetURL(context.Background(), stream.FilePath)
if err != nil {
p.Warn("failed to get file URL from storage", "path", stream.FilePath, "err", err)
continue
}
absFilePath = url
}
stream.FilePath = absFilePath
if _, err := os.Stat(stream.FilePath); os.IsNotExist(err) {
p.Warn("mp4 file not found", "path", stream.FilePath)
continue
}
}
// 计算此文件的有效时间范围(与请求时间范围的交集)
@@ -775,9 +826,26 @@ func (p *SnapPlugin) executePlayBackSnapTask(streamPath string, startTime, endTi
// 检查文件是否存在
if _, err := os.Stat(targetStream.FilePath); os.IsNotExist(err) {
p.Warn("mp4 file not found", "path", targetStream.FilePath)
failCount++
continue
// 如果文件不存在,尝试从全局存储获取完整路径
if localStorage, ok := p.Server.Storage.(*storage.LocalStorage); ok {
// 使用本地存储的方法根据存储级别获取完整路径
targetStream.FilePath = localStorage.GetFullPath(targetStream.FilePath, targetStream.StorageLevel)
} else {
// 非本地存储,尝试使用 GetURL 获取路径
url, err := p.Server.Storage.GetURL(context.Background(), targetStream.FilePath)
if err != nil {
p.Warn("mp4 file not found and failed to get URL", "path", targetStream.FilePath, "err", err)
failCount++
continue
}
targetStream.FilePath = url
}
// 再次检查文件是否存在
if _, err := os.Stat(targetStream.FilePath); os.IsNotExist(err) {
p.Warn("mp4 file not found", "path", targetStream.FilePath)
failCount++
continue
}
}
// 计算在文件中的时间偏移(毫秒)
@@ -824,10 +892,16 @@ func (p *SnapPlugin) executePlayBackSnapTask(streamPath string, startTime, endTi
snapTime.Format("20060102150405"),
granularityInfo)
filename = strings.ReplaceAll(filename, "/", "_")
filePath := filepath.Join(savePath, filename)
relPath := filepath.Join(savePath, filename)
fullPath := localStorage.GetFullPath(relPath, 1)
if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
p.Error("create snapshot directory failed", "err", err, "path", fullPath)
failCount++
continue
}
// 调用截图函数
err := p.snapFromMP4(targetStream.FilePath, filePath, timeOffset)
// 调用截图函数输出到本地完整路径
err := p.snapFromMP4(targetStream.FilePath, fullPath, timeOffset)
if err != nil {
p.Error("playback snap failed", "error", err.Error(), "time", snapTime.Format(time.RFC3339))
failCount++
@@ -840,7 +914,7 @@ func (p *SnapPlugin) executePlayBackSnapTask(streamPath string, startTime, endTi
StreamName: streamPath,
SnapMode: 4, // 回放截图模式
SnapTime: snapTime,
SnapPath: filePath,
SnapPath: relPath,
}
if err := p.DB.Create(&record).Error; err != nil {
p.Error("save playback snapshot record failed", "error", err.Error())
@@ -913,18 +987,18 @@ func (p *SnapPlugin) extractKeyFrameTimes(mp4FilePath string, startTime, endTime
// 获取MP4文件的开始时间信息
// 注意ffprobe返回的时间戳是相对于文件开始的秒数
// 我们需要将其转换为绝对时间
fileStartTimeUnix := time.Time{}
var fileStartTime time.Time
// 使用数据库中记录的文件开始时间
// 查询数据库获取文件信息
var fileInfo m7s.RecordStream
if err := p.DB.Where("file_path = ?", mp4FilePath).First(&fileInfo).Error; err == nil {
fileStartTimeUnix = fileInfo.StartTime
fileStartTime = fileInfo.StartTime
} else {
p.Warn("failed to get file start time from database, using request start time", "error", err.Error())
fileStartTimeUnix = startTime
fileStartTime = startTime
}
p.Info("file start time", "time", fileStartTimeUnix.Format(time.RFC3339))
p.Info("file start time", "time", fileStartTime.Format(time.RFC3339))
// 存储关键帧时间点
var keyFrameTimes []time.Time
@@ -944,7 +1018,7 @@ func (p *SnapPlugin) extractKeyFrameTimes(mp4FilePath string, startTime, endTime
}
// 计算实际时间:文件开始时间 + 偏移秒数
frameTime := fileStartTimeUnix.Add(time.Duration(timeOffsetSec * float64(time.Second)))
frameTime := fileStartTime.Add(time.Duration(timeOffsetSec * float64(time.Second)))
// 只保留在请求时间范围内的关键帧
if (frameTime.Equal(startTime) || frameTime.After(startTime)) &&

View File

@@ -74,7 +74,10 @@ func (r *DefaultRecorder) CreateStream(start time.Time, customFileName func(*Rec
filePath := customFileName(recordJob)
var storageType string
recordJob.storage, storageType = r.createStorage(recordJob.RecConf.Storage)
recordJob.storage = recordJob.Plugin.Server.Storage
if recordJob.storage != nil {
storageType = recordJob.storage.GetKey()
}
if recordJob.storage == nil {
return fmt.Errorf("storage config is required")

View File

@@ -15,6 +15,8 @@ import (
"sync"
"time"
"m7s.live/v5/pkg/storage"
"gopkg.in/yaml.v3"
"github.com/shirou/gopsutil/v4/cpu"
@@ -56,8 +58,9 @@ var (
type (
ServerConfig struct {
FatalDir string `default:"fatal" desc:""`
PulseInterval time.Duration `default:"5s" desc:"心跳事件间隔"` //心跳事件间隔
DisableAll bool `default:"false" desc:"禁用所有插件"` //禁用所有插件
PulseInterval time.Duration `default:"5s" desc:"心跳事件间隔"` //心跳事件间隔
DisableAll bool `default:"false" desc:"禁用所有插件"` //禁用所有插件
Armed bool `default:"false" desc:"布防状态,true=布防(启用录像),false=撤防(禁用录像)"` //布防状态
StreamAlias map[config.Regexp]string `desc:"流别名"`
Location map[config.Regexp]string `desc:"HTTP路由转发规则,key为正则表达式,value为目标地址"`
PullProxy []*PullProxyConfig
@@ -75,6 +78,7 @@ type (
Role string `default:"user" desc:"角色,可选值:admin,user"`
} `desc:"用户列表,仅在启用登录机制时生效"`
} `desc:"管理员界面配置"`
Storage map[string]any
}
WaitStream struct {
StreamPath string
@@ -122,6 +126,7 @@ type (
configFileContent []byte
disabledPlugins []*Plugin
prometheusDesc prometheusDesc
Storage storage.Storage
}
CheckSubWaitTimeout struct {
task.TickTask
@@ -265,6 +270,7 @@ func (s *Server) Start() (err error) {
s.Config.ParseUserFile(cg["global"])
}
s.LogHandler.SetLevel(ParseLevel(s.config.LogLevel))
s.initStorage()
err = debug.SetCrashOutput(util.InitFatalLog(s.FatalDir), debug.CrashOptions{})
if err != nil {
s.Error("SetCrashOutput", "error", err)
@@ -625,3 +631,23 @@ func (s *Server) OnSubscribe(streamPath string, args url.Values) {
}
}
}
// initStorage 创建全局存储实例,失败时回落到本地存储(空配置)
func (s *Server) initStorage() {
for t, conf := range s.ServerConfig.Storage {
st, err := storage.CreateStorage(t, conf)
if err == nil {
s.Storage = st
s.Info("global storage created", "type", t)
return
}
s.Warn("create storage failed", "type", t, "err", err)
}
// 兜底local 需要路径,这里用当前目录
if st, err := storage.CreateStorage("local", "."); err == nil {
s.Storage = st
s.Info("fallback to local storage", "path", ".")
} else {
s.Error("fallback local storage failed", "err", err)
}
}