优化分片上传

2025-12-24 08:12:55 +08:00 · 2025-07-21 02:37:07 +08:00
parent 446c9bef60
commit 5d790bb913
3 changed files with 155 additions and 88 deletions
--- a/admin/src/utils/FileUploader.ts
+++ b/admin/src/utils/FileUploader.ts
@@ -12,17 +12,25 @@ export default class FileUploader {
    file: File
    fileMd5: string
    fileName: string
+    fileSize: number
    chunkSize: number = 1024 * 1024 // 1MB
    chunkCount: number = 0
+
+    private startChunkIndex = -1
    onSuccess: FileUploaderOptions['onSuccess'] = () => {}
-    onError: FileUploaderOptions['onError'] = () => {}
-    onUploadProgress(chunkIndex, percent) {
-        console.log(`当前分片: ${chunkIndex}，进度: ${percent}%`)
+    onError: FileUploaderOptions['onError'] = (error: Error) => {
+        console.log(error)
+    }
+    onUploadProgress(chunkIndex: number, chunkLoaded: number, chunkTotal: number, loaded: number) {
+        console.log(
+            `当前分片: ${chunkIndex}/${this.chunkCount},分片进度${chunkLoaded}/${chunkTotal}，总进度: ${loaded}/${this.fileSize}`
+        )
    }

    constructor(file: File, options: FileUploaderOptions) {
        this.file = file
        this.fileName = file.name
+        this.fileSize = file.size

        if (options?.chunkSize) {
            this.chunkSize = options.chunkSize
@@ -58,13 +66,20 @@ export default class FileUploader {
    async start() {
        try {
            const arrayBuffer = await this.readerFile(this.file)
-            this.fileMd5 = this.getMd5(arrayBuffer)
+            const fileMd5 = this.getMd5(arrayBuffer)
+            this.fileMd5 = fileMd5 + '_' + this.fileSize
            const isExistFilePath = await this.checkFileExist()
+
            if (isExistFilePath) {
                this.complete(isExistFilePath)
                return
            }
-            this.splitChunks()
+            const hasChunk = await this.getHasChunk()
+            this.startChunkIndex = hasChunk && hasChunk.length ? Math.max(...hasChunk) : 0
+            console.log('hasChunk', hasChunk)
+
+            await this.splitChunks()
+            await this.mergeChunk()
        } catch (error) {
            this.onError(error)
        }
@@ -78,79 +93,81 @@ export default class FileUploader {
    }
    // 检查上传状态
    getMd5(arrayBuffer: ArrayBuffer): string {
+        console.time('SparkMD5')
        const spark = new SparkMD5.ArrayBuffer()
        spark.append(arrayBuffer)
-        return spark.end()
+        const hash = spark.end()
+        console.timeEnd('SparkMD5')
+        return hash
    }
    // 检查文件是否存在,可实现秒传
    async checkFileExist(): Promise<string> {
-        try {
-            /* 检查文件是否存在 */
-            const res = await axios.get('/api/admin/common/uploadChunk/CheckFileExist', {
-                params: {
-                    fileMd5: this.fileMd5,
-                    fileName: this.file.name
-                }
-            })
-            console.log('Init', res)
-
-            if (res.data.code === 200 && res.data.data) {
-                return res.data.data
+        /* 检查文件是否存在 */
+        const res = await axios.get('/api/admin/common/uploadChunk/CheckFileExist', {
+            params: {
+                fileMd5: this.fileMd5,
+                fileName: this.fileName
            }
-            return ''
-        } catch (error) {
-            return ''
+        })
+        if (res.data.code === 200) {
+            return res.data.data
        }
+        throw new Error(res.data.message)
    }
+    async getHasChunk(): Promise<number[]> {
+        const hasChunkRes = await axios.get('/api/admin/common/uploadChunk/HasChunk', {
+            params: {
+                fileMd5: this.fileMd5,
+                chunkSize: this.chunkSize,
+                fileName: this.fileName
+            }
+        })
+        console.log('HasChunk', hasChunkRes)
+
+        if (hasChunkRes.data.code === 200) {
+            return hasChunkRes.data.data || []
+        }
+        throw new Error(hasChunkRes.data.message)
+    }
+
    async splitChunks() {
-        for (let index = 0; index < this.chunkCount; index++) {
+        for (let index = this.startChunkIndex; index < this.chunkCount; index++) {
            const chunkStart = index * this.chunkSize
            const chunkEnd = Math.min(chunkStart + this.chunkSize, this.file.size)
            const chunk = this.file.slice(chunkStart, chunkEnd)
            await this.uploadChunk(this.fileMd5, index, chunk)
        }
-        await this.mergeChunk()
    }
    async uploadChunk(fileMd5: string, index: number, chunk: Blob) {
        try {
-            const checkResult = await axios.get('/api/admin/common/uploadChunk/CheckChunkExist', {
-                params: {
-                    index,
-                    fileMd5
+            const formData = new FormData()
+            formData.append('fileMd5', fileMd5)
+            formData.append('chunk', chunk)
+            formData.append('chunkSize', String(this.chunkSize))
+            formData.append('index', String(index))
+
+            const result = await axios.post('/api/admin/common/uploadChunk/UploadChunk', formData, {
+                onUploadProgress: (progressEvent) => {
+                    // const percentCompleted = (
+                    //     ((this.chunkSize * index + progressEvent.loaded) * 100) /
+                    //     this.fileSize
+                    // ).toFixed(3)
+                    const loaded = this.chunkSize * index + progressEvent.loaded //TODO progressEvent.loaded体积比文件大，不能直接相加
+
+                    this.onUploadProgress(index, progressEvent.loaded, progressEvent.total, loaded)
                }
            })
-            console.log('checkResult', checkResult)
+            chunk = null
+            console.log('result', result)

-            if (checkResult.data.code === 200) {
-                console.log(`分片 ${index + 1}/${this.chunkCount} 已存在`)
-                return
-            } else if (checkResult.data.code === 500) {
-                const formData = new FormData()
-                formData.append('chunk', chunk)
-                formData.append('index', String(index))
-                formData.append('fileMd5', fileMd5)
-                const result = await axios.post(
-                    '/api/admin/common/uploadChunk/UploadChunk',
-                    formData,
-                    {
-                        onUploadProgress: (progressEvent) => {
-                            const percentCompleted = Math.round(
-                                (progressEvent.loaded * 100) / progressEvent.total
-                            )
-                            this.onUploadProgress(index, percentCompleted)
-                        }
-                    }
-                )
-                console.log('result', result)
-
-                if (result.data.code === 200) {
-                    console.log(`分片 ${index + 1}/${this.chunkCount} 上传成功`)
-                } else {
-                    console.error(`分片 ${index + 1}/${this.chunkCount} 上传失败: ${result}`)
-                    // break
-                }
+            if (result.data.code === 200) {
+                console.log(`分片 ${index + 1}/${this.chunkCount} 上传成功`)
+            } else {
+                console.error(`分片 ${index + 1}/${this.chunkCount} 上传失败: ${result}`)
+                // break
            }
        } catch (error) {
+            chunk = null
            console.error(`分片 ${index + 1}/${this.chunkCount} 上传失败: ${error}`)
            this.onError(error)
        }
@@ -159,8 +176,9 @@ export default class FileUploader {
        try {
            const res = await axios.post('/api/admin/common/uploadChunk/MergeChunk', {
                fileMd5: this.fileMd5,
-                fileName: this.file.name,
-                chunkCount: this.chunkCount
+                fileName: this.fileName,
+                chunkCount: this.chunkCount,
+                chunkSize: this.chunkSize
            })
            if (res.data.code === 200) {
                console.log('合并分片成功')
--- a/server/controller/admin_ctl/commonController/uploadChunk.go
+++ b/server/controller/admin_ctl/commonController/uploadChunk.go
@@ -2,9 +2,7 @@ package commonController

 import (
 	"fmt"
-	"net/url"
 	"os"
-	"regexp"
 	"x_admin/core/response"
 	"x_admin/service/commonService"

@@ -14,14 +12,16 @@ import (
 func UploadChunkRoute(rg *gin.RouterGroup) {
 	handle := uploadChunkHandler{
 		uploadPath: "./uploads",
-		tmpPath:    "./tmp",
+		tmpPath:    "./uploads/.tmp",
 	}
 	os.MkdirAll(handle.uploadPath, 0755)
 	os.MkdirAll(handle.tmpPath, 0755)

 	rg = rg.Group("/common")
 	rg.GET("/uploadChunk/CheckFileExist", handle.CheckFileExist)
-	rg.GET("/uploadChunk/CheckChunkExist", handle.CheckChunkExist)
+	// rg.GET("/uploadChunk/CheckChunkExist", handle.CheckChunkExist)
+	rg.GET("/uploadChunk/HasChunk", handle.HasChunk)
+
 	rg.POST("/uploadChunk/UploadChunk", handle.UploadChunk)
 	rg.POST("/uploadChunk/MergeChunk", handle.MergeChunk)
 }
@@ -31,16 +31,27 @@ type uploadChunkHandler struct {
 	tmpPath    string
 }

+func (uh uploadChunkHandler) getFilePath(fileMd5 string) string {
+	return fmt.Sprintf("%s/%s", uh.uploadPath, fileMd5)
+}
+func (uh uploadChunkHandler) getChunkDir(fileMd5 string, chunkSize string) string {
+	return fmt.Sprintf("%s/%s_%s", uh.tmpPath, fileMd5, chunkSize)
+}
+func (uh uploadChunkHandler) getChunkPath(fileMd5 string, chunkSize string, index string) string {
+	return fmt.Sprintf("%s/%s_%s/%s", uh.tmpPath, fileMd5, chunkSize, index)
+}
+
 func (uh uploadChunkHandler) CheckFileExist(c *gin.Context) {
 	var fileMd5 = c.Query("fileMd5")
-	var fileName = url.QueryEscape(c.Query("fileName"))
+
+	// var fileName = url.QueryEscape(c.Query("fileName"))
 	// 正则检查MD5
-	reg := regexp.MustCompile(`^[a-fA-F0-9]{32}$`)
-	if !reg.MatchString(fileMd5) {
-		response.FailWithMsg(c, response.SystemError, "MD5格式错误")
-		return
-	}
-	var filePath = fmt.Sprintf("%s/%s_%s", uh.uploadPath, fileMd5, fileName)
+	// reg := regexp.MustCompile(`^[a-fA-F0-9_]{32+}$`)
+	// if !reg.MatchString(fileMd5) {
+	// 	response.FailWithMsg(c, response.SystemError, "文件hash错误")
+	// 	return
+	// }
+	var filePath = fmt.Sprintf("%s/%s", uh.uploadPath, fileMd5)
 	// 检查文件是否存在
 	if commonService.UploadChunkService.CheckFileExist(filePath) {
 		response.OkWithData(c, filePath)
@@ -48,27 +59,37 @@ func (uh uploadChunkHandler) CheckFileExist(c *gin.Context) {
 	}
 	response.OkWithData(c, nil)
 }
+func (uh uploadChunkHandler) HasChunk(c *gin.Context) {
+	var fileMd5 = c.Query("fileMd5")
+	var chunkSize = c.Query("chunkSize")
+	var chunkDir = uh.getChunkDir(fileMd5, chunkSize)
+	var HasChunk = commonService.UploadChunkService.HasChunk(chunkDir)
+	response.OkWithData(c, HasChunk)
+}

 // 检查chunk是否存在
-func (uh uploadChunkHandler) CheckChunkExist(c *gin.Context) {
-	fileMd5 := c.Query("fileMd5") // 上传文件的md5
-	index := c.Query("index")     // 分片序号
-	chunkPath := fmt.Sprintf("%s/%s/%s", uh.tmpPath, fileMd5, index)
-	if commonService.UploadChunkService.CheckFileExist(chunkPath) {
-		response.Ok(c)
-		return
-	}
-	response.FailWithMsg(c, response.SystemError, "分片不存在")
-}
+// func (uh uploadChunkHandler) CheckChunkExist(c *gin.Context) {
+// 	fileMd5 := c.Query("fileMd5") // 上传文件的md5
+// 	chunkSize := c.Query("chunkSize")
+// 	index := c.Query("index") // 分片序号
+// 	// chunkPath := fmt.Sprintf("%s/%s/%s", uh.tmpPath, fileMd5, index)
+// 	chunkPath := uh.getChunkPath(fileMd5, chunkSize, index)
+// 	if commonService.UploadChunkService.CheckFileExist(chunkPath) {
+// 		response.OkWithData(c, 1)
+// 		return
+// 	}
+// 	response.OkWithData(c, 0)
+// }

 // UploadChunk 上传分片
 func (uh uploadChunkHandler) UploadChunk(c *gin.Context) {
-	chunk, _ := c.FormFile("chunk")  // 分片文件
-	index := c.PostForm("index")     // 分片序号
-	fileMd5 := c.PostForm("fileMd5") // 上传文件的md5
+	chunk, _ := c.FormFile("chunk")      // 分片文件
+	chunkSize := c.PostForm("chunkSize") // 分片分割的大小
+	index := c.PostForm("index")         // 分片序号
+	fileMd5 := c.PostForm("fileMd5")     // 上传文件的md5

-	chunkDir := fmt.Sprintf("%s/%s", uh.tmpPath, fileMd5)
-	chunkPath := fmt.Sprintf("%s/%s", chunkDir, index)
+	chunkDir := uh.getChunkDir(fileMd5, chunkSize)
+	chunkPath := uh.getChunkPath(fileMd5, chunkSize, index)
 	err := commonService.UploadChunkService.UploadChunk(chunkDir, chunkPath, chunk)
 	if err != nil {
 		response.FailWithMsg(c, response.SystemError, err.Error())
@@ -81,14 +102,16 @@ func (uh uploadChunkHandler) MergeChunk(c *gin.Context) {
 		FileMd5    string `json:"fileMd5"`    // 上传文件的md5
 		FileName   string `json:"fileName"`   // 文件名
 		ChunkCount int    `json:"chunkCount"` // 分片数量
+		ChunkSize  int    `json:"chunkSize"`  // 分片分割的大小,作用：确保不同分片大小不放在同一目录
 	}
 	bindErr := c.ShouldBindJSON(&MergeChunk)
 	if bindErr != nil {
 		response.FailWithMsg(c, response.SystemError, bindErr.Error())
 		return
 	}
-	var filePath = fmt.Sprintf("%s/%s_%s", uh.uploadPath, MergeChunk.FileMd5, url.QueryEscape(MergeChunk.FileName))
-	err := commonService.UploadChunkService.MergeChunk(MergeChunk.FileMd5, filePath, MergeChunk.ChunkCount)
+	var filePath = uh.getFilePath(MergeChunk.FileMd5)
+	var chunkDir = uh.getChunkDir(MergeChunk.FileMd5, fmt.Sprintf("%d", MergeChunk.ChunkSize))
+	err := commonService.UploadChunkService.MergeChunk(chunkDir, filePath, MergeChunk.ChunkCount)
 	if err != nil {
 		response.FailWithMsg(c, response.SystemError, err.Error())
 		return
--- a/server/service/commonService/uploadChunkService.go
+++ b/server/service/commonService/uploadChunkService.go
@@ -7,6 +7,7 @@ import (
 	"mime/multipart"
 	"os"
 	"path/filepath"
+	"strconv"
 	"x_admin/util"
 )

@@ -40,9 +41,34 @@ func (upSrv uploadChunkService) UploadChunk(chunkDir string, chunkPath string, c
 	}
 	return os.WriteFile(chunkPath, chunkBytes, 0644)
 }
-func (upSrv uploadChunkService) MergeChunk(fileMd5, filePath string, chunkCount int) error {

-	chunkDir := fmt.Sprintf("./tmp/%s", fileMd5)
+// 通过文件列表中的文件名获取最大chunk
+func (upSrv uploadChunkService) HasChunk(chunkDir string) []int {
+	// chunks, err := os.ReadDir(chunkDir)
+	files, err := os.ReadDir(chunkDir)
+	var chunks []int
+	if err != nil {
+		fmt.Printf("读取目录失败: %v\n", err)
+		return chunks
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue // 跳过目录
+		}
+
+		filename := file.Name()
+		num, err := strconv.Atoi(filename) // 直接转换整个文件名
+		if err != nil {
+			continue // 跳过非数字文件名
+		}
+		chunks = append(chunks, num)
+	}
+	return chunks
+}
+func (upSrv uploadChunkService) MergeChunk(chunkDir, filePath string, chunkCount int) error {
+
+	// chunkDir := fmt.Sprintf("./tmp/%s", fileMd5)

 	chunks, err := os.ReadDir(chunkDir)
 	if err != nil {