初始

library/upload_chunk/consts.go

@@ -0,0 +1,7 @@
+package upload_chunk
+
+const (
+	Tmp          = "./resource/public/big_file/" // 文件保存目录
+	RelativePath = "big_file/"                   // 返回前端的相对路径
+	FileName     = "upfile"                      // 上传文件的文件名
+)

library/upload_chunk/helper.go

@@ -0,0 +1,81 @@
+package upload_chunk
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+)
+
+// 解析文件目录，文件前缀及文件后缀
+func parseFilePath(filename string) (dir, prefix, suffix string) {
+	filenameall := path.Base(filename)                        //不含目录的文件名
+	filesuffix := path.Ext(filename)                          // 后缀
+	fileprefix := strings.TrimSuffix(filenameall, filesuffix) // 前缀
+
+	dir = strings.TrimRight(filename, filenameall) // 文件目录
+	prefix = fileprefix                            // 文件前缀
+	suffix = filesuffix                            // 文件后缀
+	return
+}
+
+// 创建文件
+func createFile(fullFilePath string) (ok bool, err error) {
+	ok = fileExists(fullFilePath)
+	if ok {
+		return
+	}
+	dir, _, _ := parseFilePath(fullFilePath)
+	if dir != "" {
+		err = mkDir(dir)
+		if err != nil {
+			return
+		}
+	}
+	newFile, err := os.Create(fullFilePath)
+	defer newFile.Close()
+	if err != nil {
+		return
+	}
+	return
+}
+
+// 创建文件夹
+func mkDir(path string) (err error) {
+	if !fileExists(path) {
+		if err = os.MkdirAll(path, os.ModePerm); err != nil {
+			return errors.New(fmt.Sprintf(`os.MkdirAll failed for path "%s" with perm "%d" , err : %v`, path, os.ModePerm, err))
+		}
+		return nil
+	}
+
+	return
+}
+
+// 判断所给路径是否为文件夹
+func IsDir(path string) bool {
+	s, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return s.IsDir()
+}
+
+// 是否文件
+func IsFile(path string) bool {
+	return !IsDir(path)
+}
+
+// 判断文件或文件夹是否存在
+func fileExists(path string) bool {
+	if stat, err := os.Stat(path); stat != nil && !os.IsNotExist(err) {
+		return true
+	}
+	return false
+}
+
+// 生成分片文件名
+func getChunkFilename(identifier, chunkNumber string) string {
+	return fmt.Sprintf("uploader-%s.%s", identifier, chunkNumber)
+}

library/upload_chunk/option.go

@@ -0,0 +1,241 @@
+package upload_chunk
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"strconv"
+)
+
+type checkErr func() error
+
+type UploadReq struct {
+	ChunkNumber      int    `json:"chunkNumber"`      // 分片序列号
+	ChunkSize        int    `json:"chunkSize"`        // 分片大小
+	CurrentChunkSize int    `json:"currentChunkSize"` // 当前分片大小
+	TotalSize        int64  `json:"totalSize"`        // 文件总大小
+	Identifier       string `json:"identifier"`       // 标识
+	Filename         string `json:"filename"`         // 文件名
+	RelativePath     string `json:"relativePath"`     //文件夹上传的时候文件的相对路径属性
+	TotalChunks      int    `json:"totalChunks"`      // 分片总数
+	File             multipart.File
+	FileHeader       *multipart.FileHeader
+
+	verification map[interface{}][]checkErr
+}
+
+func (u *UploadReq) verificationInit() {
+	if u.verification == nil {
+		u.verification = make(map[interface{}][]checkErr)
+	}
+}
+
+func (u *UploadReq) MustChunkNumber() *UploadReq {
+	u.verificationInit()
+	u.verification["ChunkNumber"] = append(u.verification["ChunkNumber"], func() error {
+		if u.ChunkNumber == 0 {
+			return errors.New("ChunkNumber不能为空")
+		}
+		return nil
+	})
+
+	return u
+}
+
+func (u *UploadReq) MustChunkSize() *UploadReq {
+	u.verificationInit()
+	u.verification["ChunkSize"] = append(u.verification["ChunkSize"], func() error {
+		if u.ChunkSize == 0 {
+			return errors.New("ChunkSize不能为空")
+		}
+		return nil
+	})
+
+	return u
+}
+
+func (u *UploadReq) MustCurrentChunkSize() *UploadReq {
+	u.verificationInit()
+	u.verification["CurrentChunkSize"] = append(u.verification["CurrentChunkSize"], func() error {
+		if u.CurrentChunkSize == 0 {
+			return errors.New("CurrentChunkSize不能为空")
+		}
+		return nil
+	})
+	return u
+}
+
+func (u *UploadReq) MustTotalSize() *UploadReq {
+	u.verificationInit()
+	u.verification["TotalSize"] = append(u.verification["TotalSize"], func() error {
+		if u.TotalSize == 0 {
+			return errors.New("TotalSize不能为空")
+		}
+		return nil
+	})
+	return u
+}
+
+func (u *UploadReq) MustIdentifier() *UploadReq {
+	u.verificationInit()
+	u.verification["Identifier"] = append(u.verification["Identifier"], func() error {
+		if u.Identifier == "" {
+			return errors.New("Identifier不能为空")
+		}
+		return nil
+	})
+	return u
+}
+
+func (u *UploadReq) MustFilename() *UploadReq {
+	u.verificationInit()
+	u.verification["Filename"] = append(u.verification["Filename"], func() error {
+		if u.Filename == "" {
+			return errors.New("Filename不能为空")
+		}
+		return nil
+	})
+	return u
+}
+
+func (u *UploadReq) MustTotalChunks() *UploadReq {
+	u.verificationInit()
+	u.verification["TotalChunks"] = append(u.verification["TotalChunks"], func() error {
+		if u.TotalChunks == 0 {
+			return errors.New("TotalChunks不能为空")
+		}
+		return nil
+	})
+	return u
+}
+
+func (u *UploadReq) MustFile() *UploadReq {
+	u.verificationInit()
+	u.verification["File"] = append(u.verification["File"], func() error {
+		if u.File == nil || u.FileHeader == nil {
+			return errors.New("File不能为空")
+		}
+		return nil
+	})
+	return u
+}
+
+func (u *UploadReq) Check() (err error) {
+	for _, item := range u.verification {
+		for _, fn := range item {
+			err = fn()
+			if err != nil {
+				return
+			}
+
+		}
+	}
+	return
+}
+
+func (u *UploadReq) Bind(r *http.Request) (err error) {
+	if r.Header.Get("Content-Type") == "application/json" {
+		l := r.ContentLength
+		if l == 0 {
+			return
+		}
+		buf := make([]byte, l)
+		_, err = r.Body.Read(buf)
+
+		if err != nil && err != io.EOF {
+			return
+		}
+		err = json.Unmarshal(buf, u)
+		if err != nil {
+			return
+		}
+
+	} else {
+		chunkNumber := r.FormValue("chunkNumber")
+		if chunkNumber != "" {
+			u.ChunkNumber, _ = strconv.Atoi(chunkNumber)
+		}
+
+		chunkSize := r.FormValue("chunkSize")
+		if chunkSize != "" {
+			u.ChunkSize, _ = strconv.Atoi(chunkSize)
+		}
+
+		currentChunkSize := r.FormValue("currentChunkSize")
+		if currentChunkSize != "" {
+			u.CurrentChunkSize, _ = strconv.Atoi(currentChunkSize)
+		}
+
+		totalSize := r.FormValue("totalSize")
+		if totalSize != "" {
+			u.TotalSize, _ = strconv.ParseInt(totalSize, 10, 64)
+		}
+
+		identifier := r.FormValue("identifier")
+		if identifier != "" {
+			u.Identifier = identifier
+		}
+
+		filename := r.FormValue("filename")
+		if filename != "" {
+			u.Filename = filename
+		}
+
+		relativePath := r.FormValue("relativePath")
+		if relativePath != "" {
+			u.RelativePath = relativePath
+		}
+
+		totalChunks := r.FormValue("totalChunks")
+		if totalChunks != "" {
+			u.TotalChunks, _ = strconv.Atoi(totalChunks)
+		}
+	}
+
+	u.File, u.FileHeader, _ = r.FormFile(FileName)
+	return
+}
+
+type BaseRes struct {
+	Filename  string `json:"filename"`  // 文件名
+	TotalSize int64  `json:"totalSize"` // 文件总大小
+	Url       string `json:"url"`       // 上传文件路径
+}
+
+// 返回前端的相对路径
+func (b *BaseRes) RelativePath() {
+	_, prefix, suffix := parseFilePath(b.Url)
+	b.Url = RelativePath + prefix + suffix
+}
+
+type UpLoadRes struct {
+	BaseRes
+	NeedMerge   bool   `json:"needMerge"`   // 是否合并文件
+	Identifier  string `json:"identifier"`  // 标识
+	TotalChunks int    `json:"totalChunks"` // 分片总数
+}
+
+type CheckRes struct {
+	SkipUpload bool  `json:"skipUpload"` // 秒传
+	Uploaded   []int `json:"uploaded"`   // 已上传过的分片
+	BaseRes
+}
+
+func (r *CheckRes) HasUploaded(i int) (ok bool) {
+	for _, item := range r.Uploaded {
+		if item == i {
+			return true
+		}
+	}
+	return
+}
+
+func (r *CheckRes) HasFirst() (ok bool) {
+	return r.HasUploaded(1)
+}
+
+type MergeRes struct {
+	BaseRes
+}

library/upload_chunk/upload_chunk.go

@@ -0,0 +1,309 @@
+package upload_chunk
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"mime/multipart"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type UploadChunk struct{}
+
+// 检查分片
+func (u *UploadChunk) CheckChunk(uploadReq UploadReq) (result *CheckRes, err error) {
+
+	err = uploadReq.MustIdentifier().MustTotalChunks().MustFilename().MustTotalSize().Check()
+	if err != nil {
+		return
+	}
+	identifier, totalChunks, filename, totalSize := uploadReq.Identifier, uploadReq.TotalChunks, uploadReq.Filename, uploadReq.TotalSize
+
+	dir, prefix, suffix := parseFilePath(filename)
+	_, _, _ = dir, prefix, suffix
+
+	if !strings.Contains(suffix, ".") {
+		err = errors.New("文件名解析错误")
+		return
+	}
+
+	result = &CheckRes{}
+
+	// 秒传
+	resultFilePath := u.Tmp() + identifier + suffix
+	if fileExists(resultFilePath) {
+		result.SkipUpload = true
+		result.Url = resultFilePath
+		result.Filename = filename
+		result.TotalSize = totalSize
+		result.RelativePath()
+		return
+	}
+
+	// 断点续传
+	for i := 1; i <= totalChunks; i++ {
+		chunkFilePath := u.chunkPath(identifier, strconv.FormatInt(int64(i), 10))
+		if fileExists(chunkFilePath) {
+			result.Uploaded = append(result.Uploaded, i)
+		}
+	}
+
+	return
+}
+
+// 合并文件
+func (u *UploadChunk) MergeChunk(uploadReq UploadReq) (result *MergeRes, err error) {
+	err = uploadReq.MustIdentifier().MustTotalChunks().MustTotalSize().MustFilename().Check()
+	if err != nil {
+		return
+	}
+	identifier, totalChunks, totalSize, filename := uploadReq.Identifier, uploadReq.TotalChunks, uploadReq.TotalSize, uploadReq.Filename
+
+	_, _, suffix := parseFilePath(filename)
+	if !strings.Contains(suffix, ".") {
+		return nil, errors.New("文件名解析错误")
+	}
+
+	// 合并后的文件
+	resultFilePath := u.Tmp() + identifier + suffix
+	ok, err := createFile(resultFilePath)
+	if err != nil {
+		return
+	}
+	result = new(MergeRes)
+	// 文件已存在
+	if ok {
+		result.Url = resultFilePath
+		result.Filename = filename
+		result.TotalSize = totalSize
+		result.RelativePath()
+		return
+	}
+
+	// 检查分片文件是否完整
+	ok, err = u.checkChunkAll(identifier, totalChunks, totalSize)
+	if err != nil {
+		err = fmt.Errorf("分片文件检查错误:%s", err)
+		return
+	}
+
+	if !ok {
+		err = errors.New("分片文件不完整")
+		return
+	}
+
+	var chunkSize int64
+	var wg sync.WaitGroup
+	ch := make(chan struct{}, 10)
+	for i := 1; i <= totalChunks; i++ {
+		// 分片文件
+		filePath := u.chunkPath(identifier, fmt.Sprintf("%d", i))
+		if chunkSize == 0 {
+			fi, _ := os.Stat(filePath)
+			if chunkSize = fi.Size(); chunkSize == 0 {
+				err = errors.New("分片文件大小为0")
+				return
+			}
+		}
+		ch <- struct{}{}
+		wg.Add(1)
+		go func(i int) {
+			defer func() {
+				<-ch
+				wg.Done()
+			}()
+			uploadChunk := &UploadChunk{}
+			err = uploadChunk.mergeFile(filePath, resultFilePath, chunkSize*int64(i-1))
+			if err != nil {
+				log.Println(err)
+				return
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	result.Url = resultFilePath
+	result.Filename = filename
+	result.TotalSize = totalSize
+	result.RelativePath()
+	return
+}
+
+// 上传分片文件
+func (u *UploadChunk) Upload(uploadReq UploadReq) (result *UpLoadRes, err error) {
+	err = uploadReq.MustTotalChunks().MustChunkNumber().MustIdentifier().MustFile().MustTotalSize().MustFilename().Check()
+	if err != nil {
+		return
+	}
+	totalChunks, chunkNumber, identifier, upFile, filename := uploadReq.TotalChunks, uploadReq.ChunkNumber, uploadReq.Identifier, uploadReq.File, uploadReq.Filename
+	var fullFilePath string
+	if totalChunks > 1 {
+		// 分片文件路径
+		fullFilePath = u.chunkPath(identifier, strconv.Itoa(chunkNumber))
+	} else {
+		_, _, suffix := parseFilePath(filename)
+		if !strings.Contains(suffix, ".") {
+			return nil, errors.New("文件名解析错误")
+		}
+		fullFilePath = u.Tmp() + identifier + suffix
+	}
+
+	// 创建文件
+	ok, err := createFile(fullFilePath)
+
+	if err != nil {
+		return
+	}
+	if ok {
+		// 文件已经存在
+		result = &UpLoadRes{
+			BaseRes: BaseRes{
+				Filename:  uploadReq.Filename,
+				TotalSize: uploadReq.TotalSize,
+				Url:       fullFilePath,
+			},
+			NeedMerge:   totalChunks > 1,
+			Identifier:  uploadReq.Identifier,
+			TotalChunks: uploadReq.TotalChunks,
+		}
+		result.RelativePath()
+		return
+	}
+
+	// 打开分片文件
+	file, err := os.OpenFile(fullFilePath, os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return
+	}
+	defer file.Close()
+	// 文件偏移量
+	var fi os.FileInfo
+	fi, err = os.Stat(fullFilePath)
+	if err != nil {
+		return
+	}
+	start := fi.Size()
+	// 写入分片文件
+	_, err = u.writeFile(upFile, start, file, start)
+	if err != nil {
+		return
+	}
+	result = &UpLoadRes{
+		BaseRes: BaseRes{
+			Filename:  uploadReq.Filename,
+			TotalSize: uploadReq.TotalSize,
+			Url:       fullFilePath,
+		},
+		NeedMerge:   totalChunks > 1,
+		Identifier:  uploadReq.Identifier,
+		TotalChunks: uploadReq.TotalChunks,
+	}
+	result.RelativePath()
+	return
+}
+
+func (u *UploadChunk) Tmp() string {
+	return Tmp
+}
+
+// 合并文件
+func (u *UploadChunk) mergeFile(chunkFile, mergeFile string, offset int64) (err error) {
+
+	// 合并后的文件
+	file, err := os.OpenFile(mergeFile, os.O_CREATE|os.O_WRONLY, os.ModePerm)
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	_, err = file.Seek(offset, 0)
+	if err != nil {
+		return
+	}
+	// 分片文件
+	chunkFileObj, err := os.Open(chunkFile)
+	if err != nil {
+		return
+	}
+	defer chunkFileObj.Close()
+	// 写入数据
+	data := make([]byte, 1024, 1024)
+
+	for {
+		tal, e := chunkFileObj.Read(data)
+		if e == io.EOF {
+			chunkFileObj.Close()
+			os.Remove(chunkFile)
+			break
+		}
+		_, e = file.Write(data[:tal])
+		if e != nil {
+			return e
+		}
+	}
+	return
+}
+
+// 检查分片文件是否完整
+func (u *UploadChunk) checkChunkAll(identifier string, totalChunks int, totalSize int64) (ok bool, err error) {
+	if identifier == "" || totalChunks == 0 {
+		return false, errors.New("checkChunkAll 参数错误")
+	}
+	var _totalSize int64
+	for i := 1; i <= totalChunks; i++ {
+		filePath := u.chunkPath(identifier, fmt.Sprintf("%d", i))
+		fi, e := os.Stat(filePath)
+		if e != nil {
+			return false, e
+		}
+		_totalSize += fi.Size()
+	}
+
+	return _totalSize == totalSize, nil
+}
+
+// 获取分片文件路径
+func (u *UploadChunk) chunkPath(identifier string, chunkNumber string) string {
+	return fmt.Sprintf("%s%s", u.Tmp(), getChunkFilename(identifier, chunkNumber))
+}
+
+// 检查文件完整性
+func (u *UploadChunk) verifyFileSize(fullFilePath string, size int64) (ok bool, err error) {
+	fi, err := os.Stat(fullFilePath)
+	if err != nil {
+		return
+	}
+	if fi.Size() == size {
+		return true, nil
+	}
+	return false, nil
+}
+
+// 写入文件
+func (u *UploadChunk) writeFile(upfile multipart.File, upSeek int64, file *os.File, fSeek int64) (result int, err error) {
+	// 上传文件大小记录
+	fileSzie := 0
+	// 设置上传偏移量
+	upfile.Seek(upSeek, 0)
+	// 设置文件偏移量
+	file.Seek(fSeek, 0)
+	data := make([]byte, 1024, 1024)
+	for {
+		total, e := upfile.Read(data)
+		if e == io.EOF {
+			// 文件复制完毕
+			break
+		}
+		l, e := file.Write(data[:total])
+		if e != nil {
+			return 0, errors.New("文件上传失败")
+		}
+		// 记录上传长度
+		fileSzie += l
+	}
+	return fileSzie, nil
+}