object storage supports webdav

cmd/initializer.go

@@ -158,5 +158,9 @@ func getConfigWithoutSensitiveData(config *settings.Config) *settings.Config {
 	clonedConfig.SecretKey = "****"
 	clonedConfig.AmapApplicationSecret = "****"
 
+	if clonedConfig.WebDAVConfig != nil {
+		clonedConfig.WebDAVConfig.Password = "****"
+	}
+
 	return clonedConfig
 }

conf/ezbookkeeping.ini

@@ -115,7 +115,7 @@ log_file_max_size = 104857600
 log_file_max_days = 7
 
 [storage]
-# Object storage type, supports "local_filesystem" and "minio" currently
+# Object storage type, supports "local_filesystem", "minio" and "webdav" currently
 type = local_filesystem
 
 # For "local_filesystem" storage only, the storage root path (relative or absolute path)
@@ -139,6 +139,28 @@ minio_bucket = ezbookkeeping
 # For "minio" storage only, the root path to store files in minio
 minio_root_path = /
 
+# For "webdav" storage only, the webdav url
+webdav_url =
+
+# For "webdav" storage only, the webdav username
+webdav_username =
+
+# For "webdav" storage only, the webdav password
+webdav_password =
+
+# For "webdav" storage only, the webdav root path to store files
+webdav_root_path = /
+
+# For "webdav" storage only, requesting webdav url timeout (0 - 4294967295 milliseconds)
+# Set to 0 to disable timeout for requesting webdav url, default is 10000 (10 seconds)
+webdav_request_timeout = 10000
+
+# For "webdav" storage only, proxy for requesting webdav url, supports "system" (use system proxy), "none" (do not use proxy), or proxy URL which starts with "http://", "https://" or "socks5://", default is "system"
+webdav_proxy = system
+
+# For "webdav" storage only, set to true to skip tls verification when connect webdav
+webdav_skip_tls_verify = false
+
 [uuid]
 # Uuid generator type, supports "internal" currently
 generator_type = internal

pkg/settings/setting.go

@@ -63,6 +63,7 @@ const (
 const (
 	LocalFileSystemObjectStorageType string = "local_filesystem"
 	MinIOStorageType                 string = "minio"
+	WebDAVStorageType                string = "webdav"
 )
 
 // Uuid generator types
@@ -137,6 +138,8 @@ const (
 	defaultLogFileMaxSize uint32 = 104857600 // 100 MB
 	defaultLogFileMaxDays uint32 = 7         // days
 
+	defaultWebDAVRequestTimeout uint32 = 10000 // 10 seconds
+
 	defaultInMemoryDuplicateCheckerCleanupInterval uint32 = 60  // 1 minutes
 	defaultDuplicateSubmissionsInterval            uint32 = 300 // 5 minutes
 
@@ -195,6 +198,17 @@ type MinIOConfig struct {
 	RootPath        string
 }
 
+// WebDAVConfig represents the WebDAV setting config
+type WebDAVConfig struct {
+	Url            string
+	Username       string
+	Password       string
+	RootPath       string
+	RequestTimeout uint32
+	Proxy          string
+	SkipTLSVerify  bool
+}
+
 // TipConfig represents a tip setting config
 type TipConfig struct {
 	Enabled              bool
@@ -264,6 +278,7 @@ type Config struct {
 	StorageType         string
 	LocalFileSystemPath string
 	MinIOConfig         *MinIOConfig
+	WebDAVConfig        *WebDAVConfig
 
 	// Uuid
 	UuidGeneratorType string
@@ -697,6 +712,8 @@ func loadStorageConfiguration(config *Config, configFile *ini.File, sectionName
 		config.StorageType = LocalFileSystemObjectStorageType
 	} else if getConfigItemStringValue(configFile, sectionName, "type") == MinIOStorageType {
 		config.StorageType = MinIOStorageType
+	} else if getConfigItemStringValue(configFile, sectionName, "type") == WebDAVStorageType {
+		config.StorageType = WebDAVStorageType
 	} else {
 		return errs.ErrInvalidStorageType
 	}
@@ -718,9 +735,18 @@ func loadStorageConfiguration(config *Config, configFile *ini.File, sectionName
 	minIOConfig.SkipTLSVerify = getConfigItemBoolValue(configFile, sectionName, "minio_skip_tls_verify", false)
 	minIOConfig.Bucket = getConfigItemStringValue(configFile, sectionName, "minio_bucket")
 	minIOConfig.RootPath = getConfigItemStringValue(configFile, sectionName, "minio_root_path")
-
 	config.MinIOConfig = minIOConfig
 
+	webDAVConfig := &WebDAVConfig{}
+	webDAVConfig.Url = getConfigItemStringValue(configFile, sectionName, "webdav_url")
+	webDAVConfig.Username = getConfigItemStringValue(configFile, sectionName, "webdav_username")
+	webDAVConfig.Password = getConfigItemStringValue(configFile, sectionName, "webdav_password")
+	webDAVConfig.RootPath = getConfigItemStringValue(configFile, sectionName, "webdav_root_path")
+	webDAVConfig.RequestTimeout = getConfigItemUint32Value(configFile, sectionName, "webdav_request_timeout", defaultWebDAVRequestTimeout)
+	webDAVConfig.Proxy = getConfigItemStringValue(configFile, sectionName, "webdav_proxy", "system")
+	webDAVConfig.SkipTLSVerify = getConfigItemBoolValue(configFile, sectionName, "webdav_skip_tls_verify", false)
+	config.WebDAVConfig = webDAVConfig
+
 	return nil
 }
 

pkg/storage/bytes_slice_object.go

@@ -0,0 +1,22 @@
+package storage
+
+import (
+	"bytes"
+)
+
+// bytesSliceObject represents a byte slice object in storage
+type bytesSliceObject struct {
+	*bytes.Reader
+}
+
+// Close does nothing because it does not hold any resources that need to be released
+func (b *bytesSliceObject) Close() error {
+	return nil
+}
+
+// newByteSliceObject creates a new byte slice object from the specified byte slice
+func newByteSliceObject(data []byte) ObjectInStorage {
+	return &bytesSliceObject{
+		Reader: bytes.NewReader(data),
+	}
+}

pkg/storage/storage_container.go

@@ -86,6 +86,8 @@ func newObjectStorage(config *settings.Config, pathPrefix string) (ObjectStorage
 		return NewLocalFileSystemObjectStorage(config, pathPrefix)
 	} else if config.StorageType == settings.MinIOStorageType {
 		return NewMinIOObjectStorage(config, pathPrefix)
+	} else if config.StorageType == settings.WebDAVStorageType {
+		return NewWebDAVObjectStorage(config, pathPrefix)
 	}
 
 	return nil, errs.ErrInvalidStorageType

pkg/storage/webdav_storage.go

@@ -0,0 +1,360 @@
+package storage
+
+import (
+	"bytes"
+	"crypto/tls"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/mayswind/ezbookkeeping/pkg/core"
+	"github.com/mayswind/ezbookkeeping/pkg/errs"
+	"github.com/mayswind/ezbookkeeping/pkg/log"
+	"github.com/mayswind/ezbookkeeping/pkg/settings"
+	"github.com/mayswind/ezbookkeeping/pkg/utils"
+)
+
+// WebDAVObjectStorage represents WebDAV object storage
+type WebDAVObjectStorage struct {
+	httpClient   *http.Client
+	webDavConfig *settings.WebDAVConfig
+	rootPath     string
+}
+
+// NewWebDAVObjectStorage returns a WebDAV object storage
+func NewWebDAVObjectStorage(config *settings.Config, pathPrefix string) (*WebDAVObjectStorage, error) {
+	webDavConfig := config.WebDAVConfig
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	utils.SetProxyUrl(transport, webDavConfig.Proxy)
+
+	if webDavConfig.SkipTLSVerify {
+		transport.TLSClientConfig = &tls.Config{
+			InsecureSkipVerify: true,
+		}
+	}
+
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   time.Duration(webDavConfig.RequestTimeout) * time.Millisecond,
+	}
+
+	storage := &WebDAVObjectStorage{
+		httpClient:   client,
+		webDavConfig: webDavConfig,
+		rootPath:     webDavConfig.RootPath,
+	}
+
+	storage.rootPath = storage.getFinalPath(pathPrefix)
+	storage.rootPath = strings.ReplaceAll(storage.rootPath, "\\", "/")
+
+	ctx := core.NewNullContext()
+	exists, err := storage.directoryExists(ctx, storage.rootPath)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if !exists {
+		err := storage.createAllDirectories(ctx, "", storage.rootPath)
+
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return storage, nil
+}
+
+// Exists returns whether the file exists
+func (s *WebDAVObjectStorage) Exists(ctx core.Context, path string) (bool, error) {
+	req, err := http.NewRequest("HEAD", s.getFinalFileUrl(path), nil)
+
+	if err != nil {
+		return false, err
+	}
+
+	req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password)
+	resp, err := s.httpClient.Do(req)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.Exists] cannot check file exists, because %s", err.Error())
+		return false, err
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		return true, nil
+	} else if resp.StatusCode == http.StatusNotFound {
+		return false, nil
+	}
+
+	log.Errorf(ctx, "[webdav_storage.Exists] cannot check file exists, http status code is %d", resp.StatusCode)
+	return false, errs.ErrSystemError
+}
+
+// Read returns the object instance according to specified the file path
+func (s *WebDAVObjectStorage) Read(ctx core.Context, path string) (ObjectInStorage, error) {
+	req, err := http.NewRequest("GET", s.getFinalFileUrl(path), nil)
+
+	if err != nil {
+		return nil, err
+	}
+
+	req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password)
+	resp, err := s.httpClient.Do(req)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.Read] cannot get file, because %s", err.Error())
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.Read] cannot read response (http status code %d) body, because %s", resp.StatusCode, err.Error())
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		log.Errorf(ctx, "[webdav_storage.Read] cannot get file, http status code is %d, response is %s", resp.StatusCode, string(body))
+		return nil, errs.ErrSystemError
+	}
+
+	return newByteSliceObject(body), nil
+}
+
+// Save returns whether save the object instance successfully
+func (s *WebDAVObjectStorage) Save(ctx core.Context, path string, object ObjectInStorage) error {
+	finalPath := s.getFinalPath(path)
+	dir := strings.ReplaceAll(filepath.Dir(finalPath), "\\", "/")
+
+	exists, err := s.directoryExists(ctx, dir)
+
+	if err != nil {
+		return err
+	}
+
+	if !exists {
+		rootExists, err := s.directoryExists(ctx, s.rootPath)
+
+		if err != nil {
+			return err
+		}
+
+		if !rootExists {
+			err := s.createAllDirectories(ctx, "", s.rootPath)
+
+			if err != nil {
+				return err
+			}
+		}
+
+		err = s.createAllDirectories(ctx, s.rootPath, strings.ReplaceAll(filepath.Dir(path), "\\", "/"))
+
+		if err != nil {
+			return err
+		}
+	}
+
+	data, err := io.ReadAll(object)
+
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("PUT", s.getFinalFileUrl(path), bytes.NewReader(data))
+
+	if err != nil {
+		return err
+	}
+
+	req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password)
+	resp, err := s.httpClient.Do(req)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.Save] cannot save file, because %s", err.Error())
+		return err
+	}
+
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.Save] cannot read response (http status code %d) body, because %s", resp.StatusCode, err.Error())
+		return err
+	}
+
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+		log.Errorf(ctx, "[webdav_storage.Save] cannot save file, http status code is %d, response is %s", resp.StatusCode, string(body))
+		return errs.ErrSystemError
+	}
+
+	return nil
+}
+
+// Delete returns whether delete the object according to specified the file path successfully
+func (s *WebDAVObjectStorage) Delete(ctx core.Context, path string) error {
+	req, err := http.NewRequest("DELETE", s.getFinalFileUrl(path), nil)
+
+	if err != nil {
+		return err
+	}
+
+	req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password)
+	resp, err := s.httpClient.Do(req)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.Delete] cannot delete file, because %s", err.Error())
+		return err
+	}
+
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.Delete] cannot read response (http status code %d) body, because %s", resp.StatusCode, err.Error())
+		return err
+	}
+
+	if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound {
+		log.Errorf(ctx, "[webdav_storage.Delete] cannot delete file, http status code is %d, response is %s", resp.StatusCode, string(body))
+		return errs.ErrSystemError
+	}
+
+	return nil
+}
+
+func (s *WebDAVObjectStorage) directoryExists(ctx core.Context, path string) (bool, error) {
+	req, err := http.NewRequest("PROPFIND", s.getFinalDirectoryUrl(path), nil)
+
+	if err != nil {
+		return false, err
+	}
+
+	req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password)
+	resp, err := s.httpClient.Do(req)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.directoryExists] cannot check directory exists, because %s", err.Error())
+		return false, err
+	}
+
+	if resp.StatusCode == http.StatusMultiStatus || resp.StatusCode == http.StatusOK {
+		return true, nil
+	} else if resp.StatusCode == http.StatusNotFound {
+		return false, nil
+	}
+
+	log.Errorf(ctx, "[webdav_storage.directoryExists] cannot check directory exists, http status code is %d", resp.StatusCode)
+	return false, errs.ErrSystemError
+}
+
+func (s *WebDAVObjectStorage) createDirectory(ctx core.Context, path string) error {
+	req, err := http.NewRequest("MKCOL", s.getFinalDirectoryUrl(path), nil)
+
+	if err != nil {
+		return err
+	}
+
+	req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password)
+	resp, err := s.httpClient.Do(req)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.createDirectory] cannot create directory, because %s", err.Error())
+		return err
+	}
+
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+
+	if err != nil {
+		log.Errorf(ctx, "[webdav_storage.createDirectory] cannot read response (http status code %d) body, because %s", resp.StatusCode, err.Error())
+		return err
+	}
+
+	if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusMethodNotAllowed {
+		log.Errorf(ctx, "[webdav_storage.createDirectory] cannot create directory, http status code is %d, response is %s", resp.StatusCode, string(body))
+		return errs.ErrSystemError
+	}
+
+	return nil
+}
+
+func (s *WebDAVObjectStorage) createAllDirectories(ctx core.Context, currentPath string, path string) error {
+	directories := strings.Split(path, "/")
+
+	for _, dir := range directories {
+		if len(dir) == 0 {
+			continue
+		}
+
+		currentPath = currentPath + "/" + dir
+		exists, err := s.directoryExists(ctx, currentPath)
+
+		if err != nil {
+			return err
+		}
+
+		if !exists {
+			err = s.createDirectory(ctx, currentPath)
+
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (s *WebDAVObjectStorage) getFinalFileUrl(filePath string) string {
+	finalUrl := s.webDavConfig.Url
+
+	if len(finalUrl) < 1 || finalUrl[len(finalUrl)-1] != '/' {
+		finalUrl = finalUrl + "/"
+	}
+
+	finalPath := s.getFinalPath(filePath)
+
+	if len(finalPath) > 0 && finalPath[0] == '/' {
+		finalPath = finalPath[1:]
+	}
+
+	return finalUrl + finalPath
+}
+
+func (s *WebDAVObjectStorage) getFinalDirectoryUrl(dirPath string) string {
+	finalUrl := s.webDavConfig.Url
+
+	if len(finalUrl) < 1 || finalUrl[len(finalUrl)-1] != '/' {
+		finalUrl = finalUrl + "/"
+	}
+
+	if len(dirPath) > 0 && dirPath[0] == '/' {
+		dirPath = dirPath[1:]
+	}
+
+	if len(dirPath) > 0 && dirPath[len(dirPath)-1] != '/' {
+		dirPath = dirPath + "/"
+	}
+
+	return finalUrl + dirPath
+}
+
+func (s *WebDAVObjectStorage) getFinalPath(path string) string {
+	rootPath := s.rootPath
+
+	if len(rootPath) < 1 || rootPath[len(rootPath)-1] != '/' {
+		rootPath = rootPath + "/"
+	}
+
+	if len(path) > 0 && path[0] == '/' {
+		path = path[1:]
+	}
+
+	path = strings.ReplaceAll(path, "\\", "/")
+
+	return rootPath + path
+}