From 29a87dcfaf2be7a75cea06155685e6b39f6ed54b Mon Sep 17 00:00:00 2001 From: MaysWind Date: Sat, 2 Aug 2025 01:26:29 +0800 Subject: [PATCH] object storage supports webdav --- cmd/initializer.go | 4 + conf/ezbookkeeping.ini | 24 +- pkg/settings/setting.go | 28 ++- pkg/storage/bytes_slice_object.go | 22 ++ pkg/storage/storage_container.go | 2 + pkg/storage/webdav_storage.go | 360 ++++++++++++++++++++++++++++++ 6 files changed, 438 insertions(+), 2 deletions(-) create mode 100644 pkg/storage/bytes_slice_object.go create mode 100644 pkg/storage/webdav_storage.go diff --git a/cmd/initializer.go b/cmd/initializer.go index 70bdce52..a26ec124 100644 --- a/cmd/initializer.go +++ b/cmd/initializer.go @@ -158,5 +158,9 @@ func getConfigWithoutSensitiveData(config *settings.Config) *settings.Config { clonedConfig.SecretKey = "****" clonedConfig.AmapApplicationSecret = "****" + if clonedConfig.WebDAVConfig != nil { + clonedConfig.WebDAVConfig.Password = "****" + } + return clonedConfig } diff --git a/conf/ezbookkeeping.ini b/conf/ezbookkeeping.ini index fc21ba18..40917042 100644 --- a/conf/ezbookkeeping.ini +++ b/conf/ezbookkeeping.ini @@ -115,7 +115,7 @@ log_file_max_size = 104857600 log_file_max_days = 7 [storage] -# Object storage type, supports "local_filesystem" and "minio" currently +# Object storage type, supports "local_filesystem", "minio" and "webdav" currently type = local_filesystem # For "local_filesystem" storage only, the storage root path (relative or absolute path) @@ -139,6 +139,28 @@ minio_bucket = ezbookkeeping # For "minio" storage only, the root path to store files in minio minio_root_path = / +# For "webdav" storage only, the webdav url +webdav_url = + +# For "webdav" storage only, the webdav username +webdav_username = + +# For "webdav" storage only, the webdav password +webdav_password = + +# For "webdav" storage only, the webdav root path to store files +webdav_root_path = / + +# For "webdav" storage only, requesting webdav url timeout (0 - 4294967295 milliseconds) +# Set to 0 to disable timeout for requesting webdav url, default is 10000 (10 seconds) +webdav_request_timeout = 10000 + +# For "webdav" storage only, proxy for requesting webdav url, supports "system" (use system proxy), "none" (do not use proxy), or proxy URL which starts with "http://", "https://" or "socks5://", default is "system" +webdav_proxy = system + +# For "webdav" storage only, set to true to skip tls verification when connect webdav +webdav_skip_tls_verify = false + [uuid] # Uuid generator type, supports "internal" currently generator_type = internal diff --git a/pkg/settings/setting.go b/pkg/settings/setting.go index b8abbcb3..f15a9209 100644 --- a/pkg/settings/setting.go +++ b/pkg/settings/setting.go @@ -63,6 +63,7 @@ const ( const ( LocalFileSystemObjectStorageType string = "local_filesystem" MinIOStorageType string = "minio" + WebDAVStorageType string = "webdav" ) // Uuid generator types @@ -137,6 +138,8 @@ const ( defaultLogFileMaxSize uint32 = 104857600 // 100 MB defaultLogFileMaxDays uint32 = 7 // days + defaultWebDAVRequestTimeout uint32 = 10000 // 10 seconds + defaultInMemoryDuplicateCheckerCleanupInterval uint32 = 60 // 1 minutes defaultDuplicateSubmissionsInterval uint32 = 300 // 5 minutes @@ -195,6 +198,17 @@ type MinIOConfig struct { RootPath string } +// WebDAVConfig represents the WebDAV setting config +type WebDAVConfig struct { + Url string + Username string + Password string + RootPath string + RequestTimeout uint32 + Proxy string + SkipTLSVerify bool +} + // TipConfig represents a tip setting config type TipConfig struct { Enabled bool @@ -264,6 +278,7 @@ type Config struct { StorageType string LocalFileSystemPath string MinIOConfig *MinIOConfig + WebDAVConfig *WebDAVConfig // Uuid UuidGeneratorType string @@ -697,6 +712,8 @@ func loadStorageConfiguration(config *Config, configFile *ini.File, sectionName config.StorageType = LocalFileSystemObjectStorageType } else if getConfigItemStringValue(configFile, sectionName, "type") == MinIOStorageType { config.StorageType = MinIOStorageType + } else if getConfigItemStringValue(configFile, sectionName, "type") == WebDAVStorageType { + config.StorageType = WebDAVStorageType } else { return errs.ErrInvalidStorageType } @@ -718,9 +735,18 @@ func loadStorageConfiguration(config *Config, configFile *ini.File, sectionName minIOConfig.SkipTLSVerify = getConfigItemBoolValue(configFile, sectionName, "minio_skip_tls_verify", false) minIOConfig.Bucket = getConfigItemStringValue(configFile, sectionName, "minio_bucket") minIOConfig.RootPath = getConfigItemStringValue(configFile, sectionName, "minio_root_path") - config.MinIOConfig = minIOConfig + webDAVConfig := &WebDAVConfig{} + webDAVConfig.Url = getConfigItemStringValue(configFile, sectionName, "webdav_url") + webDAVConfig.Username = getConfigItemStringValue(configFile, sectionName, "webdav_username") + webDAVConfig.Password = getConfigItemStringValue(configFile, sectionName, "webdav_password") + webDAVConfig.RootPath = getConfigItemStringValue(configFile, sectionName, "webdav_root_path") + webDAVConfig.RequestTimeout = getConfigItemUint32Value(configFile, sectionName, "webdav_request_timeout", defaultWebDAVRequestTimeout) + webDAVConfig.Proxy = getConfigItemStringValue(configFile, sectionName, "webdav_proxy", "system") + webDAVConfig.SkipTLSVerify = getConfigItemBoolValue(configFile, sectionName, "webdav_skip_tls_verify", false) + config.WebDAVConfig = webDAVConfig + return nil } diff --git a/pkg/storage/bytes_slice_object.go b/pkg/storage/bytes_slice_object.go new file mode 100644 index 00000000..8696024c --- /dev/null +++ b/pkg/storage/bytes_slice_object.go @@ -0,0 +1,22 @@ +package storage + +import ( + "bytes" +) + +// bytesSliceObject represents a byte slice object in storage +type bytesSliceObject struct { + *bytes.Reader +} + +// Close does nothing because it does not hold any resources that need to be released +func (b *bytesSliceObject) Close() error { + return nil +} + +// newByteSliceObject creates a new byte slice object from the specified byte slice +func newByteSliceObject(data []byte) ObjectInStorage { + return &bytesSliceObject{ + Reader: bytes.NewReader(data), + } +} diff --git a/pkg/storage/storage_container.go b/pkg/storage/storage_container.go index 044f38a6..5ca7b12a 100644 --- a/pkg/storage/storage_container.go +++ b/pkg/storage/storage_container.go @@ -86,6 +86,8 @@ func newObjectStorage(config *settings.Config, pathPrefix string) (ObjectStorage return NewLocalFileSystemObjectStorage(config, pathPrefix) } else if config.StorageType == settings.MinIOStorageType { return NewMinIOObjectStorage(config, pathPrefix) + } else if config.StorageType == settings.WebDAVStorageType { + return NewWebDAVObjectStorage(config, pathPrefix) } return nil, errs.ErrInvalidStorageType diff --git a/pkg/storage/webdav_storage.go b/pkg/storage/webdav_storage.go new file mode 100644 index 00000000..77d065a4 --- /dev/null +++ b/pkg/storage/webdav_storage.go @@ -0,0 +1,360 @@ +package storage + +import ( + "bytes" + "crypto/tls" + "io" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/mayswind/ezbookkeeping/pkg/core" + "github.com/mayswind/ezbookkeeping/pkg/errs" + "github.com/mayswind/ezbookkeeping/pkg/log" + "github.com/mayswind/ezbookkeeping/pkg/settings" + "github.com/mayswind/ezbookkeeping/pkg/utils" +) + +// WebDAVObjectStorage represents WebDAV object storage +type WebDAVObjectStorage struct { + httpClient *http.Client + webDavConfig *settings.WebDAVConfig + rootPath string +} + +// NewWebDAVObjectStorage returns a WebDAV object storage +func NewWebDAVObjectStorage(config *settings.Config, pathPrefix string) (*WebDAVObjectStorage, error) { + webDavConfig := config.WebDAVConfig + transport := http.DefaultTransport.(*http.Transport).Clone() + utils.SetProxyUrl(transport, webDavConfig.Proxy) + + if webDavConfig.SkipTLSVerify { + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } + + client := &http.Client{ + Transport: transport, + Timeout: time.Duration(webDavConfig.RequestTimeout) * time.Millisecond, + } + + storage := &WebDAVObjectStorage{ + httpClient: client, + webDavConfig: webDavConfig, + rootPath: webDavConfig.RootPath, + } + + storage.rootPath = storage.getFinalPath(pathPrefix) + storage.rootPath = strings.ReplaceAll(storage.rootPath, "\\", "/") + + ctx := core.NewNullContext() + exists, err := storage.directoryExists(ctx, storage.rootPath) + + if err != nil { + return nil, err + } + + if !exists { + err := storage.createAllDirectories(ctx, "", storage.rootPath) + + if err != nil { + return nil, err + } + } + + return storage, nil +} + +// Exists returns whether the file exists +func (s *WebDAVObjectStorage) Exists(ctx core.Context, path string) (bool, error) { + req, err := http.NewRequest("HEAD", s.getFinalFileUrl(path), nil) + + if err != nil { + return false, err + } + + req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password) + resp, err := s.httpClient.Do(req) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.Exists] cannot check file exists, because %s", err.Error()) + return false, err + } + + if resp.StatusCode == http.StatusOK { + return true, nil + } else if resp.StatusCode == http.StatusNotFound { + return false, nil + } + + log.Errorf(ctx, "[webdav_storage.Exists] cannot check file exists, http status code is %d", resp.StatusCode) + return false, errs.ErrSystemError +} + +// Read returns the object instance according to specified the file path +func (s *WebDAVObjectStorage) Read(ctx core.Context, path string) (ObjectInStorage, error) { + req, err := http.NewRequest("GET", s.getFinalFileUrl(path), nil) + + if err != nil { + return nil, err + } + + req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password) + resp, err := s.httpClient.Do(req) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.Read] cannot get file, because %s", err.Error()) + return nil, err + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.Read] cannot read response (http status code %d) body, because %s", resp.StatusCode, err.Error()) + return nil, err + } + + if resp.StatusCode != http.StatusOK { + log.Errorf(ctx, "[webdav_storage.Read] cannot get file, http status code is %d, response is %s", resp.StatusCode, string(body)) + return nil, errs.ErrSystemError + } + + return newByteSliceObject(body), nil +} + +// Save returns whether save the object instance successfully +func (s *WebDAVObjectStorage) Save(ctx core.Context, path string, object ObjectInStorage) error { + finalPath := s.getFinalPath(path) + dir := strings.ReplaceAll(filepath.Dir(finalPath), "\\", "/") + + exists, err := s.directoryExists(ctx, dir) + + if err != nil { + return err + } + + if !exists { + rootExists, err := s.directoryExists(ctx, s.rootPath) + + if err != nil { + return err + } + + if !rootExists { + err := s.createAllDirectories(ctx, "", s.rootPath) + + if err != nil { + return err + } + } + + err = s.createAllDirectories(ctx, s.rootPath, strings.ReplaceAll(filepath.Dir(path), "\\", "/")) + + if err != nil { + return err + } + } + + data, err := io.ReadAll(object) + + if err != nil { + return err + } + + req, err := http.NewRequest("PUT", s.getFinalFileUrl(path), bytes.NewReader(data)) + + if err != nil { + return err + } + + req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password) + resp, err := s.httpClient.Do(req) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.Save] cannot save file, because %s", err.Error()) + return err + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.Save] cannot read response (http status code %d) body, because %s", resp.StatusCode, err.Error()) + return err + } + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + log.Errorf(ctx, "[webdav_storage.Save] cannot save file, http status code is %d, response is %s", resp.StatusCode, string(body)) + return errs.ErrSystemError + } + + return nil +} + +// Delete returns whether delete the object according to specified the file path successfully +func (s *WebDAVObjectStorage) Delete(ctx core.Context, path string) error { + req, err := http.NewRequest("DELETE", s.getFinalFileUrl(path), nil) + + if err != nil { + return err + } + + req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password) + resp, err := s.httpClient.Do(req) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.Delete] cannot delete file, because %s", err.Error()) + return err + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.Delete] cannot read response (http status code %d) body, because %s", resp.StatusCode, err.Error()) + return err + } + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusNotFound { + log.Errorf(ctx, "[webdav_storage.Delete] cannot delete file, http status code is %d, response is %s", resp.StatusCode, string(body)) + return errs.ErrSystemError + } + + return nil +} + +func (s *WebDAVObjectStorage) directoryExists(ctx core.Context, path string) (bool, error) { + req, err := http.NewRequest("PROPFIND", s.getFinalDirectoryUrl(path), nil) + + if err != nil { + return false, err + } + + req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password) + resp, err := s.httpClient.Do(req) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.directoryExists] cannot check directory exists, because %s", err.Error()) + return false, err + } + + if resp.StatusCode == http.StatusMultiStatus || resp.StatusCode == http.StatusOK { + return true, nil + } else if resp.StatusCode == http.StatusNotFound { + return false, nil + } + + log.Errorf(ctx, "[webdav_storage.directoryExists] cannot check directory exists, http status code is %d", resp.StatusCode) + return false, errs.ErrSystemError +} + +func (s *WebDAVObjectStorage) createDirectory(ctx core.Context, path string) error { + req, err := http.NewRequest("MKCOL", s.getFinalDirectoryUrl(path), nil) + + if err != nil { + return err + } + + req.SetBasicAuth(s.webDavConfig.Username, s.webDavConfig.Password) + resp, err := s.httpClient.Do(req) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.createDirectory] cannot create directory, because %s", err.Error()) + return err + } + + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + + if err != nil { + log.Errorf(ctx, "[webdav_storage.createDirectory] cannot read response (http status code %d) body, because %s", resp.StatusCode, err.Error()) + return err + } + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusMethodNotAllowed { + log.Errorf(ctx, "[webdav_storage.createDirectory] cannot create directory, http status code is %d, response is %s", resp.StatusCode, string(body)) + return errs.ErrSystemError + } + + return nil +} + +func (s *WebDAVObjectStorage) createAllDirectories(ctx core.Context, currentPath string, path string) error { + directories := strings.Split(path, "/") + + for _, dir := range directories { + if len(dir) == 0 { + continue + } + + currentPath = currentPath + "/" + dir + exists, err := s.directoryExists(ctx, currentPath) + + if err != nil { + return err + } + + if !exists { + err = s.createDirectory(ctx, currentPath) + + if err != nil { + return err + } + } + } + + return nil +} + +func (s *WebDAVObjectStorage) getFinalFileUrl(filePath string) string { + finalUrl := s.webDavConfig.Url + + if len(finalUrl) < 1 || finalUrl[len(finalUrl)-1] != '/' { + finalUrl = finalUrl + "/" + } + + finalPath := s.getFinalPath(filePath) + + if len(finalPath) > 0 && finalPath[0] == '/' { + finalPath = finalPath[1:] + } + + return finalUrl + finalPath +} + +func (s *WebDAVObjectStorage) getFinalDirectoryUrl(dirPath string) string { + finalUrl := s.webDavConfig.Url + + if len(finalUrl) < 1 || finalUrl[len(finalUrl)-1] != '/' { + finalUrl = finalUrl + "/" + } + + if len(dirPath) > 0 && dirPath[0] == '/' { + dirPath = dirPath[1:] + } + + if len(dirPath) > 0 && dirPath[len(dirPath)-1] != '/' { + dirPath = dirPath + "/" + } + + return finalUrl + dirPath +} + +func (s *WebDAVObjectStorage) getFinalPath(path string) string { + rootPath := s.rootPath + + if len(rootPath) < 1 || rootPath[len(rootPath)-1] != '/' { + rootPath = rootPath + "/" + } + + if len(path) > 0 && path[0] == '/' { + path = path[1:] + } + + path = strings.ReplaceAll(path, "\\", "/") + + return rootPath + path +}