llm provider supports LM Studio

2026-01-31 22:38:08 +08:00
parent 548461ade0
commit 85db6e96af
6 changed files with 328 additions and 1 deletions
@@ -198,6 +198,10 @@ func getConfigWithoutSensitiveData(config *settings.Config) *settings.Config {
 		if clonedConfig.ReceiptImageRecognitionLLMConfig.OpenRouterAPIKey != "" {
 			clonedConfig.ReceiptImageRecognitionLLMConfig.OpenRouterAPIKey = "****"
 		}
+
+		if clonedConfig.ReceiptImageRecognitionLLMConfig.LMStudioToken != "" {
+			clonedConfig.ReceiptImageRecognitionLLMConfig.LMStudioToken = "****"
+		}
 	}

 	if clonedConfig.OAuth2ClientSecret != "" {
@@ -169,7 +169,7 @@ transaction_from_ai_image_recognition = false
 max_ai_recognition_picture_size = 10485760

 [llm_image_recognition]
-# Large Language Model (LLM) provider for receipt image recognition, supports the following types: "openai", "openai_compatible", "openrouter", "ollama", "google_ai"
+# Large Language Model (LLM) provider for receipt image recognition, supports the following types: "openai", "openai_compatible", "openrouter", "ollama", "lm_studio", "google_ai"
 llm_provider =

 # For "openai" llm provider only, OpenAI API secret key, please visit https://platform.openai.com/api-keys for more information
@@ -199,6 +199,15 @@ ollama_server_url =
 # For "ollama" llm provider only, receipt image recognition model for creating transactions from images
 ollama_model_id =

+# For "lm_studio" llm provider only, LM Studio server url, e.g. "http://127.0.0.1:1234/"
+lm_studio_server_url =
+
+# For "lm_studio" llm provider only, LM Studio API token, if "require authentication" is not enabled in LM Studio, leave it blank
+lm_studio_token =
+
+# For "lm_studio" llm provider only, receipt image recognition model for creating transactions from images
+lm_studio_model_id =
+
 # For "google_ai" llm provider only, Google AI Studio API key, please visit https://aistudio.google.com/apikey for more information
 google_ai_api_key =

@@ -6,6 +6,7 @@ import (
 	"github.com/mayswind/ezbookkeeping/pkg/llm/data"
 	"github.com/mayswind/ezbookkeeping/pkg/llm/provider"
 	"github.com/mayswind/ezbookkeeping/pkg/llm/provider/googleai"
+	"github.com/mayswind/ezbookkeeping/pkg/llm/provider/lmstudio"
 	"github.com/mayswind/ezbookkeeping/pkg/llm/provider/ollama"
 	"github.com/mayswind/ezbookkeeping/pkg/llm/provider/openai"
 	"github.com/mayswind/ezbookkeeping/pkg/settings"
@@ -45,6 +46,8 @@ func initializeLargeLanguageModelProvider(llmConfig *settings.LLMConfig, enableR
 		return openai.NewOpenRouterLargeLanguageModelProvider(llmConfig, enableResponseLog), nil
 	} else if llmConfig.LLMProvider == settings.OllamaLLMProvider {
 		return ollama.NewOllamaLargeLanguageModelProvider(llmConfig, enableResponseLog), nil
+	} else if llmConfig.LLMProvider == settings.LMStudioLLMProvider {
+		return lmstudio.NewLMStudioLargeLanguageModelProvider(llmConfig, enableResponseLog), nil
 	} else if llmConfig.LLMProvider == settings.GoogleAILLMProvider {
 		return googleai.NewGoogleAILargeLanguageModelProvider(llmConfig, enableResponseLog), nil
 	} else if llmConfig.LLMProvider == "" {
@@ -0,0 +1,155 @@
+package lmstudio
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+
+	"github.com/mayswind/ezbookkeeping/pkg/core"
+	"github.com/mayswind/ezbookkeeping/pkg/errs"
+	"github.com/mayswind/ezbookkeeping/pkg/llm/data"
+	"github.com/mayswind/ezbookkeeping/pkg/llm/provider"
+	"github.com/mayswind/ezbookkeeping/pkg/llm/provider/common"
+	"github.com/mayswind/ezbookkeeping/pkg/log"
+	"github.com/mayswind/ezbookkeeping/pkg/settings"
+)
+
+const lmStudioChatPath = "api/v1/chat"
+
+// LMStudioLargeLanguageModelAdapter defines the structure of LM Studio large language model adapter
+type LMStudioLargeLanguageModelAdapter struct {
+	common.HttpLargeLanguageModelAdapter
+	LMStudioServerURL string
+	LMStudioToken     string
+	LMStudioModelID   string
+}
+
+// LMStudioChatRequest defines the structure of LM Studio chat request
+type LMStudioChatRequest struct {
+	Model        string                      `json:"model"`
+	SystemPrompt string                      `json:"system_prompt,omitempty"`
+	Input        []*LMStudioChatRequestInput `json:"input"`
+}
+
+// LMStudioChatRequestInput defines the structure of LM Studio chat request message
+type LMStudioChatRequestInput struct {
+	Type    string `json:"type"`
+	Content string `json:"content,omitempty"`
+	DataUrl string `json:"data_url,omitempty"`
+}
+
+// LMStudioChatResponse defines the structure of LM Studio chat response
+type LMStudioChatResponse struct {
+	Output []*LMStudioChatResponseOutput `json:"output"`
+}
+
+// LMStudioChatResponseOutput defines the structure of LM Studio chat response message
+type LMStudioChatResponseOutput struct {
+	Content *string `json:"content"`
+}
+
+// BuildTextualRequest returns the http request by LM Studio large language model adapter
+func (p *LMStudioLargeLanguageModelAdapter) BuildTextualRequest(c core.Context, uid int64, request *data.LargeLanguageModelRequest, responseType data.LargeLanguageModelResponseFormat) (*http.Request, error) {
+	requestBody, err := p.buildJsonRequestBody(c, uid, request, responseType)
+
+	if err != nil {
+		return nil, err
+	}
+
+	httpRequest, err := http.NewRequest("POST", p.getLMStudioRequestUrl(), bytes.NewReader(requestBody))
+
+	if err != nil {
+		return nil, err
+	}
+
+	if p.LMStudioToken != "" {
+		httpRequest.Header.Set("Authorization", "Bearer "+p.LMStudioToken)
+	}
+
+	httpRequest.Header.Set("Content-Type", "application/json")
+
+	return httpRequest, nil
+}
+
+// ParseTextualResponse returns the textual response by LM Studio large language model adapter
+func (p *LMStudioLargeLanguageModelAdapter) ParseTextualResponse(c core.Context, uid int64, body []byte, responseType data.LargeLanguageModelResponseFormat) (*data.LargeLanguageModelTextualResponse, error) {
+	chatResponse := &LMStudioChatResponse{}
+	err := json.Unmarshal(body, &chatResponse)
+
+	if err != nil {
+		log.Errorf(c, "[lm_studio_large_language_model_adapter.ParseTextualResponse] failed to parse chat response for user \"uid:%d\", because %s", uid, err.Error())
+		return nil, errs.ErrFailedToRequestRemoteApi
+	}
+
+	if chatResponse == nil || len(chatResponse.Output) < 1 || chatResponse.Output[0].Content == nil {
+		log.Errorf(c, "[lm_studio_large_language_model_adapter.ParseTextualResponse] chat response is invalid for user \"uid:%d\"", uid)
+		return nil, errs.ErrFailedToRequestRemoteApi
+	}
+
+	textualResponse := &data.LargeLanguageModelTextualResponse{
+		Content: *chatResponse.Output[0].Content,
+	}
+
+	return textualResponse, nil
+}
+
+func (p *LMStudioLargeLanguageModelAdapter) buildJsonRequestBody(c core.Context, uid int64, request *data.LargeLanguageModelRequest, responseType data.LargeLanguageModelResponseFormat) ([]byte, error) {
+	if p.LMStudioModelID == "" {
+		return nil, errs.ErrInvalidLLMModelId
+	}
+
+	chatRequest := &LMStudioChatRequest{
+		Model: p.LMStudioModelID,
+		Input: make([]*LMStudioChatRequestInput, 0, 1),
+	}
+
+	if request.SystemPrompt != "" {
+		chatRequest.SystemPrompt = request.SystemPrompt
+	}
+
+	if len(request.UserPrompt) > 0 {
+		if request.UserPromptType == data.LARGE_LANGUAGE_MODEL_REQUEST_PROMPT_TYPE_IMAGE_URL {
+			imageBase64Data := "data:" + request.UserPromptContentType + ";base64," + base64.StdEncoding.EncodeToString(request.UserPrompt)
+			chatRequest.Input = append(chatRequest.Input, &LMStudioChatRequestInput{
+				Type:    "image",
+				DataUrl: imageBase64Data,
+			})
+		} else {
+			chatRequest.Input = append(chatRequest.Input, &LMStudioChatRequestInput{
+				Type:    "text",
+				Content: string(request.UserPrompt),
+			})
+		}
+	}
+
+	requestBodyBytes, err := json.Marshal(chatRequest)
+
+	if err != nil {
+		log.Errorf(c, "[lm_studio_large_language_model_adapter.buildJsonRequestBody] failed to marshal request body for user \"uid:%d\", because %s", uid, err.Error())
+		return nil, errs.ErrOperationFailed
+	}
+
+	log.Debugf(c, "[lm_studio_large_language_model_adapter.buildJsonRequestBody] request body is %s", requestBodyBytes)
+	return requestBodyBytes, nil
+}
+
+func (p *LMStudioLargeLanguageModelAdapter) getLMStudioRequestUrl() string {
+	url := p.LMStudioServerURL
+
+	if url[len(url)-1] != '/' {
+		url += "/"
+	}
+
+	url += lmStudioChatPath
+	return url
+}
+
+// NewLMStudioLargeLanguageModelProvider creates a new LM Studio large language model provider instance
+func NewLMStudioLargeLanguageModelProvider(llmConfig *settings.LLMConfig, enableResponseLog bool) provider.LargeLanguageModelProvider {
+	return common.NewCommonHttpLargeLanguageModelProvider(llmConfig, enableResponseLog, &LMStudioLargeLanguageModelAdapter{
+		LMStudioServerURL: llmConfig.LMStudioServerURL,
+		LMStudioToken:     llmConfig.LMStudioToken,
+		LMStudioModelID:   llmConfig.LMStudioModelID,
+	})
+}
@@ -0,0 +1,146 @@
+package lmstudio
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mayswind/ezbookkeeping/pkg/core"
+	"github.com/mayswind/ezbookkeeping/pkg/llm/data"
+)
+
+func TestLMStudioLargeLanguageModelAdapter_buildJsonRequestBody_TextualUserPrompt(t *testing.T) {
+	adapter := &LMStudioLargeLanguageModelAdapter{
+		LMStudioModelID: "test",
+	}
+
+	request := &data.LargeLanguageModelRequest{
+		SystemPrompt: "You are a helpful assistant.",
+		UserPrompt:   []byte("Hello, how are you?"),
+	}
+
+	bodyBytes, err := adapter.buildJsonRequestBody(core.NewNullContext(), 0, request, data.LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
+	assert.Nil(t, err)
+
+	var body map[string]interface{}
+	err = json.Unmarshal(bodyBytes, &body)
+	assert.Nil(t, err)
+
+	assert.Equal(t, "{\"model\":\"test\",\"system_prompt\":\"You are a helpful assistant.\",\"input\":[{\"type\":\"text\",\"content\":\"Hello, how are you?\"}]}", string(bodyBytes))
+}
+
+func TestLMStudioLargeLanguageModelAdapter_buildJsonRequestBody_ImageUserPrompt(t *testing.T) {
+	adapter := &LMStudioLargeLanguageModelAdapter{
+		LMStudioModelID: "test",
+	}
+
+	request := &data.LargeLanguageModelRequest{
+		SystemPrompt:          "What's in this image?",
+		UserPrompt:            []byte("fakedata"),
+		UserPromptType:        data.LARGE_LANGUAGE_MODEL_REQUEST_PROMPT_TYPE_IMAGE_URL,
+		UserPromptContentType: "image/png",
+	}
+
+	bodyBytes, err := adapter.buildJsonRequestBody(core.NewNullContext(), 0, request, data.LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
+	assert.Nil(t, err)
+
+	var body map[string]interface{}
+	err = json.Unmarshal(bodyBytes, &body)
+	assert.Nil(t, err)
+
+	assert.Equal(t, "{\"model\":\"test\",\"system_prompt\":\"What's in this image?\",\"input\":[{\"type\":\"image\",\"data_url\":\"data:image/png;base64,ZmFrZWRhdGE=\"}]}", string(bodyBytes))
+}
+
+func TestLMStudioLargeLanguageModelAdapter_ParseTextualResponse_ValidJsonResponse(t *testing.T) {
+	adapter := &LMStudioLargeLanguageModelAdapter{}
+
+	response := `{
+		"model_instance_id": "test",
+		"output": [
+			{
+				"type": "message",
+				"content": "This is a test response"
+			}
+		]
+	}`
+
+	result, err := adapter.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), data.LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
+	assert.Nil(t, err)
+	assert.Equal(t, "This is a test response", result.Content)
+}
+
+func TestLMStudioLargeLanguageModelAdapter_ParseTextualResponse_EmptyOutputContent(t *testing.T) {
+	adapter := &LMStudioLargeLanguageModelAdapter{}
+
+	response := `{
+		"model_instance_id": "test",
+		"output": [
+			{
+				"type": "message",
+				"content": ""
+			}
+		]
+	}`
+
+	result, err := adapter.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), data.LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
+	assert.Nil(t, err)
+	assert.Equal(t, "", result.Content)
+}
+
+func TestLMStudioLargeLanguageModelAdapter_ParseTextualResponse_EmptyOutput(t *testing.T) {
+	adapter := &LMStudioLargeLanguageModelAdapter{}
+
+	response := `{
+		"model_instance_id": "test",
+		"output": []
+	}`
+
+	_, err := adapter.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), data.LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
+	assert.EqualError(t, err, "failed to request third party api")
+}
+
+func TestLMStudioLargeLanguageModelAdapter_ParseTextualResponse_NoContentFieldInOutput(t *testing.T) {
+	adapter := &LMStudioLargeLanguageModelAdapter{}
+
+	response := `{
+		"model_instance_id": "test",
+		"output": [
+			{
+				"type": "message"
+			}
+		]
+	}`
+
+	_, err := adapter.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), data.LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
+	assert.EqualError(t, err, "failed to request third party api")
+}
+
+func TestLMStudioLargeLanguageModelAdapter_ParseTextualResponse_InvalidJson(t *testing.T) {
+	adapter := &LMStudioLargeLanguageModelAdapter{}
+
+	response := "error"
+
+	_, err := adapter.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), data.LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
+	assert.EqualError(t, err, "failed to request third party api")
+}
+
+func TestLMStudioLargeLanguageModelAdapter_GetOllamaRequestUrl(t *testing.T) {
+	adapter := &LMStudioLargeLanguageModelAdapter{
+		LMStudioServerURL: "http://localhost:1234/",
+	}
+	url := adapter.getLMStudioRequestUrl()
+	assert.Equal(t, "http://localhost:1234/api/v1/chat", url)
+
+	adapter = &LMStudioLargeLanguageModelAdapter{
+		LMStudioServerURL: "http://localhost:1234",
+	}
+	url = adapter.getLMStudioRequestUrl()
+	assert.Equal(t, "http://localhost:1234/api/v1/chat", url)
+
+	adapter = &LMStudioLargeLanguageModelAdapter{
+		LMStudioServerURL: "http://example.com/lmstudio/",
+	}
+	url = adapter.getLMStudioRequestUrl()
+	assert.Equal(t, "http://example.com/lmstudio/api/v1/chat", url)
+}
@@ -73,6 +73,7 @@ const (
 	OpenAICompatibleLLMProvider string = "openai_compatible"
 	OpenRouterLLMProvider       string = "openrouter"
 	OllamaLLMProvider           string = "ollama"
+	LMStudioLLMProvider         string = "lm_studio"
 	GoogleAILLMProvider         string = "google_ai"
 )

@@ -248,6 +249,9 @@ type LLMConfig struct {
 	OpenRouterModelID                   string
 	OllamaServerURL                     string
 	OllamaModelID                       string
+	LMStudioServerURL                   string
+	LMStudioToken                       string
+	LMStudioModelID                     string
 	GoogleAIAPIKey                      string
 	GoogleAIModelID                     string
 	LargeLanguageModelAPIRequestTimeout uint32
@@ -864,6 +868,8 @@ func loadLLMConfiguration(configFile *ini.File, sectionName string) (*LLMConfig,
 		llmConfig.LLMProvider = OpenRouterLLMProvider
 	} else if llmProvider == OllamaLLMProvider {
 		llmConfig.LLMProvider = OllamaLLMProvider
+	} else if llmProvider == LMStudioLLMProvider {
+		llmConfig.LLMProvider = LMStudioLLMProvider
 	} else if llmProvider == GoogleAILLMProvider {
 		llmConfig.LLMProvider = GoogleAILLMProvider
 	} else {
@@ -883,6 +889,10 @@ func loadLLMConfiguration(configFile *ini.File, sectionName string) (*LLMConfig,
 	llmConfig.OllamaServerURL = getConfigItemStringValue(configFile, sectionName, "ollama_server_url")
 	llmConfig.OllamaModelID = getConfigItemStringValue(configFile, sectionName, "ollama_model_id")

+	llmConfig.LMStudioServerURL = getConfigItemStringValue(configFile, sectionName, "lm_studio_server_url")
+	llmConfig.LMStudioToken = getConfigItemStringValue(configFile, sectionName, "lm_studio_token")
+	llmConfig.LMStudioModelID = getConfigItemStringValue(configFile, sectionName, "lm_studio_model_id")
+
 	llmConfig.GoogleAIAPIKey = getConfigItemStringValue(configFile, sectionName, "google_ai_api_key")
 	llmConfig.GoogleAIModelID = getConfigItemStringValue(configFile, sectionName, "google_ai_model_id")