From e93151d1c2b7a0835a550aa5570912507ee2c664 Mon Sep 17 00:00:00 2001 From: tbphp Date: Wed, 16 Jul 2025 21:34:30 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20=E6=97=A5=E5=BF=97=E8=A1=A8=E6=A0=BC?= =?UTF-8?q?=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- web/src/components/logs/LogTable.vue | 38 +++++++++++++++++----------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/web/src/components/logs/LogTable.vue b/web/src/components/logs/LogTable.vue index 032fde5..e934d4a 100644 --- a/web/src/components/logs/LogTable.vue +++ b/web/src/components/logs/LogTable.vue @@ -43,7 +43,7 @@ const filters = reactive({ }); const successOptions = [ - { label: "全部状态", value: "" }, + { label: "状态", value: "" }, { label: "成功", value: "true" }, { label: "失败", value: "false" }, ]; @@ -102,13 +102,13 @@ const createColumns = () => [ { title: "时间", key: "timestamp", - width: 180, + width: 160, render: (row: LogRow) => formatDateTime(row.timestamp), }, { title: "状态", key: "is_success", - width: 80, + width: 50, render: (row: LogRow) => h( NTag, @@ -119,7 +119,7 @@ const createColumns = () => [ { title: "类型", key: "is_stream", - width: 80, + width: 50, render: (row: LogRow) => h( NTag, @@ -127,9 +127,9 @@ const createColumns = () => [ { default: () => (row.is_stream ? "流式" : "非流") } ), }, - { title: "状态码", key: "status_code", width: 80 }, - { title: "耗时(ms)", key: "duration_ms", width: 100 }, - { title: "重试", key: "retries", width: 60 }, + { title: "状态码", key: "status_code", width: 60 }, + { title: "耗时(ms)", key: "duration_ms", width: 80 }, + { title: "重试", key: "retries", width: 50 }, { title: "分组", key: "group_name", width: 120 }, { title: "Key", @@ -155,18 +155,21 @@ const createColumns = () => [ { title: "请求路径", key: "request_path", + width: 220, render: (row: LogRow) => h(NEllipsis, { style: "max-width: 200px" }, { default: () => row.request_path }), }, { title: "上游地址", key: "upstream_addr", + width: 220, render: (row: LogRow) => h(NEllipsis, { style: "max-width: 200px" }, { default: () => row.upstream_addr }), }, { title: "源IP", key: "source_ip", width: 130 }, { title: "错误信息", + width: 270, key: "error_message", render: (row: LogRow) => h(NEllipsis, { style: "max-width: 250px" }, { default: () => row.error_message || "-" }), @@ -174,6 +177,7 @@ const createColumns = () => [ { title: "User Agent", key: "user_agent", + width: 220, render: (row: LogRow) => h(NEllipsis, { style: "max-width: 200px" }, { default: () => row.user_agent }), }, @@ -223,7 +227,7 @@ function changePageSize(size: number) { v-model:value="filters.is_success" :options="successOptions" size="small" - style="width: 120px" + style="width: 80px" @update:value="handleSearch" /> - - -
- + @@ -340,7 +348,7 @@ function changePageSize(size: number) { background: white; border-radius: 8px; display: flex; - justify-content: space-between; + justify-content: left; align-items: center; padding: 12px; border-bottom: 1px solid #f0f0f0; From dc72e5850ab80b63d2a64cd04fa8c5aacc1c789e Mon Sep 17 00:00:00 2001 From: tbphp Date: Wed, 16 Jul 2025 22:17:13 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E8=A1=A8=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/app/app.go | 3 + internal/db/migrations/migration.go | 10 ++ .../db/migrations/v1.0.13_fix_request_logs.go | 130 ++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 internal/db/migrations/migration.go create mode 100644 internal/db/migrations/v1.0.13_fix_request_logs.go diff --git a/internal/app/app.go b/internal/app/app.go index 13d32e1..a0b2704 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -9,6 +9,7 @@ import ( "time" "gpt-load/internal/config" + db "gpt-load/internal/db/migrations" "gpt-load/internal/keypool" "gpt-load/internal/models" "gpt-load/internal/proxy" @@ -88,6 +89,8 @@ func (a *App) Start() error { ); err != nil { return fmt.Errorf("database auto-migration failed: %w", err) } + // 数据修复 + db.MigrateDatabase(a.db) logrus.Info("Database auto-migration completed.") // 初始化系统设置 diff --git a/internal/db/migrations/migration.go b/internal/db/migrations/migration.go new file mode 100644 index 0000000..fac2eb6 --- /dev/null +++ b/internal/db/migrations/migration.go @@ -0,0 +1,10 @@ +package db + +import ( + "gorm.io/gorm" +) + +func MigrateDatabase(db *gorm.DB) error { + // v1.0.13 修复请求日志数据 + return V1_0_13_FixRequestLogs(db) +} diff --git a/internal/db/migrations/v1.0.13_fix_request_logs.go b/internal/db/migrations/v1.0.13_fix_request_logs.go new file mode 100644 index 0000000..a17e15d --- /dev/null +++ b/internal/db/migrations/v1.0.13_fix_request_logs.go @@ -0,0 +1,130 @@ +package db + +import ( + "gpt-load/internal/models" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +func V1_0_13_FixRequestLogs(db *gorm.DB) error { + return db.Transaction(func(tx *gorm.DB) error { + // 如果有key_id，就执行修复 + if !tx.Migrator().HasColumn(&models.RequestLog{}, "key_id") { + return nil + } + + logrus.Info("Old schema detected. Starting data migration for request_logs...") + + if !tx.Migrator().HasColumn(&models.RequestLog{}, "group_name") { + logrus.Info("Adding 'group_name' column to request_logs table...") + if err := tx.Migrator().AddColumn(&models.RequestLog{}, "group_name"); err != nil { + return err + } + } + if !tx.Migrator().HasColumn(&models.RequestLog{}, "key_value") { + logrus.Info("Adding 'key_value' column to request_logs table...") + if err := tx.Migrator().AddColumn(&models.RequestLog{}, "key_value"); err != nil { + return err + } + } + + type OldRequestLog struct { + ID string + KeyID uint `gorm:"column:key_id"` + GroupID uint + } + + batchSize := 1000 + for i := 0; ; i++ { + logrus.Infof("Processing batch %d...", i+1) + var oldLogs []OldRequestLog + + result := tx.Model(&models.RequestLog{}). + Select("id", "key_id", "group_id"). + Where("key_value IS NULL OR group_name IS NULL"). + Limit(batchSize). + Find(&oldLogs) + + if result.Error != nil { + return result.Error + } + + if len(oldLogs) == 0 { + logrus.Info("All batches processed.") + break + } + + keyIDMap := make(map[uint]bool) + groupIDMap := make(map[uint]bool) + for _, logEntry := range oldLogs { + if logEntry.KeyID > 0 { + keyIDMap[logEntry.KeyID] = true + } + if logEntry.GroupID > 0 { + groupIDMap[logEntry.GroupID] = true + } + } + + var apiKeys []models.APIKey + if len(keyIDMap) > 0 { + var keyIDs []uint + for id := range keyIDMap { + keyIDs = append(keyIDs, id) + } + if err := tx.Model(&models.APIKey{}).Where("id IN ?", keyIDs).Find(&apiKeys).Error; err != nil { + return err + } + } + keyValueMapping := make(map[uint]string) + for _, key := range apiKeys { + keyValueMapping[key.ID] = key.KeyValue + } + + var groups []models.Group + if len(groupIDMap) > 0 { + var groupIDs []uint + for id := range groupIDMap { + groupIDs = append(groupIDs, id) + } + if err := tx.Model(&models.Group{}).Where("id IN ?", groupIDs).Find(&groups).Error; err != nil { + return err + } + } + groupNameMapping := make(map[uint]string) + for _, group := range groups { + groupNameMapping[group.ID] = group.Name + } + + for _, logEntry := range oldLogs { + groupName, gExists := groupNameMapping[logEntry.GroupID] + if !gExists { + logrus.Warnf("Log ID %s: Could not find Group for group_id %d. Setting group_name to empty string.", logEntry.ID, logEntry.GroupID) + } + + keyValue, kExists := keyValueMapping[logEntry.KeyID] + if !kExists { + logrus.Warnf("Log ID %s: Could not find APIKey for key_id %d. Setting key_value to empty string.", logEntry.ID, logEntry.KeyID) + } + + updates := map[string]any{ + "group_name": groupName, + "key_value": keyValue, + } + if err := tx.Model(&models.RequestLog{}).Where("id = ?", logEntry.ID).UpdateColumns(updates).Error; err != nil { + logrus.WithError(err).Errorf("Failed to update log entry with ID: %s", logEntry.ID) + continue + } + } + logrus.Infof("Successfully updated %d log entries in batch %d.", len(oldLogs), i+1) + } + + logrus.Info("Data migration complete. Dropping 'key_id' column from request_logs table...") + if err := tx.Migrator().DropColumn(&models.RequestLog{}, "key_id"); err != nil { + logrus.WithError(err).Warn("Failed to drop 'key_id' column. This can be done manually.") + } + + logrus.Info("Database migration successful!") + return nil + }) +} From a64d48439f55a7095b8d2b1e47827373e5b5c88f Mon Sep 17 00:00:00 2001 From: tbphp Date: Wed, 16 Jul 2025 22:39:41 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=E5=86=97=E4=BD=99=E6=97=A5=E5=BF=97?= =?UTF-8?q?group=E5=92=8Ckey=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/handler/log_handler.go | 92 ++---------------------- internal/models/types.go | 3 +- internal/proxy/server.go | 21 +++--- internal/services/request_log_service.go | 18 ++--- internal/types/types.go | 2 +- 5 files changed, 30 insertions(+), 106 deletions(-) diff --git a/internal/handler/log_handler.go b/internal/handler/log_handler.go index 1691616..898a0c1 100644 --- a/internal/handler/log_handler.go +++ b/internal/handler/log_handler.go @@ -4,50 +4,29 @@ import ( "strconv" "time" - "github.com/gin-gonic/gin" "gpt-load/internal/db" app_errors "gpt-load/internal/errors" "gpt-load/internal/models" "gpt-load/internal/response" + + "github.com/gin-gonic/gin" ) -// LogResponse defines the structure for log entries in the API response, -// enriching the base log with related data. +// LogResponse defines the structure for log entries in the API response type LogResponse struct { models.RequestLog - GroupName string `json:"group_name"` - KeyValue string `json:"key_value"` } // GetLogs Get request logs func GetLogs(c *gin.Context) { - // --- 1. Build WHERE conditions --- query := db.DB.Model(&models.RequestLog{}) if groupName := c.Query("group_name"); groupName != "" { - var groupIDs []uint - db.DB.Model(&models.Group{}).Where("name LIKE ? OR display_name LIKE ?", "%"+groupName+"%", "%"+groupName+"%").Pluck("id", &groupIDs) - if len(groupIDs) == 0 { - response.Success(c, &response.PaginatedResponse{ - Items: []LogResponse{}, - Pagination: response.Pagination{TotalItems: 0, Page: 1, PageSize: response.DefaultPageSize}, - }) - return - } - query = query.Where("group_id IN ?", groupIDs) + query = query.Where("group_name LIKE ?", "%"+groupName+"%") } if keyValue := c.Query("key_value"); keyValue != "" { - var keyIDs []uint likePattern := "%" + keyValue[1:len(keyValue)-1] + "%" - db.DB.Model(&models.APIKey{}).Where("key_value LIKE ?", likePattern).Pluck("id", &keyIDs) - if len(keyIDs) == 0 { - response.Success(c, &response.PaginatedResponse{ - Items: []LogResponse{}, - Pagination: response.Pagination{TotalItems: 0, Page: 1, PageSize: response.DefaultPageSize}, - }) - return - } - query = query.Where("key_id IN ?", keyIDs) + query = query.Where("key_value LIKE ?", likePattern) } if isSuccessStr := c.Query("is_success"); isSuccessStr != "" { if isSuccess, err := strconv.ParseBool(isSuccessStr); err == nil { @@ -76,71 +55,14 @@ func GetLogs(c *gin.Context) { } } - // --- 2. Get Paginated Logs --- var logs []models.RequestLog - query = query.Order("timestamp desc") // Apply ordering before pagination + query = query.Order("timestamp desc") pagination, err := response.Paginate(c, query, &logs) if err != nil { response.Error(c, app_errors.ParseDBError(err)) return } - // --- 3. Enrich Logs with GroupName and KeyValue --- - if len(logs) == 0 { - response.Success(c, pagination) // Return empty pagination response - return - } - - // Collect IDs for enrichment - groupIds := make(map[uint]bool) - keyIds := make(map[uint]bool) - for _, log := range logs { - if log.GroupID != 0 { - groupIds[log.GroupID] = true - } - if log.KeyID != 0 { - keyIds[log.KeyID] = true - } - } - - // Fetch enrichment data - groupMap := make(map[uint]string) - if len(groupIds) > 0 { - var groups []models.Group - var ids []uint - for id := range groupIds { - ids = append(ids, id) - } - db.DB.Where("id IN ?", ids).Find(&groups) - for _, group := range groups { - groupMap[group.ID] = group.Name - } - } - - keyMap := make(map[uint]string) - if len(keyIds) > 0 { - var keys []models.APIKey - var ids []uint - for id := range keyIds { - ids = append(ids, id) - } - db.DB.Where("id IN ?", ids).Find(&keys) - for _, key := range keys { - keyMap[key.ID] = key.KeyValue - } - } - - // Build final response - logResponses := make([]LogResponse, len(logs)) - for i, log := range logs { - logResponses[i] = LogResponse{ - RequestLog: log, - GroupName: groupMap[log.GroupID], - KeyValue: keyMap[log.KeyID], - } - } - - // --- 4. Send Response --- - pagination.Items = logResponses + pagination.Items = logs response.Success(c, pagination) } diff --git a/internal/models/types.go b/internal/models/types.go index f619f37..ef2db1d 100644 --- a/internal/models/types.go +++ b/internal/models/types.go @@ -76,7 +76,8 @@ type RequestLog struct { ID string `gorm:"type:varchar(36);primaryKey" json:"id"` Timestamp time.Time `gorm:"type:datetime(3);not null;index" json:"timestamp"` GroupID uint `gorm:"not null;index" json:"group_id"` - KeyID uint `gorm:"not null;index" json:"key_id"` + GroupName string `gorm:"type:varchar(255);index" json:"group_name"` + KeyValue string `gorm:"type:varchar(512)" json:"key_value"` IsSuccess bool `gorm:"not null" json:"is_success"` SourceIP string `gorm:"type:varchar(45)" json:"source_ip"` StatusCode int `gorm:"not null" json:"status_code"` diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 0437eb3..52ae8e3 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "net/http" - "strconv" "time" "gpt-load/internal/channel" @@ -115,12 +114,11 @@ func (ps *ProxyServer) executeRequestWithRetry( } logrus.Debugf("Max retries exceeded for group %s after %d attempts. Parsed Error: %s", group.Name, retryCount, logMessage) - keyID, _ := strconv.ParseUint(lastError.KeyID, 10, 64) - ps.logRequest(c, group, uint(keyID), startTime, lastError.StatusCode, retryCount, errors.New(logMessage), isStream, lastError.UpstreamAddr) + ps.logRequest(c, group, &models.APIKey{KeyValue: lastError.KeyValue}, startTime, lastError.StatusCode, retryCount, errors.New(logMessage), isStream, lastError.UpstreamAddr) } else { response.Error(c, app_errors.ErrMaxRetriesExceeded) logrus.Debugf("Max retries exceeded for group %s after %d attempts.", group.Name, retryCount) - ps.logRequest(c, group, 0, startTime, http.StatusServiceUnavailable, retryCount, app_errors.ErrMaxRetriesExceeded, isStream, "") + ps.logRequest(c, group, nil, startTime, http.StatusServiceUnavailable, retryCount, app_errors.ErrMaxRetriesExceeded, isStream, "") } return } @@ -129,7 +127,7 @@ func (ps *ProxyServer) executeRequestWithRetry( if err != nil { logrus.Errorf("Failed to select a key for group %s on attempt %d: %v", group.Name, retryCount+1, err) response.Error(c, app_errors.NewAPIError(app_errors.ErrNoKeysAvailable, err.Error())) - ps.logRequest(c, group, 0, startTime, http.StatusServiceUnavailable, retryCount, err, isStream, "") + ps.logRequest(c, group, nil, startTime, http.StatusServiceUnavailable, retryCount, err, isStream, "") return } @@ -177,7 +175,7 @@ func (ps *ProxyServer) executeRequestWithRetry( if err != nil || (resp != nil && resp.StatusCode >= 400) { if err != nil && app_errors.IsIgnorableError(err) { logrus.Debugf("Client-side ignorable error for key %s, aborting retries: %v", utils.MaskAPIKey(apiKey.KeyValue), err) - ps.logRequest(c, group, apiKey.ID, startTime, 499, retryCount+1, err, isStream, upstreamURL) + ps.logRequest(c, group, apiKey, startTime, 499, retryCount+1, err, isStream, upstreamURL) return } @@ -210,7 +208,7 @@ func (ps *ProxyServer) executeRequestWithRetry( StatusCode: statusCode, ErrorMessage: errorMessage, ParsedErrorMessage: parsedError, - KeyID: fmt.Sprintf("%d", apiKey.ID), + KeyValue: apiKey.KeyValue, Attempt: retryCount + 1, UpstreamAddr: upstreamURL, }) @@ -220,7 +218,7 @@ func (ps *ProxyServer) executeRequestWithRetry( // ps.keyProvider.UpdateStatus(apiKey, group, true) // 请求成功不再重置成功次数，减少IO消耗 logrus.Debugf("Request for group %s succeeded on attempt %d with key %s", group.Name, retryCount+1, utils.MaskAPIKey(apiKey.KeyValue)) - ps.logRequest(c, group, apiKey.ID, startTime, resp.StatusCode, retryCount+1, nil, isStream, upstreamURL) + ps.logRequest(c, group, apiKey, startTime, resp.StatusCode, retryCount+1, nil, isStream, upstreamURL) for key, values := range resp.Header { for _, value := range values { @@ -240,7 +238,7 @@ func (ps *ProxyServer) executeRequestWithRetry( func (ps *ProxyServer) logRequest( c *gin.Context, group *models.Group, - keyID uint, + apiKey *models.APIKey, startTime time.Time, statusCode int, retries int, @@ -256,7 +254,7 @@ func (ps *ProxyServer) logRequest( logEntry := &models.RequestLog{ GroupID: group.ID, - KeyID: keyID, + GroupName: group.Name, IsSuccess: finalError == nil && statusCode < 400, SourceIP: c.ClientIP(), StatusCode: statusCode, @@ -267,6 +265,9 @@ func (ps *ProxyServer) logRequest( IsStream: isStream, UpstreamAddr: utils.TruncateString(upstreamAddr, 500), } + if apiKey != nil { + logEntry.KeyValue = apiKey.KeyValue + } if finalError != nil { logEntry.ErrorMessage = finalError.Error() diff --git a/internal/services/request_log_service.go b/internal/services/request_log_service.go index dd232f0..26d57e1 100644 --- a/internal/services/request_log_service.go +++ b/internal/services/request_log_service.go @@ -207,24 +207,24 @@ func (s *RequestLogService) writeLogsToDB(logs []*models.RequestLog) error { return fmt.Errorf("failed to batch insert request logs: %w", err) } - keyStats := make(map[uint]int64) + keyStats := make(map[string]int64) for _, log := range logs { - if log.IsSuccess { - keyStats[log.KeyID]++ + if log.IsSuccess && log.KeyValue != "" { + keyStats[log.KeyValue]++ } } if len(keyStats) > 0 { var caseStmt strings.Builder - var keyIDs []uint - caseStmt.WriteString("CASE id ") - for keyID, count := range keyStats { - caseStmt.WriteString(fmt.Sprintf("WHEN %d THEN request_count + %d ", keyID, count)) - keyIDs = append(keyIDs, keyID) + var keyValues []string + caseStmt.WriteString("CASE key_value ") + for keyValue, count := range keyStats { + caseStmt.WriteString(fmt.Sprintf("WHEN '%s' THEN request_count + %d ", keyValue, count)) + keyValues = append(keyValues, keyValue) } caseStmt.WriteString("END") - if err := tx.Model(&models.APIKey{}).Where("id IN ?", keyIDs). + if err := tx.Model(&models.APIKey{}).Where("key_value IN ?", keyValues). Updates(map[string]any{ "request_count": gorm.Expr(caseStmt.String()), "last_used_at": time.Now(), diff --git a/internal/types/types.go b/internal/types/types.go index ccee089..ffa276a 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -85,7 +85,7 @@ type RetryError struct { StatusCode int `json:"status_code"` ErrorMessage string `json:"error_message"` ParsedErrorMessage string `json:"-"` - KeyID string `json:"key_id"` + KeyValue string `json:"key_value"` Attempt int `json:"attempt"` UpstreamAddr string `json:"-"` }