From ea838441c1d493830c3c08b21ef42ce624147c93 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 17 Mar 2026 18:01:50 +0800 Subject: [PATCH] =?UTF-8?q?api=E7=9B=91=E6=8E=A7=EF=BC=8Ces=E5=85=BC?= =?UTF-8?q?=E5=AE=B9=E8=96=AA=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- es/es_config.go | 170 +++++++++ es/es_search.go | 342 +++++++++++++++--- health_api.go | 455 ++++++++++++------------ main.go | 34 +- monitor/api_monitor.go | 774 +++++++++++++++++++++++++++++++++++++++++ monitor/health_api.go | 773 ++++++++++++++++++++++++++++++++++++++++ service/book.go | 144 ++++---- 7 files changed, 2320 insertions(+), 372 deletions(-) create mode 100644 es/es_config.go create mode 100644 monitor/api_monitor.go create mode 100644 monitor/health_api.go diff --git a/es/es_config.go b/es/es_config.go new file mode 100644 index 0000000..8fe5f76 --- /dev/null +++ b/es/es_config.go @@ -0,0 +1,170 @@ +package es + +// ESFieldConfig ES 字段配置 +type ESFieldConfig struct { + // AllowUpdate 允许更新的字段列表 + AllowUpdate map[string]bool + + // AllowAdd 允许新增时填充的字段列表 + AllowAdd map[string]bool + + // FieldMappings 字段映射（Go 结构体字段名 -> ES 字段名） + FieldMappings map[string]string +} + +// GetESFieldConfig 获取 ES 字段配置 +func GetESFieldConfig() *ESFieldConfig { + return &ESFieldConfig{ + AllowUpdate: map[string]bool{ + "book_pic": true, + "book_pic_s": true, + "book_pic_b": true, + "book_pic_w": true, + "book_def_pic": true, + "isbn": true, + "author": true, + "category": true, + "publisher": true, + "publication_time": true, + "binding_layout": true, + "fix_price": true, + "content": true, + "is_suit": true, + "day_sale_7": true, + "day_sale_15": true, + "day_sale_30": true, + "day_sale_60": true, + "day_sale_90": true, + "day_sale_180": true, + "day_sale_365": true, + "this_year_sale": true, + "last_year_sale": true, + "total_sale": true, + "buy_counts": true, + "sell_counts": true, + "is_illegal": true, + "is_return": true, + "is_filter": true, + "update_time": true, + "page_count": true, + "word_count": true, + "book_format": true, + "cat_id": true, + "other": true, // 允许更新 other 字段 + }, + + AllowAdd: map[string]bool{ + "id": true, + "book_name": true, + "book_pic": true, + "book_pic_s": true, + "book_pic_b": true, + "book_pic_w": true, + "isbn": true, + "author": true, + "category": true, + "publisher": true, + "publication_time": true, + "binding_layout": true, + "fix_price": true, + "content": true, + "is_suit": true, + "day_sale_7": true, + "day_sale_15": true, + "day_sale_30": true, + "day_sale_60": true, + "day_sale_90": true, + "day_sale_180": true, + "day_sale_365": true, + "this_year_sale": true, + "last_year_sale": true, + "total_sale": true, + "buy_counts": true, + "sell_counts": true, + "book_pic_obj": true, + "book_pic_obj_s": true, + "update_time": true, + "is_illegal": true, + "is_return": true, + "is_filter": true, + "page_count": true, + "word_count": true, + "book_format": true, + "other": true, // 允许新增 other 字段 + }, + + FieldMappings: map[string]string{ + "ID": "id", + "BookName": "book_name", + "BookPic": "book_pic", + "BookPicS": "book_pic_s", + "BookPicB": "book_pic_b", + "BookPicW": "book_pic_w", + "ISBN": "isbn", + "Author": "author", + "Category": "category", + "Publisher": "publisher", + "PublicationTime": "publication_time", + "BindingLayout": "binding_layout", + "FixPrice": "fix_price", + "Content": "content", + "IsSuit": "is_suit", + "DaySale7": "day_sale_7", + "DaySale15": "day_sale_15", + "DaySale30": "day_sale_30", + "DaySale60": "day_sale_60", + "DaySale90": "day_sale_90", + "DaySale180": "day_sale_180", + "DaySale365": "day_sale_365", + "ThisYearSale": "this_year_sale", + "LastYearSale": "last_year_sale", + "TotalSale": "total_sale", + "BuyCounts": "buy_counts", + "SellCounts": "sell_counts", + "BookPicObj": "book_pic_obj", + "BookPicObjS": "book_pic_obj_s", + "UpdateTime": "update_time", + "IsIllegal": "is_illegal", + "IsReturn": "is_return", + "IsFilter": "is_filter", + "PageCount": "page_count", + "WordCount": "word_count", + "BookFormat": "book_format", + "Other": "other", + }, + } +} + +// IsAllowUpdate 检查字段是否允许更新 +func (cfg *ESFieldConfig) IsAllowUpdate(field string) bool { + return cfg.AllowUpdate[field] +} + +// IsAllowAdd 检查字段是否允许新增 +func (cfg *ESFieldConfig) IsAllowAdd(field string) bool { + return cfg.AllowAdd[field] +} + +// GetESFieldName 获取 ES 字段名 +func (cfg *ESFieldConfig) GetESFieldName(goFieldName string) string { + if esName, ok := cfg.FieldMappings[goFieldName]; ok { + return esName + } + return goFieldName +} + +// BuildESDocument 构建 ES 文档 +func (cfg *ESFieldConfig) BuildESDocument(data map[string]interface{}) map[string]interface{} { + doc := make(map[string]interface{}) + + for goField, value := range data { + // 获取 ES 字段名 + esField := cfg.GetESFieldName(goField) + // 检查是否允许添加 + if !cfg.IsAllowAdd(esField) { + continue + } + doc[esField] = value + } + return doc +} diff --git a/es/es_search.go b/es/es_search.go index 09aaef6..3f7f843 100644 --- a/es/es_search.go +++ b/es/es_search.go @@ -6,11 +6,14 @@ import ( "centerBook/image" "centerBook/kongfz" "centerBook/model/request" + "centerBook/monitor" "centerBook/tail" "centerBook/util/redisClient" "context" "encoding/json" "fmt" + "github.com/gin-gonic/gin" + jsoniter "github.com/json-iterator/go" "io" "log" "net/http" @@ -18,9 +21,6 @@ import ( "strings" "time" - "github.com/gin-gonic/gin" - jsoniter "github.com/json-iterator/go" - "github.com/elastic/go-elasticsearch/v8/esapi" ) @@ -59,9 +59,14 @@ type ESBookResponse struct { BookPicObj map[string]interface{} `json:"book_pic_obj"` BookPicObjS map[string]interface{} `json:"book_pic_obj_s"` UpdateTime NumberOrString `json:"update_time"` - IsIllegal int `json:"is_illegal"` // 是否非法 示例 000000 - IsReturn int `json:"is_return"` // 是否为驳回 示例 0 否 1 是 - IsFilter string `json:"is_filter"` // 过滤字段 + IsIllegal int `json:"is_illegal"` // 是否非法 示例 000000 + IsReturn int `json:"is_return"` // 是否为驳回 示例 0 否 1 是 + IsFilter string `json:"is_filter"` // 过滤字段 + PageCount NumberOrString `json:"page_count"` // 页数 + WordCount NumberOrString `json:"word_count"` // 字数 + BookFormat NumberOrString `json:"book_format"` // 多少开 + //CatId request.CatIdObject `json:"cat_id"` // 类目 + Other map[string]interface{} `json:"other"` // 扩展字段，用于兼容未来新增字段 } // FlexibleString 处理可能是字符串或数组的字段 @@ -188,6 +193,11 @@ func (book *ESBook) ConvertToResponse() ESBookResponse { IsIllegal: book.IsIllegal, IsReturn: book.IsReturn, IsFilter: book.IsFilter, + PageCount: book.PageCount, + WordCount: book.WordCount, + BookFormat: book.BookFormat, + //CatId: book.CatId, + Other: book.Other, } } @@ -250,6 +260,7 @@ type ESBook struct { WordCount NumberOrString `json:"word_count"` // 字数 BookFormat NumberOrString `json:"book_format"` // 多少开 CatId request.CatIdObject `json:"cat_id"` // 类目 + Other map[string]interface{} `json:"other"` // 扩展字段，用于兼容未来新增字段 } // AddBookRequest 用于 Service 方法的入参 @@ -459,7 +470,7 @@ type esHitsWrapper struct { } // SearchBooks 搜索图书 -func (svc *ESSearchService) SearchBooks(keyword string) ([]ESBook, error) { +func (svc *ESSearchService) SearchBooks(keyword string, endpoint ...string) ([]ESBook, error) { keyword = strings.TrimSpace(keyword) if keyword == "" { @@ -494,7 +505,18 @@ func (svc *ESSearchService) SearchBooks(keyword string) ([]ESBook, error) { TrackTotalHits: true, } - res, err := req.Do(context.Background(), svc.ES.Client.Transport) + // 如果有传入 endpoint，使用监控 + var res *esapi.Response + var duration time.Duration + + if len(endpoint) > 0 && endpoint[0] != "" { + monitoredES := monitor.NewMonitoredESClient(svc.ES.Client, endpoint[0]) + res, duration, err = monitoredES.Search(context.Background(), &req) + log.Printf("[SearchBooks] ES 查询耗时：%dms", duration.Milliseconds()) + } else { + // 原有逻辑，不带监控 + res, err = req.Do(context.Background(), svc.ES.Client.Transport) + } if err != nil { return nil, fmt.Errorf("执行 ES 查询失败: %v", err) } @@ -1918,20 +1940,41 @@ func (svc *ESSearchService) BatchGetBookBaseInfoES(c *gin.Context) ([]ESBook, in fmt.Printf("[DEBUG] ES Query Body:\n%s\n", string(body)) // ========== 执行 ES 查询 ========== - res, err := svc.ES.Client.Search( - svc.ES.Client.Search.WithIndex(ESIndex), - svc.ES.Client.Search.WithBody(bytes.NewReader(body)), - svc.ES.Client.Search.WithTrackTotalHits(true), - ) + //res, err := svc.ES.Client.Search( + // svc.ES.Client.Search.WithIndex(ESIndex), + // svc.ES.Client.Search.WithBody(bytes.NewReader(body)), + // svc.ES.Client.Search.WithTrackTotalHits(true), + //) + // + //fmt.Printf("[DEBUG] ES Query Response:\n%s\n", res) + // + //if err != nil { + // fmt.Printf("[ERROR] ES.Client.Search error: %v\n", err) + // return nil, 0, err + //} + //defer res.Body.Close() + + endpoint := c.FullPath() + monitoredES := monitor.NewMonitoredESClient(svc.ES.Client, endpoint) + + queryBody := string(body) + req := esapi.SearchRequest{ + Index: []string{ESIndex}, + Body: bytes.NewReader([]byte(queryBody)), + TrackTotalHits: true, + Pretty: true, + } + + res, duration, err := monitoredES.Search(context.Background(), &req) fmt.Printf("[DEBUG] ES Query Response:\n%s\n", res) + fmt.Printf("[DEBUG] ES 查询耗时：%dms\n", duration.Milliseconds()) if err != nil { fmt.Printf("[ERROR] ES.Client.Search error: %v\n", err) return nil, 0, err } defer res.Body.Close() - // 读取响应 var buf bytes.Buffer // 使用CopyBuffer可以重用缓冲区 @@ -2044,25 +2087,27 @@ func (svc *ESSearchService) BatchGetBookBaseInfoESHandler(c *gin.Context) { "total": total, }) } - func (svc *ESSearchService) SearchBooksHandler(c *gin.Context) { isbn := c.Query("isbn") if isbn == "" { + log.Printf("[SearchBooksHandler] 缺少 isbn 参数") c.JSON(400, gin.H{"error": "缺少 isbn 参数"}) return } + log.Printf("[SearchBooksHandler] ISBN 模糊搜索：%s", isbn) ctx := context.Background() + endpoint := c.FullPath() - db4Client, err := redisClient.GetClientByName("db1") + // 获取监控的 Redis 客户端 + db1Client, err := redisClient.GetClientByName("db1") if err == nil { - val, err := db4Client.Get(ctx, isbn).Result() + monitoredRedis := monitor.NewMonitoredRedisClient(db1Client, endpoint) + val, _, err := monitoredRedis.Get(ctx, isbn) if err == nil && val != "" { - log.Printf("[SearchBooksHandler] 从 Redis db1 查询到数据: %s", isbn) - // 使用 RedisBookInfo 结构体解析 + log.Printf("[SearchBooksHandler] 从 Redis db1 查询到数据：%s", isbn) var redisBook request.BookInfo if err := json.Unmarshal([]byte(val), &redisBook); err == nil { - // 转换为 ESBook esBook := ConvertRedisBookToESBook(&redisBook) if esBook != nil { responseData := esBook.ConvertToResponse() @@ -2071,13 +2116,12 @@ func (svc *ESSearchService) SearchBooksHandler(c *gin.Context) { }) return } - } else { - log.Printf("[SearchBookByISBNHandler] Redis 数据解析失败：%v", err) } } } - result, err := svc.SearchBooks(isbn) + // ES 查询 + result, err := svc.SearchBooks(isbn, endpoint) if err != nil { c.JSON(500, gin.H{"error": "ES 查询失败", "details": err.Error()}) return @@ -2105,20 +2149,17 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) { log.Printf("[SearchBookByISBNHandler] 查询 ISBN: %s", isbn) ctx := context.Background() + endpoint := c.FullPath() - db4Client, err := redisClient.GetClientByName("db1") - fmt.Println(db4Client) - - if err != nil { - log.Printf("[SearchBookByISBNHandler] 获取 Redis db1 客户端失败: %v", err) - } else { - val, err := db4Client.Get(ctx, isbn).Result() + // Redis 查询（使用监控） + db1Client, err := redisClient.GetClientByName("db1") + if err == nil { + monitoredRedis := monitor.NewMonitoredRedisClient(db1Client, endpoint) + val, _, err := monitoredRedis.Get(ctx, isbn) if err == nil && val != "" { - log.Printf("[SearchBookByISBNHandler] 从 Redis db1 查询到数据: %s", isbn) - // 使用 RedisBookInfo 结构体解析 + log.Printf("[SearchBookByISBNHandler] 从 Redis db1 查询到数据：%s", isbn) var redisBook request.BookInfo if err := json.Unmarshal([]byte(val), &redisBook); err == nil { - // 转换为 ESBook esBook := ConvertRedisBookToESBook(&redisBook) if esBook != nil { responseData := esBook.ConvertToResponse() @@ -2130,24 +2171,70 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) { } else { log.Printf("[SearchBookByISBNHandler] Redis 数据解析失败：%v", err) } - } else { - log.Printf("[SearchBookByISBNHandler] Redis db1 中未找到 ISBN: %s", isbn) } } - result, err := svc.SearchBookByISBN(isbn) + // ES 查询（使用监控） + query := map[string]interface{}{ + "query": map[string]interface{}{ + "term": map[string]interface{}{ + "isbn": isbn, + }, + }, + "_source": true, + } + + body, err := json.Marshal(query) if err != nil { - log.Printf("[SearchBookByISBNHandler] ES 查询失败: %v", err) - c.JSON(500, gin.H{"error": err.Error()}) + log.Printf("[SearchBookByISBNHandler] 构建查询 JSON 失败：%v", err) + c.JSON(500, gin.H{"error": "构建查询失败"}) return } + req := esapi.SearchRequest{ + Index: []string{ESIndex}, + Body: bytes.NewReader(body), + TrackTotalHits: true, + Pretty: true, + } + + // 创建监控客户端并执行查询 + monitoredES := monitor.NewMonitoredESClient(svc.ES.Client, endpoint) + resp, duration, err := monitoredES.Search(ctx, &req) + + log.Printf("[SearchBookByISBNHandler] ES 查询耗时：%dms", duration.Milliseconds()) + + if err != nil { + log.Printf("[SearchBookByISBNHandler] ES 查询失败：%v", err) + c.JSON(500, gin.H{"error": "ES 查询失败：" + err.Error()}) + return + } + defer resp.Body.Close() + + if resp.IsError() { + log.Printf("[SearchBookByISBNHandler] ES 返回错误：%s", resp.String()) + c.JSON(500, gin.H{"error": "ES 返回错误：" + resp.String()}) + return + } + + var parsed esHitsWrapper + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + log.Printf("[SearchBookByISBNHandler] 解析 ES 响应失败：%v", err) + c.JSON(500, gin.H{"error": "解析响应失败"}) + return + } + + var result *ESBook + if len(parsed.Hits.Hits) > 0 { + result = &parsed.Hits.Hits[0].Source + } + if result == nil { log.Printf("[SearchBookByISBNHandler] ES 中未找到 ISBN: %s，从孔夫子抓取", isbn) apiBook, err := kongfz.GetBookImageByISBN(isbn, "CALF_ELEPHANT_PROXY", "1297757178467602432", "QgQBvP7f") if err != nil { - log.Printf("[SearchBookByISBNHandler] 孔夫子 API 查询失败: %v", err) + log.Printf("[SearchBookByISBNHandler] 孔夫子 API 查询失败：%v", err) c.JSON(500, gin.H{"error": err.Error()}) return } @@ -2157,16 +2244,16 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) { return } - log.Printf("[SearchBookByISBNHandler] 获取到图书信息: %+v", apiBook.Data) + log.Printf("[SearchBookByISBNHandler] 获取到图书信息：%+v", apiBook.Data) pddBookPicURL := "" if apiBook.Data.BookPic != "" { url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPic, isbn, "true", apiBook.Data.BookName, "true") if err != nil { - log.Printf("[SearchBookByISBNHandler] 上传 book_pic 失败: %v", err) + log.Printf("[SearchBookByISBNHandler] 上传 book_pic 失败：%v", err) } else { pddBookPicURL = url - log.Printf("[SearchBookByISBNHandler] 上传 book_pic 成功: %s", url) + log.Printf("[SearchBookByISBNHandler] 上传 book_pic 成功：%s", url) } } @@ -2174,10 +2261,10 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) { if apiBook.Data.BookPicS != "" { url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPicS, isbn, "true", apiBook.Data.BookName, "true") if err != nil { - log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 失败: %v", err) + log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 失败：%v", err) } else { pddBookPicSURL = url - log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 成功: %s", url) + log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 成功：%s", url) } } @@ -2185,18 +2272,15 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) { esBook.BookPicS.PddResponse = pddBookPicSURL esBook.BookPic.PddPath = pddBookPicURL - //log.Printf("[SearchBookByISBNHandler] 写入 ES: %+v", esBook) result, err = svc.AddBookToES(c.Request.Context(), esBook) if err != nil { - log.Printf("[SearchBookByISBNHandler] 写入 ES 失败: %v", err) + log.Printf("[SearchBookByISBNHandler] 写入 ES 失败：%v", err) c.JSON(500, gin.H{"error": err.Error()}) return } - log.Printf("[SearchBookByISBNHandler] 写入 ES 成功, ISBN: %s", isbn) - } else { - //log.Printf("[SearchBookByISBNHandler] 从 ES 查询到图书: %+v", result) + log.Printf("[SearchBookByISBNHandler] 写入 ES 成功，ISBN: %s", isbn) } responseData := result.ConvertToResponse() @@ -2205,6 +2289,166 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) { }) } +// func (svc *ESSearchService) SearchBooksHandler(c *gin.Context) { +// isbn := c.Query("isbn") +// if isbn == "" { +// c.JSON(400, gin.H{"error": "缺少 isbn 参数"}) +// return +// } +// +// ctx := context.Background() +// +// db4Client, err := redisClient.GetClientByName("db1") +// if err == nil { +// val, err := db4Client.Get(ctx, isbn).Result() +// if err == nil && val != "" { +// log.Printf("[SearchBooksHandler] 从 Redis db1 查询到数据: %s", isbn) +// // 使用 RedisBookInfo 结构体解析 +// var redisBook request.BookInfo +// if err := json.Unmarshal([]byte(val), &redisBook); err == nil { +// // 转换为 ESBook +// esBook := ConvertRedisBookToESBook(&redisBook) +// if esBook != nil { +// responseData := esBook.ConvertToResponse() +// c.JSON(200, gin.H{ +// "data": responseData, +// }) +// return +// } +// } else { +// log.Printf("[SearchBookByISBNHandler] Redis 数据解析失败：%v", err) +// } +// } +// } +// +// result, err := svc.SearchBooks(isbn) +// if err != nil { +// c.JSON(500, gin.H{"error": "ES 查询失败", "details": err.Error()}) +// return +// } +// +// responseList := make([]ESBookResponse, 0, len(result)) +// for _, book := range result { +// responseList = append(responseList, book.ConvertToResponse()) +// } +// +// c.JSON(200, gin.H{ +// "count": len(result), +// "data": responseList, +// }) +// } +//func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) { +// +// isbn := c.Query("isbn") +// if isbn == "" { +// log.Printf("[SearchBookByISBNHandler] 缺少 isbn 参数") +// c.JSON(400, gin.H{"error": "缺少 isbn 参数"}) +// return +// } +// +// log.Printf("[SearchBookByISBNHandler] 查询 ISBN: %s", isbn) +// +// ctx := context.Background() +// +// db4Client, err := redisClient.GetClientByName("db1") +// fmt.Println(db4Client) +// +// if err != nil { +// log.Printf("[SearchBookByISBNHandler] 获取 Redis db1 客户端失败: %v", err) +// } else { +// val, err := db4Client.Get(ctx, isbn).Result() +// if err == nil && val != "" { +// log.Printf("[SearchBookByISBNHandler] 从 Redis db1 查询到数据: %s", isbn) +// // 使用 RedisBookInfo 结构体解析 +// var redisBook request.BookInfo +// if err := json.Unmarshal([]byte(val), &redisBook); err == nil { +// // 转换为 ESBook +// esBook := ConvertRedisBookToESBook(&redisBook) +// if esBook != nil { +// responseData := esBook.ConvertToResponse() +// c.JSON(200, gin.H{ +// "data": responseData, +// }) +// return +// } +// } else { +// log.Printf("[SearchBookByISBNHandler] Redis 数据解析失败：%v", err) +// } +// } else { +// log.Printf("[SearchBookByISBNHandler] Redis db1 中未找到 ISBN: %s", isbn) +// } +// } +// +// result, err := svc.SearchBookByISBN(isbn) +// if err != nil { +// log.Printf("[SearchBookByISBNHandler] ES 查询失败: %v", err) +// c.JSON(500, gin.H{"error": err.Error()}) +// return +// } +// +// if result == nil { +// log.Printf("[SearchBookByISBNHandler] ES 中未找到 ISBN: %s，从孔夫子抓取", isbn) +// +// apiBook, err := kongfz.GetBookImageByISBN(isbn, "CALF_ELEPHANT_PROXY", "1297757178467602432", "QgQBvP7f") +// if err != nil { +// log.Printf("[SearchBookByISBNHandler] 孔夫子 API 查询失败: %v", err) +// c.JSON(500, gin.H{"error": err.Error()}) +// return +// } +// if apiBook == nil || apiBook.Data.ISBN == "" { +// log.Printf("[SearchBookByISBNHandler] 孔夫子 API 未找到图书信息 ISBN: %s", isbn) +// c.JSON(404, gin.H{"error": "未找到图书信息"}) +// return +// } +// +// log.Printf("[SearchBookByISBNHandler] 获取到图书信息: %+v", apiBook.Data) +// +// pddBookPicURL := "" +// if apiBook.Data.BookPic != "" { +// url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPic, isbn, "true", apiBook.Data.BookName, "true") +// if err != nil { +// log.Printf("[SearchBookByISBNHandler] 上传 book_pic 失败: %v", err) +// } else { +// pddBookPicURL = url +// log.Printf("[SearchBookByISBNHandler] 上传 book_pic 成功: %s", url) +// } +// } +// +// pddBookPicSURL := "" +// if apiBook.Data.BookPicS != "" { +// url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPicS, isbn, "true", apiBook.Data.BookName, "true") +// if err != nil { +// log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 失败: %v", err) +// } else { +// pddBookPicSURL = url +// log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 成功: %s", url) +// } +// } +// +// esBook := ConvertKongfzToESBook(apiBook) +// +// esBook.BookPicS.PddResponse = pddBookPicSURL +// esBook.BookPic.PddPath = pddBookPicURL +// //log.Printf("[SearchBookByISBNHandler] 写入 ES: %+v", esBook) +// +// result, err = svc.AddBookToES(c.Request.Context(), esBook) +// if err != nil { +// log.Printf("[SearchBookByISBNHandler] 写入 ES 失败: %v", err) +// c.JSON(500, gin.H{"error": err.Error()}) +// return +// } +// +// log.Printf("[SearchBookByISBNHandler] 写入 ES 成功, ISBN: %s", isbn) +// } else { +// //log.Printf("[SearchBookByISBNHandler] 从 ES 查询到图书: %+v", result) +// } +// +// responseData := result.ConvertToResponse() +// c.JSON(200, gin.H{ +// "data": responseData, +// }) +//} + // ConvertKongfzToESBook 将第三方接口返回的数据转换为 ESBook 结构 func ConvertKongfzToESBook(apiBook *kongfz.BookResponse) *ESBook { if apiBook == nil || apiBook.Data.ISBN == "" { diff --git a/health_api.go b/health_api.go index 1cba748..c6bddcd 100644 --- a/health_api.go +++ b/health_api.go @@ -1,11 +1,10 @@ package main import ( + "github.com/gin-gonic/gin" "net/http" "strconv" "time" - - "github.com/gin-gonic/gin" ) // SQLHealthController SQL健康监控控制器 @@ -140,253 +139,253 @@ func (shc *SQLHealthController) GetSQLHealthDashboard(c *gin.Context) { - - - SQL健康监控仪表板 - + + + SQL健康监控仪表板 + -
-

SQL健康监控仪表板

- -
- - - - - -
+
+

SQL健康监控仪表板

-
-

SQL执行统计

-
- -
-
+
+ + + + + +
-
-

最近SQL执行记录

-
- -
-
+
+

SQL执行统计

+
+ +
+
-
-

慢查询 (>1000ms)

-
- -
-
+
+

最近SQL执行记录

+
+ +
+
-
-

失败查询

-
- -
-
-
+
+

慢查询 (>1000ms)

+
+ +
+
- + let html = ''; + failedQueries.forEach(query => { + html += ` + "`" + ` + + + + + + ` + "`" + `; + }); + html += '
ID查询语句错误信息时间接口
${query.id}${query.query}${query.error}${new Date(query.timestamp).toLocaleString()}${query.endpoint}
'; + container.innerHTML = html; + }) + .catch(error => console.error('加载失败查询失败:', error)); + } + + function clearRecords() { + if (confirm('确定要清空所有SQL记录吗？')) { + fetch('/api/sql-health/clear', { method: 'POST' }) + .then(response => response.json()) + .then(data => { + alert('记录已清空'); + refreshData(); + }) + .catch(error => console.error('清空记录失败:', error)); + } + } + + function updateLimit() { + const limitInput = document.getElementById('limitInput'); + const newLimit = parseInt(limitInput.value); + if (newLimit > 0 && newLimit <= 500) { + currentLimit = newLimit; + loadRecentRecords(); + } else { + alert('记录数量必须在1-500之间'); + } + } + + function toggleAutoRefresh() { + const checkbox = document.getElementById('autoRefresh'); + if (checkbox.checked) { + autoRefreshInterval = setInterval(refreshData, 10000); + } else { + clearInterval(autoRefreshInterval); + } + } + + // 初始化 + document.getElementById('autoRefresh').addEventListener('change', toggleAutoRefresh); + refreshData(); + toggleAutoRefresh(); + - ` + ` c.Header("Content-Type", "text/html; charset=utf-8") c.String(http.StatusOK, dashboardHTML) diff --git a/main.go b/main.go index 6870c43..620de71 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ import ( "github.com/go-redis/redis/v8" "centerBook/es" + "centerBook/monitor" "centerBook/util/redisClient" "github.com/gin-gonic/gin" @@ -284,20 +285,27 @@ func main() { //r.GET("/ready", bookCenter.ReadyCheck) // 7. SQL健康监控端点 - sqlHealthController := NewSQLHealthController() - r.GET("/api/sql-health/stats", sqlHealthController.GetSQLStats) - r.GET("/api/sql-health/recent", sqlHealthController.GetRecentSQLRecords) - r.GET("/api/sql-health/slow-queries", sqlHealthController.GetSlowQueries) - r.GET("/api/sql-health/failed-queries", sqlHealthController.GetFailedQueries) - r.POST("/api/sql-health/clear", sqlHealthController.ClearSQLRecords) - r.GET("/api/sql-health/dashboard", sqlHealthController.GetSQLHealthDashboard) - + //sqlHealthController := NewSQLHealthController() + //r.GET("/api/sql-health/stats", sqlHealthController.GetSQLStats) + //r.GET("/api/sql-health/recent", sqlHealthController.GetRecentSQLRecords) + //r.GET("/api/sql-health/slow-queries", sqlHealthController.GetSlowQueries) + //r.GET("/api/sql-health/failed-queries", sqlHealthController.GetFailedQueries) + //r.POST("/api/sql-health/clear", sqlHealthController.ClearSQLRecords) + //r.GET("/api/sql-health/dashboard", sqlHealthController.GetSQLHealthDashboard) + // 8. API 监控端点（ES 和 Redis 调用监控） + apiMonitorController := monitor.NewAPIMonitorController() + r.GET("/api/api-monitor/stats", apiMonitorController.GetAPIStats) + r.GET("/api/api-monitor/all-stats", apiMonitorController.GetAllAPIStats) + r.GET("/api/api-monitor/es-calls", apiMonitorController.GetAPIESCalls) + r.GET("/api/api-monitor/redis-calls", apiMonitorController.GetAPIRedisCalls) + r.GET("/api/api-monitor/dashboard", apiMonitorController.GetAPIMonitorDashboard) + r.GET("/api/api-monitor/call-detail", apiMonitorController.GetAPICallDetail) // =================== ⭐ ES ===================== // ISBN 模糊搜索 - r.GET("/api/es/searchByISBNLike", esService.SearchBooksHandler) + r.GET("/api/es/searchByISBNLike", esService.SearchBooksHandler) //监控 // ISBN 精确搜索 - r.GET("/api/es/searchByISBN", esService.SearchBookByISBNHandler) //1 + r.GET("/api/es/searchByISBN", esService.SearchBookByISBNHandler) //监控 // 书名搜索 r.GET("/api/es/searchByBookName", esService.SearchBookByBookNameHandler) // 全字段搜索 @@ -312,7 +320,7 @@ func main() { // 根据条件查询 ES 图书信息 r.GET("/api/es/getBookBaseInfoES", bookController.SearchBookBaseInfoHandler) // 新增：根据ISBN查询ES中是否存在，不存在则新增数据,存在则根据参数更新 - //r.POST("/api/es/addBookToES", bookController.AddBookToESHandler) + r.POST("/api/es/addBookToES", bookController.AddBookToESHandler) // 更新：根据ISBN通用更新图书字段 r.POST("/api/es/updateBookFieldsByISBN", bookController.UpdateBookFieldsByISBNHandler) // 更新：根据ISBN通用更新图书字段 @@ -324,7 +332,7 @@ func main() { //------------------------------------------------------------------------ // 新：核价软件用批量获取 - r.GET("/api/es/batchGetBookBaseInfoES", esService.BatchGetBookBaseInfoESHandler) + r.GET("/api/es/batchGetBookBaseInfoES", esService.BatchGetBookBaseInfoESHandler) //监控 // 多条件高级搜索 r.GET("/api/es/searchAdvanced", esService.SearchBooksByConditionsHandler) // ID 范围计数 @@ -338,7 +346,7 @@ func main() { // 新增：根据ISBN通用更新图书字段 //r.POST("/api/es/updateBookFieldsByISBN", esService.UpdateBookFieldsByISBNHandler) //1 // 新增：根据ISBN查询ES中是否存在，不存在则新增数据,存在则根据参数更新 - r.POST("/api/es/addBookToES", esService.AddBookToESHandler) //1 + //r.POST("/api/es/addBookToES", esService.AddBookToESHandler) //1 // 新增：完整插入接口，支持所有字段， r.POST("/api/es/addBookFullToES", esService.AddBookFullToESHandler) // 新增：批量插入接口，支持同时插入多本图书 diff --git a/monitor/api_monitor.go b/monitor/api_monitor.go new file mode 100644 index 0000000..ea359bd --- /dev/null +++ b/monitor/api_monitor.go @@ -0,0 +1,774 @@ +package monitor + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "io" + "strings" + "sync" + "time" + + "github.com/go-redis/redis/v8" +) + +// APIMonitor API 监控器 - 监控特定接口的 ES 和 Redis 调用 +type APIMonitor struct { + endpoint string + esCalls []APICallRecord + redisCalls []APICallRecord + mutex sync.RWMutex + //maxRecords int + nextID int64 + qpsWindow time.Duration // QPS 统计时间窗口 +} + +// APICallRecord API 调用记录 +type APICallRecord struct { + ID int64 `json:"id"` + Timestamp time.Time `json:"timestamp"` + Duration int64 `json:"duration_ms"` + Success bool `json:"success"` + Error string `json:"error,omitempty"` + Operation string `json:"operation"` // ES: search/get, Redis: GET/SET/HGET + KeyOrIndex string `json:"key_or_index"` // ES 索引或 Redis Key + Query string `json:"query,omitempty"` // 查询内容 + Request string `json:"request,omitempty"` // 请求内容 + Response string `json:"response,omitempty"` // 响应内容 +} + +// APIStats API 统计数据 +type APIStats struct { + Endpoint string `json:"endpoint"` + TotalCalls int64 `json:"total_calls"` + Escalls int64 `json:"es_calls"` + RedisCalls int64 `json:"redis_calls"` + ESQPS float64 `json:"es_qps"` // ES QPS + RedisQPS float64 `json:"redis_qps"` // Redis QPS + AvgDuration int64 `json:"avg_duration_ms"` + SuccessRate float64 `json:"success_rate"` + LastUpdate string `json:"last_update"` +} + +// 全局监控器映射表：endpoint -> APIMonitor +var ( + apiMonitors = make(map[string]*APIMonitor) + apiMonitorsMu sync.RWMutex +) + +// GetOrCreateAPIMonitor 获取或创建 API 监控器 +func GetOrCreateAPIMonitor(endpoint string, qpsWindow time.Duration) *APIMonitor { + apiMonitorsMu.Lock() + defer apiMonitorsMu.Unlock() + + if monitor, exists := apiMonitors[endpoint]; exists { + return monitor + } + + monitor := &APIMonitor{ + endpoint: endpoint, + esCalls: make([]APICallRecord, 0), + redisCalls: make([]APICallRecord, 0), + //maxRecords: maxRecords, + nextID: 1, + qpsWindow: qpsWindow, + } + apiMonitors[endpoint] = monitor + return monitor +} + +// RecordESCall 记录 ES 调用 +func (am *APIMonitor) RecordESCall(operation, keyOrIndex, query string, duration time.Duration, err error) { + am.mutex.Lock() + defer am.mutex.Unlock() + + record := APICallRecord{ + ID: am.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: operation, + KeyOrIndex: keyOrIndex, + Query: query, + } + + if err != nil { + record.Error = err.Error() + } + + am.esCalls = append(am.esCalls, record) + //if len(am.esCalls) > am.maxRecords { + // am.esCalls = am.esCalls[1:] + //} +} + +// RecordRedisCall 记录 Redis 调用 +func (am *APIMonitor) RecordRedisCall(operation, keyOrIndex, query string, duration time.Duration, err error) { + am.mutex.Lock() + defer am.mutex.Unlock() + + record := APICallRecord{ + ID: am.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: operation, + KeyOrIndex: keyOrIndex, + Query: query, + } + + if err != nil { + record.Error = err.Error() + } + + am.redisCalls = append(am.redisCalls, record) + //if len(am.redisCalls) > am.maxRecords { + // am.redisCalls = am.redisCalls[1:] + //} +} + +// getNextID 获取下一个 ID +func (am *APIMonitor) getNextID() int64 { + id := am.nextID + am.nextID++ + return id +} + +// GetStats 获取统计数据 +func (am *APIMonitor) GetStats() *APIStats { + am.mutex.RLock() + defer am.mutex.RUnlock() + + now := time.Now() + windowStart := now.Add(-am.qpsWindow) + + // 统计 ES 调用 + var esRecentCalls, esSuccessCalls int64 + var esTotalDuration int64 + for _, call := range am.esCalls { + if call.Timestamp.After(windowStart) { + esRecentCalls++ + } + esTotalDuration += call.Duration + if call.Success { + esSuccessCalls++ + } + } + + // 统计 Redis 调用 + var redisRecentCalls, redisSuccessCalls int64 + var redisTotalDuration int64 + for _, call := range am.redisCalls { + if call.Timestamp.After(windowStart) { + redisRecentCalls++ + } + redisTotalDuration += call.Duration + if call.Success { + redisSuccessCalls++ + } + } + + totalCalls := int64(len(am.esCalls)) + int64(len(am.redisCalls)) + successCalls := esSuccessCalls + redisSuccessCalls + totalDuration := esTotalDuration + redisTotalDuration + + // 计算 QPS（每秒调用数） + windowSeconds := am.qpsWindow.Seconds() + esQPS := float64(esRecentCalls) / windowSeconds + redisQPS := float64(redisRecentCalls) / windowSeconds + + var avgDuration int64 + if totalCalls > 0 { + avgDuration = totalDuration / totalCalls + } + + var successRate float64 + if totalCalls > 0 { + successRate = float64(successCalls) / float64(totalCalls) * 100 + } + + return &APIStats{ + Endpoint: am.endpoint, + TotalCalls: totalCalls, + Escalls: int64(len(am.esCalls)), + RedisCalls: int64(len(am.redisCalls)), + ESQPS: esQPS, + RedisQPS: redisQPS, + AvgDuration: avgDuration, + SuccessRate: successRate, + LastUpdate: now.Format("2006-01-02 15:04:05"), + } +} + +// GetRecentESCalls 获取最近的 ES 调用记录（支持分页） +func (am *APIMonitor) GetRecentESCalls(page, pageSize int) ([]APICallRecord, int) { + am.mutex.RLock() + defer am.mutex.RUnlock() + + total := len(am.esCalls) + if total == 0 { + return []APICallRecord{}, 0 + } + + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 50 + } + if pageSize > 500 { + pageSize = 500 + } + + startIndex := (page - 1) * pageSize + endIndex := startIndex + pageSize + + if startIndex >= total { + return []APICallRecord{}, total + } + if endIndex > total { + endIndex = total + } + + result := make([]APICallRecord, endIndex-startIndex) + copy(result, am.esCalls[startIndex:endIndex]) + + for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 { + result[i], result[j] = result[j], result[i] + } + + return result, total +} + +// GetRecentRedisCalls 获取最近的 Redis 调用记录（支持分页） +func (am *APIMonitor) GetRecentRedisCalls(page, pageSize int) ([]APICallRecord, int) { + am.mutex.RLock() + defer am.mutex.RUnlock() + + total := len(am.redisCalls) + if total == 0 { + return []APICallRecord{}, 0 + } + + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 50 + } + if pageSize > 500 { + pageSize = 500 + } + + startIndex := (page - 1) * pageSize + endIndex := startIndex + pageSize + + if startIndex >= total { + return []APICallRecord{}, total + } + if endIndex > total { + endIndex = total + } + + result := make([]APICallRecord, endIndex-startIndex) + copy(result, am.redisCalls[startIndex:endIndex]) + + for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 { + result[i], result[j] = result[j], result[i] + } + + return result, total +} + +// GetRedisCallByID 根据 ID 获取 Redis 调用记录 +func (am *APIMonitor) GetRedisCallByID(callID int64) *APICallRecord { + am.mutex.RLock() + defer am.mutex.RUnlock() + + for i := len(am.redisCalls) - 1; i >= 0; i-- { + if am.redisCalls[i].ID == callID { + return &am.redisCalls[i] + } + } + return nil +} + +/*----------------------------------------ES-----------------------------------*/ +// MonitoredESClient 带监控的 ES 客户端 +type MonitoredESClient struct { + client *elasticsearch.Client + monitor *APIMonitor + endpoint string +} + +// NewMonitoredESClient 创建带监控的 ES 客户端 +func NewMonitoredESClient(client *elasticsearch.Client, endpoint string) *MonitoredESClient { + //monitor := GetOrCreateAPIMonitor(endpoint, 1000, 1*time.Minute) + monitor := GetOrCreateAPIMonitor(endpoint, 1*time.Minute) + return &MonitoredESClient{ + client: client, + monitor: monitor, + endpoint: endpoint, + } +} + +// Search 带监控的 ES 搜索（使用 esapi.SearchRequest） +func (m *MonitoredESClient) Search(ctx context.Context, req *esapi.SearchRequest) (*esapi.Response, time.Duration, error) { + startTime := time.Now() + + indexName := "" + if len(req.Index) > 0 { + indexName = strings.Join(req.Index, ",") + } + + queryBody := "" + if req.Body != nil { + buf := new(bytes.Buffer) + buf.ReadFrom(req.Body) + queryBody = buf.String() + // 重新设置 Body，因为 ReadFrom 已经读取了 + req.Body = io.NopCloser(buf) + } + + req.Pretty = true + resp, err := req.Do(ctx, m.client) + duration := time.Since(startTime) + + record := APICallRecord{ + ID: m.monitor.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: "search", + KeyOrIndex: indexName, + Query: queryBody, + } + + if err != nil { + record.Error = err.Error() + } + + if err != nil { + record.Error = err.Error() + } + + //m.monitor.RecordAPICall(record, "es") + m.monitor.RecordESCall("search", indexName, queryBody, duration, err) + + return resp, duration, err +} + +// RecordAPICall 通用记录方法 +func (am *APIMonitor) RecordAPICall(record APICallRecord, callType string) { + am.mutex.Lock() + defer am.mutex.Unlock() + + if callType == "es" { + am.esCalls = append(am.esCalls, record) + //if len(am.esCalls) > am.maxRecords { + // am.esCalls = am.esCalls[1:] + //} + } else if callType == "redis" { + am.redisCalls = append(am.redisCalls, record) + //if len(am.redisCalls) > am.maxRecords { + // am.redisCalls = am.redisCalls[1:] + //} + } +} + +// SearchWithQuery 带监控的 ES 搜索（使用查询字符串） +func (m *MonitoredESClient) SearchWithQuery(ctx context.Context, index, query string) (*esapi.Response, time.Duration, error) { + startTime := time.Now() + + req := esapi.SearchRequest{ + Index: []string{index}, + Body: strings.NewReader(query), + Pretty: true, + } + + req.Pretty = true + resp, err := req.Do(ctx, m.client) + duration := time.Since(startTime) + + m.monitor.RecordESCall("search", index, query, duration, err) + + return resp, duration, err +} + +// Get 带监控的 ES Get 文档 +func (m *MonitoredESClient) Get(ctx context.Context, index, docID string) (*esapi.Response, time.Duration, error) { + startTime := time.Now() + + req := esapi.GetRequest{ + Index: index, + DocumentID: docID, + } + + resp, err := req.Do(ctx, m.client) + duration := time.Since(startTime) + + m.monitor.RecordESCall("get", index, "docID: "+docID, duration, err) + + return resp, duration, err +} + +// Index 带监控的 ES Index 文档 +func (m *MonitoredESClient) Index(ctx context.Context, index, docID string, body io.Reader) (*esapi.Response, time.Duration, error) { + startTime := time.Now() + + req := esapi.IndexRequest{ + Index: index, + DocumentID: docID, + Body: body, + } + + resp, err := req.Do(ctx, m.client) + duration := time.Since(startTime) + + bodyStr := "" + if body != nil { + buf := new(bytes.Buffer) + buf.ReadFrom(body) + bodyStr = buf.String() + } + + m.monitor.RecordESCall("index", index, bodyStr, duration, err) + + return resp, duration, err +} + +// Delete 带监控的 ES Delete 文档 +func (m *MonitoredESClient) Delete(ctx context.Context, index, docID string) (*esapi.Response, time.Duration, error) { + startTime := time.Now() + + req := esapi.DeleteRequest{ + Index: index, + DocumentID: docID, + } + + resp, err := req.Do(ctx, m.client) + duration := time.Since(startTime) + + m.monitor.RecordESCall("delete", index, "docID: "+docID, duration, err) + + return resp, duration, err +} + +// Update 带监控的 ES Update 文档 +func (m *MonitoredESClient) Update(ctx context.Context, index, docID string, body io.Reader) (*esapi.Response, time.Duration, error) { + startTime := time.Now() + + req := esapi.UpdateRequest{ + Index: index, + DocumentID: docID, + Body: body, + } + + resp, err := req.Do(ctx, m.client) + duration := time.Since(startTime) + + bodyStr := "" + if body != nil { + buf := new(bytes.Buffer) + buf.ReadFrom(body) + bodyStr = buf.String() + } + + m.monitor.RecordESCall("update", index, bodyStr, duration, err) + + return resp, duration, err +} + +// Count 带监控的 ES Count +func (m *MonitoredESClient) Count(ctx context.Context, index string, query string) (*esapi.Response, time.Duration, error) { + startTime := time.Now() + + req := esapi.CountRequest{ + Index: []string{index}, + } + + if query != "" { + req.Body = strings.NewReader(query) + } + + resp, err := req.Do(ctx, m.client) + duration := time.Since(startTime) + + m.monitor.RecordESCall("count", index, query, duration, err) + + return resp, duration, err +} + +// GetESCallByID 根据 ID 获取 ES 调用记录 +func (am *APIMonitor) GetESCallByID(callID int64) *APICallRecord { + am.mutex.RLock() + defer am.mutex.RUnlock() + + for i := len(am.esCalls) - 1; i >= 0; i-- { + if am.esCalls[i].ID == callID { + return &am.esCalls[i] + } + } + return nil +} + +/*-----------------------------------REDIS-----------------------------------*/ +// MonitoredRedisClient 带监控的 Redis 客户端 +type MonitoredRedisClient struct { + client *redis.Client + monitor *APIMonitor + endpoint string +} + +// NewMonitoredRedisClient 创建带监控的 Redis 客户端 +func NewMonitoredRedisClient(client *redis.Client, endpoint string) *MonitoredRedisClient { + //monitor := GetOrCreateAPIMonitor(endpoint, 1000, 1*time.Minute) + monitor := GetOrCreateAPIMonitor(endpoint, 1*time.Minute) + return &MonitoredRedisClient{ + client: client, + monitor: monitor, + endpoint: endpoint, + } +} + +// Get 带监控的 Redis Get 操作 +func (m *MonitoredRedisClient) Get(ctx context.Context, key string) (string, time.Duration, error) { + startTime := time.Now() + val, err := m.client.Get(ctx, key).Result() + duration := time.Since(startTime) + + record := APICallRecord{ + ID: m.monitor.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: "GET", + KeyOrIndex: key, + Query: "", + Response: val, + } + + if err != nil { + record.Error = err.Error() + } + //m.monitor.RecordAPICall(record, "redis") + m.monitor.RecordRedisCall("GET", key, "", duration, err) + return val, duration, err +} + +// Set 带监控的 Redis Set 操作 +func (m *MonitoredRedisClient) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) (string, time.Duration, error) { + startTime := time.Now() + err := m.client.Set(ctx, key, value, expiration).Err() + duration := time.Since(startTime) + + query := "" + if str, ok := value.(string); ok { + query = str + } else if data, err := json.Marshal(value); err == nil { + query = string(data) + } + + record := APICallRecord{ + ID: m.monitor.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: "SET", + KeyOrIndex: key, + Query: query, + Response: "OK", + } + + if err != nil { + record.Error = err.Error() + } + + //m.monitor.RecordAPICall(record, "redis") + m.monitor.RecordRedisCall("SET", key, query, duration, err) + + return "OK", duration, err +} + +// ... existing code ... + +// HGet 带监控的 Redis HGet 操作 +func (m *MonitoredRedisClient) HGet(ctx context.Context, key, field string) (string, time.Duration, error) { + startTime := time.Now() + val, err := m.client.HGet(ctx, key, field).Result() + duration := time.Since(startTime) + + record := APICallRecord{ + ID: m.monitor.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: "HGET", + KeyOrIndex: key, + Query: "field: " + field, + Response: val, + } + + if err != nil { + record.Error = err.Error() + } + + m.monitor.RecordAPICall(record, "redis") + + return val, duration, err +} + +// ... existing code ... + +// HSet 带监控的 Redis HSet 操作 +func (m *MonitoredRedisClient) HSet(ctx context.Context, key, field string, value interface{}) (int64, time.Duration, error) { + startTime := time.Now() + result, err := m.client.HSet(ctx, key, field, value).Result() + duration := time.Since(startTime) + + queryValue := "" + if str, ok := value.(string); ok { + queryValue = str + } else if data, err := json.Marshal(value); err == nil { + queryValue = string(data) + } + + record := APICallRecord{ + ID: m.monitor.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: "HSET", + KeyOrIndex: key, + Query: fmt.Sprintf("field: %s, value: %v", field, queryValue), + Response: fmt.Sprintf("%d", result), + } + + if err != nil { + record.Error = err.Error() + } + + m.monitor.RecordAPICall(record, "redis") + + return result, duration, err +} + +// ... existing code ... + +// Exists 带监控的 Redis Exists 操作 +func (m *MonitoredRedisClient) Exists(ctx context.Context, keys ...string) (int64, time.Duration, error) { + startTime := time.Now() + result, err := m.client.Exists(ctx, keys...).Result() + duration := time.Since(startTime) + + record := APICallRecord{ + ID: m.monitor.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: "EXISTS", + KeyOrIndex: strings.Join(keys, ", "), + Query: "", + Response: fmt.Sprintf("%d", result), + } + + if err != nil { + record.Error = err.Error() + } + + m.monitor.RecordAPICall(record, "redis") + + return result, duration, err +} + +// ... existing code ... + +// Del 带监控的 Redis Del 操作 +func (m *MonitoredRedisClient) Del(ctx context.Context, keys ...string) (int64, time.Duration, error) { + startTime := time.Now() + result, err := m.client.Del(ctx, keys...).Result() + duration := time.Since(startTime) + + record := APICallRecord{ + ID: m.monitor.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: "DEL", + KeyOrIndex: strings.Join(keys, ", "), + Query: "", + Response: fmt.Sprintf("%d", result), + } + + if err != nil { + record.Error = err.Error() + } + + m.monitor.RecordAPICall(record, "redis") + + return result, duration, err +} + +// ... existing code ... + +// MGet 带监控的 Redis MGet 操作 +func (m *MonitoredRedisClient) MGet(ctx context.Context, keys ...string) ([]interface{}, time.Duration, error) { + startTime := time.Now() + result, err := m.client.MGet(ctx, keys...).Result() + duration := time.Since(startTime) + + resultJSON := "" + if data, marshalErr := json.Marshal(result); marshalErr == nil { + resultJSON = string(data) + } + + record := APICallRecord{ + ID: m.monitor.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: "MGET", + KeyOrIndex: strings.Join(keys, ", "), + Query: "", + Response: resultJSON, + } + + if err != nil { + record.Error = err.Error() + } + + m.monitor.RecordAPICall(record, "redis") + + return result, duration, err +} + +// ... existing code ... + +// Ping 带监控的 Redis Ping 操作 +func (m *MonitoredRedisClient) Ping(ctx context.Context) (string, time.Duration, error) { + startTime := time.Now() + val, err := m.client.Ping(ctx).Result() + duration := time.Since(startTime) + + record := APICallRecord{ + ID: m.monitor.getNextID(), + Timestamp: time.Now(), + Duration: duration.Milliseconds(), + Success: err == nil, + Operation: "PING", + KeyOrIndex: "", + Query: "PING", + Response: val, + } + + if err != nil { + record.Error = err.Error() + } + + m.monitor.RecordAPICall(record, "redis") + + return val, duration, err +} diff --git a/monitor/health_api.go b/monitor/health_api.go new file mode 100644 index 0000000..ab7af1c --- /dev/null +++ b/monitor/health_api.go @@ -0,0 +1,773 @@ +package monitor + +import ( + "net/http" + "sort" + "strconv" + "time" + + "github.com/gin-gonic/gin" +) + +// APIMonitorController API 监控控制器 +type APIMonitorController struct{} + +// NewAPIMonitorController 创建 API 监控控制器 +func NewAPIMonitorController() *APIMonitorController { + return &APIMonitorController{} +} + +// GetAPIStats 获取指定 API 的统计信息 +func (amc *APIMonitorController) GetAPIStats(c *gin.Context) { + endpoint := c.Query("endpoint") + if endpoint == "" { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "缺少 endpoint 参数", + }) + return + } + + apiMonitorsMu.RLock() + monitor, exists := apiMonitors[endpoint] + apiMonitorsMu.RUnlock() + + if !exists { + c.JSON(http.StatusNotFound, gin.H{ + "error": "该接口暂无监控数据", + }) + return + } + + stats := monitor.GetStats() + + c.JSON(http.StatusOK, gin.H{ + "status": "success", + "data": stats, + "timestamp": time.Now().Format("2006-01-02 15:04:05"), + }) +} + +// GetAllAPIStats 获取所有 API 的统计信息 +func (amc *APIMonitorController) GetAllAPIStats(c *gin.Context) { + apiMonitorsMu.RLock() + defer apiMonitorsMu.RUnlock() + + allStats := make([]*APIStats, 0, len(apiMonitors)) + endpoints := make([]string, 0, len(apiMonitors)) + + // 收集所有 endpoint + for endpoint := range apiMonitors { + endpoints = append(endpoints, endpoint) + } + + // 按 endpoint 字母顺序排序 + sort.Strings(endpoints) + + // 按排序后的顺序获取统计信息 + for _, endpoint := range endpoints { + allStats = append(allStats, apiMonitors[endpoint].GetStats()) + } + + c.JSON(http.StatusOK, gin.H{ + "status": "success", + "data": allStats, + "count": len(allStats), + "timestamp": time.Now().Format("2006-01-02 15:04:05"), + }) +} + +// GetAPICallDetail 获取指定 API 调用的详细信息 +func (amc *APIMonitorController) GetAPICallDetail(c *gin.Context) { + endpoint := c.Query("endpoint") + callIDStr := c.Query("call_id") + callType := c.Query("type") // es or redis + + if endpoint == "" || callIDStr == "" { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "缺少 endpoint 或 call_id 参数", + }) + return + } + + callID, err := strconv.ParseInt(callIDStr, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "无效的 call_id", + }) + return + } + + apiMonitorsMu.RLock() + monitor, exists := apiMonitors[endpoint] + apiMonitorsMu.RUnlock() + + if !exists { + c.JSON(http.StatusNotFound, gin.H{ + "error": "该接口暂无监控数据", + }) + return + } + + // 查找对应的调用记录 + var foundRecord *APICallRecord + var recordType string + + if callType == "es" { + foundRecord = monitor.GetESCallByID(callID) + if foundRecord != nil { + recordType = "ES" + } + } else if callType == "redis" { + foundRecord = monitor.GetRedisCallByID(callID) + if foundRecord != nil { + recordType = "Redis" + } + } else { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "type 参数必须是 es 或 redis", + }) + return + } + + if foundRecord == nil { + c.JSON(http.StatusNotFound, gin.H{ + "error": "未找到对应的调用记录", + }) + return + } + + c.JSON(http.StatusOK, gin.H{ + "status": "success", + "data": gin.H{ + "id": foundRecord.ID, + "type": recordType, + "timestamp": foundRecord.Timestamp.Format("2006-01-02 15:04:05"), + "duration_ms": foundRecord.Duration, + "success": foundRecord.Success, + "error": foundRecord.Error, + "operation": foundRecord.Operation, + "key_or_index": foundRecord.KeyOrIndex, + "query": foundRecord.Query, + "request": foundRecord.Request, + "response": foundRecord.Response, + }, + }) +} + +// GetAPIESCalls 获取指定 API 的 ES 调用记录 +func (amc *APIMonitorController) GetAPIESCalls(c *gin.Context) { + endpoint := c.Query("endpoint") + if endpoint == "" { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "缺少 endpoint 参数", + }) + return + } + + pageStr := c.DefaultQuery("page", "1") + pageSizeStr := c.DefaultQuery("page_size", "50") + + page, err := strconv.Atoi(pageStr) + if err != nil || page <= 0 { + page = 1 + } + + pageSize, err := strconv.Atoi(pageSizeStr) + if err != nil || pageSize <= 0 { + pageSize = 50 + } + if pageSize > 500 { + pageSize = 500 + } + + apiMonitorsMu.RLock() + monitor, exists := apiMonitors[endpoint] + apiMonitorsMu.RUnlock() + + if !exists { + c.JSON(http.StatusNotFound, gin.H{ + "error": "该接口暂无监控数据", + }) + return + } + + calls, total := monitor.GetRecentESCalls(page, pageSize) + totalPages := (total + pageSize - 1) / pageSize + + c.JSON(http.StatusOK, gin.H{ + "status": "success", + "data": gin.H{ + "calls": calls, + "count": len(calls), + "total": total, + "page": page, + "page_size": pageSize, + "total_pages": totalPages, + }, + "timestamp": time.Now().Format("2006-01-02 15:04:05"), + }) +} + +// GetAPIRedisCalls 获取指定 API 的 Redis 调用记录 +func (amc *APIMonitorController) GetAPIRedisCalls(c *gin.Context) { + endpoint := c.Query("endpoint") + if endpoint == "" { + c.JSON(http.StatusBadRequest, gin.H{ + "error": "缺少 endpoint 参数", + }) + return + } + + pageStr := c.DefaultQuery("page", "1") + pageSizeStr := c.DefaultQuery("page_size", "50") + + page, err := strconv.Atoi(pageStr) + if err != nil || page <= 0 { + page = 1 + } + + pageSize, err := strconv.Atoi(pageSizeStr) + if err != nil || pageSize <= 0 { + pageSize = 50 + } + if pageSize > 500 { + pageSize = 500 + } + + apiMonitorsMu.RLock() + monitor, exists := apiMonitors[endpoint] + apiMonitorsMu.RUnlock() + + if !exists { + c.JSON(http.StatusNotFound, gin.H{ + "error": "该接口暂无监控数据", + }) + return + } + + calls, total := monitor.GetRecentRedisCalls(page, pageSize) + totalPages := (total + pageSize - 1) / pageSize + + c.JSON(http.StatusOK, gin.H{ + "status": "success", + "data": gin.H{ + "calls": calls, + "count": len(calls), + "total": total, + "page": page, + "page_size": pageSize, + "total_pages": totalPages, + }, + "timestamp": time.Now().Format("2006-01-02 15:04:05"), + }) +} + +// GetAPIMonitorDashboard 获取 API 监控仪表板 +func (amc *APIMonitorController) GetAPIMonitorDashboard(c *gin.Context) { + dashboardHTML := ` + + + + + API 监控仪表板 + + + +
+

🔍 API 监控仪表板 - ES & Redis 调用

+ +
+ + +
+ +
+

所有接口概览

+
+
+ +
+

详细监控数据

+
+ + + +
+ +
+
+
+ +
+
+ +
+ +
+
+ +
+
+
+ + + + + + +` + + c.Header("Content-Type", "text/html; charset=utf-8") + c.String(http.StatusOK, dashboardHTML) +} diff --git a/service/book.go b/service/book.go index ba1acbe..91e2d16 100644 --- a/service/book.go +++ b/service/book.go @@ -262,46 +262,12 @@ func (svc *BookService) UpdateBookFieldsByISBN(request *request.BookUpdateReques if book == nil { return nil, fmt.Errorf("未找到该 ISBN 对应的图书") } + // 获取字段配置 + fieldConfig := es.GetESFieldConfig() // 构建更新脚本 var scriptParts []string params := make(map[string]interface{}) - allowedFields := map[string]bool{ - "book_name": true, - "book_pic": true, - "book_pic_s": true, - "book_pic_b": true, - "book_pic_w": true, - "author": true, - "category": true, - "publisher": true, - "publication_time": true, - "binding_layout": true, - "fix_price": true, - "content": true, - "is_suit": true, - "day_sale_7": true, - "day_sale_15": true, - "day_sale_30": true, - "day_sale_60": true, - "day_sale_90": true, - "day_sale_180": true, - "day_sale_365": true, - "this_year_sale": true, - "last_year_sale": true, - "total_sale": true, - "buy_counts": true, - "sell_counts": true, - "is_illegal": true, - "is_return": true, - "is_filter": true, - "update_time": true, - "page_count": true, - "word_count": true, - "book_format": true, - "cat_id": true, - } - // 判断 is_suit 是否已传递，如果没传则自动检测 if _, exists := request.Data["is_suit"]; !exists { // 未传递 is_suit，自动检测并设置 @@ -310,7 +276,9 @@ func (svc *BookService) UpdateBookFieldsByISBN(request *request.BookUpdateReques } for field, value := range request.Data { - if !allowedFields[field] { + // 使用配置检查字段是否允许更新 + if !fieldConfig.IsAllowUpdate(field) { + log.Printf("[UpdateBookFieldsByISBN] 字段 %s 不允许更新，已跳过", field) continue } scriptParts = append(scriptParts, @@ -320,7 +288,6 @@ func (svc *BookService) UpdateBookFieldsByISBN(request *request.BookUpdateReques if len(scriptParts) == 0 { return nil, fmt.Errorf("没有有效的更新字段") } - body := map[string]interface{}{ "script": map[string]interface{}{ "source": strings.Join(scriptParts, " "), @@ -588,9 +555,9 @@ func (svc *BookService) buildUpdateData(existing, new *es.ESBook) map[string]int {existing.PublicationTime == "" && new.PublicationTime != "", "publication_time", new.PublicationTime}, {existing.BookName.Value == "" && new.BookName.Value != "", "book_name", new.BookName.Value}, {existing.Author == "" && new.Author != "", "author", new.Author}, - {existing.BookPic.PddPath == "" && new.BookPic.PddPath != "", "book_pic", + {existing.BookPic.PddPath == "" && new.BookPic.PddPath != "" && strings.Contains(new.BookPic.PddPath, "http"), "book_pic", map[string]interface{}{"localPath": "", "pddPath": new.BookPic.PddPath}}, - {existing.BookPicS.PddResponse == "" && new.BookPicS.PddResponse != "", "book_pic_s", + {existing.BookPicS.PddResponse == "" && new.BookPicS.PddResponse != "" && strings.Contains(new.BookPicS.PddResponse, "http"), "book_pic_s", map[string]interface{}{"localPath": "", "pddResponse": new.BookPicS.PddResponse}}, } @@ -620,55 +587,66 @@ func (svc *BookService) addBookToES(ctx context.Context, req *es.ESBook) (*es.ES if req.PublicationTime != "" && req.PublicationTime != "0" { publicationTimeTimestamp = strconv.FormatInt(util.ParsePublicationTime(publicationTimeTimestamp), 10) } + // 准备数据映射（使用 Go 字段名） + dataMap := make(map[string]interface{}) // 构建 ES 文档 - doc := map[string]interface{}{ - "id": svc.generateNewID(), - "book_name": req.BookName.Value, - "book_pic": es.BookPicObj{LocalPath: "", PddPath: req.BookPic.PddPath}, - "book_pic_s": es.BookPicSObj{LocalPath: "", PddResponse: req.BookPicS.PddResponse}, - "book_pic_b": req.BookPicB, - "book_pic_w": make(map[string]interface{}), - "isbn": req.ISBN, - "author": req.Author, - "category": req.Category, - "publisher": req.Publisher, - "publication_time": publicationTimeTimestamp, - "binding_layout": req.BindingLayout, - "fix_price": req.FixPrice, - "content": req.Content, - "is_suit": map[bool]int{true: 1, false: 0}[es.CheckBookSuit(req.BookName.Value)], - "day_sale_7": salesInfo.DaySale7, - "day_sale_15": salesInfo.DaySale15, - "day_sale_30": salesInfo.DaySale30, - "day_sale_60": salesInfo.DaySale60, - "day_sale_90": salesInfo.DaySale90, - "day_sale_180": salesInfo.DaySale180, - "day_sale_365": salesInfo.DaySale365, - "this_year_sale": salesInfo.ThisYearSale, - "last_year_sale": salesInfo.LastYearSale, - "total_sale": salesInfo.TotalSale, - "buy_counts": req.BuyCounts, - "sell_counts": req.SellCounts, - "book_pic_obj": req.BookPicObj, - "book_pic_obj_s": req.BookPicObjS, - "is_illegal": 0, - "is_return": 0, - "is_filter": "000000", - "update_time": es.NumberOrString(fmt.Sprintf("%d", time.Now().Unix())), - "page_count": req.PageCount, - "word_count": req.WordCount, - "book_format": req.BookFormat, + dataMap["ID"] = svc.generateNewID() + dataMap["BookName"] = req.BookName.Value + dataMap["BookPic"] = es.BookPicObj{LocalPath: "", PddPath: req.BookPic.PddPath} + dataMap["BookPicS"] = es.BookPicSObj{LocalPath: "", PddResponse: req.BookPicS.PddResponse} + dataMap["BookPicB"] = req.BookPicB + dataMap["BookPicW"] = make(map[string]interface{}) + dataMap["ISBN"] = req.ISBN + dataMap["Author"] = req.Author + dataMap["Category"] = req.Category + dataMap["Publisher"] = req.Publisher + dataMap["PublicationTime"] = publicationTimeTimestamp + dataMap["BindingLayout"] = req.BindingLayout + dataMap["FixPrice"] = req.FixPrice + dataMap["Content"] = req.Content + dataMap["IsSuit"] = map[bool]int{true: 1, false: 0}[es.CheckBookSuit(req.BookName.Value)] + // 销量字段 + dataMap["DaySale7"] = salesInfo.DaySale7 + dataMap["DaySale15"] = salesInfo.DaySale15 + dataMap["DaySale30"] = salesInfo.DaySale30 + dataMap["DaySale60"] = salesInfo.DaySale60 + dataMap["DaySale90"] = salesInfo.DaySale90 + dataMap["DaySale180"] = salesInfo.DaySale180 + dataMap["DaySale365"] = salesInfo.DaySale365 + dataMap["ThisYearSale"] = salesInfo.ThisYearSale + dataMap["LastYearSale"] = salesInfo.LastYearSale + dataMap["TotalSale"] = salesInfo.TotalSale + dataMap["BuyCounts"] = req.BuyCounts + dataMap["SellCounts"] = req.SellCounts + dataMap["BookPicObj"] = req.BookPicObj + dataMap["BookPicObjS"] = req.BookPicObjS + dataMap["IsIllegal"] = 0 + dataMap["IsReturn"] = 0 + dataMap["IsFilter"] = "000000" + + // 时间字段 + dataMap["UpdateTime"] = es.NumberOrString(fmt.Sprintf("%d", time.Now().Unix())) + + // 其他字段 + dataMap["PageCount"] = req.PageCount + dataMap["WordCount"] = req.WordCount + dataMap["BookFormat"] = req.BookFormat + if req.Other != nil { + dataMap["Other"] = req.Other } - // 写入 ES + fmt.Println("dataMap:", dataMap) + + // 使用配置构建 ES 文档（自动转换字段名） + doc := es.GetESFieldConfig().BuildESDocument(dataMap) + //// 写入 ES if err := svc.indexDocumentToES(ctx, doc, req.ISBN); err != nil { return nil, err } // 同步 Redis _ = svc.SyncRedisByISBN(req.ISBN, "update") - - // 构建返回对象 - return &es.ESBook{ + // 建返回对象 + returnBook := &es.ESBook{ ID: doc["id"].(int64), BookName: es.FlexibleString{Value: req.BookName.Value}, BookPic: doc["book_pic"].(es.BookPicObj), @@ -705,7 +683,9 @@ func (svc *BookService) addBookToES(ctx context.Context, req *es.ESBook) (*es.ES PageCount: req.PageCount, WordCount: req.WordCount, BookFormat: req.BookFormat, - }, nil + Other: req.Other, + } + return returnBook, nil } // SyncRedisByISBN 同步到Redis