DaShangDao/daShangDao_centerBook
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

api监控，es兼容薪字段

Browse Source
This commit is contained in:
unknown 2026-03-17 18:01:50 +08:00
parent 551a234de1
commit ea838441c1
7 changed files with 2320 additions and 372 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
es/es_config.go Normal file
View File

@ -0,0 +1,170 @@
package es
// ESFieldConfig ES 字段配置
type ESFieldConfig struct {
// AllowUpdate 允许更新的字段列表
AllowUpdate map[string]bool
// AllowAdd 允许新增时填充的字段列表
AllowAdd map[string]bool
// FieldMappings 字段映射（Go 结构体字段名 -> ES 字段名）
FieldMappings map[string]string
}
// GetESFieldConfig 获取 ES 字段配置
func GetESFieldConfig() *ESFieldConfig {
return &ESFieldConfig{
AllowUpdate: map[string]bool{
"book_pic": true,
"book_pic_s": true,
"book_pic_b": true,
"book_pic_w": true,
"book_def_pic": true,
"isbn": true,
"author": true,
"category": true,
"publisher": true,
"publication_time": true,
"binding_layout": true,
"fix_price": true,
"content": true,
"is_suit": true,
"day_sale_7": true,
"day_sale_15": true,
"day_sale_30": true,
"day_sale_60": true,
"day_sale_90": true,
"day_sale_180": true,
"day_sale_365": true,
"this_year_sale": true,
"last_year_sale": true,
"total_sale": true,
"buy_counts": true,
"sell_counts": true,
"is_illegal": true,
"is_return": true,
"is_filter": true,
"update_time": true,
"page_count": true,
"word_count": true,
"book_format": true,
"cat_id": true,
"other": true, // 允许更新 other 字段
},
AllowAdd: map[string]bool{
"id": true,
"book_name": true,
"book_pic": true,
"book_pic_s": true,
"book_pic_b": true,
"book_pic_w": true,
"isbn": true,
"author": true,
"category": true,
"publisher": true,
"publication_time": true,
"binding_layout": true,
"fix_price": true,
"content": true,
"is_suit": true,
"day_sale_7": true,
"day_sale_15": true,
"day_sale_30": true,
"day_sale_60": true,
"day_sale_90": true,
"day_sale_180": true,
"day_sale_365": true,
"this_year_sale": true,
"last_year_sale": true,
"total_sale": true,
"buy_counts": true,
"sell_counts": true,
"book_pic_obj": true,
"book_pic_obj_s": true,
"update_time": true,
"is_illegal": true,
"is_return": true,
"is_filter": true,
"page_count": true,
"word_count": true,
"book_format": true,
"other": true, // 允许新增 other 字段
},
FieldMappings: map[string]string{
"ID": "id",
"BookName": "book_name",
"BookPic": "book_pic",
"BookPicS": "book_pic_s",
"BookPicB": "book_pic_b",
"BookPicW": "book_pic_w",
"ISBN": "isbn",
"Author": "author",
"Category": "category",
"Publisher": "publisher",
"PublicationTime": "publication_time",
"BindingLayout": "binding_layout",
"FixPrice": "fix_price",
"Content": "content",
"IsSuit": "is_suit",
"DaySale7": "day_sale_7",
"DaySale15": "day_sale_15",
"DaySale30": "day_sale_30",
"DaySale60": "day_sale_60",
"DaySale90": "day_sale_90",
"DaySale180": "day_sale_180",
"DaySale365": "day_sale_365",
"ThisYearSale": "this_year_sale",
"LastYearSale": "last_year_sale",
"TotalSale": "total_sale",
"BuyCounts": "buy_counts",
"SellCounts": "sell_counts",
"BookPicObj": "book_pic_obj",
"BookPicObjS": "book_pic_obj_s",
"UpdateTime": "update_time",
"IsIllegal": "is_illegal",
"IsReturn": "is_return",
"IsFilter": "is_filter",
"PageCount": "page_count",
"WordCount": "word_count",
"BookFormat": "book_format",
"Other": "other",
},
}
}
// IsAllowUpdate 检查字段是否允许更新
func (cfg *ESFieldConfig) IsAllowUpdate(field string) bool {
return cfg.AllowUpdate[field]
}
// IsAllowAdd 检查字段是否允许新增
func (cfg *ESFieldConfig) IsAllowAdd(field string) bool {
return cfg.AllowAdd[field]
}
// GetESFieldName 获取 ES 字段名
func (cfg *ESFieldConfig) GetESFieldName(goFieldName string) string {
if esName, ok := cfg.FieldMappings[goFieldName]; ok {
return esName
}
return goFieldName
}
// BuildESDocument 构建 ES 文档
func (cfg *ESFieldConfig) BuildESDocument(data map[string]interface{}) map[string]interface{} {
doc := make(map[string]interface{})
for goField, value := range data {
// 获取 ES 字段名
esField := cfg.GetESFieldName(goField)
// 检查是否允许添加
if !cfg.IsAllowAdd(esField) {
continue
}
doc[esField] = value
}
return doc
}

336
es/es_search.go
View File

@ -6,11 +6,14 @@ import (
"centerBook/image" "centerBook/image"
"centerBook/kongfz" "centerBook/kongfz"
"centerBook/model/request" "centerBook/model/request"
"centerBook/monitor"
"centerBook/tail" "centerBook/tail"
"centerBook/util/redisClient" "centerBook/util/redisClient"
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"github.com/gin-gonic/gin"
jsoniter "github.com/json-iterator/go"
"io" "io"
"log" "log"
"net/http" "net/http"
@ -18,9 +21,6 @@ import (
"strings" "strings"
"time" "time"
"github.com/gin-gonic/gin"
jsoniter "github.com/json-iterator/go"
"github.com/elastic/go-elasticsearch/v8/esapi" "github.com/elastic/go-elasticsearch/v8/esapi"
) )
@ -62,6 +62,11 @@ type ESBookResponse struct {
IsIllegal int `json:"is_illegal"` // 是否非法 示例 000000 IsIllegal int `json:"is_illegal"` // 是否非法 示例 000000
IsReturn int `json:"is_return"` // 是否为驳回 示例 0 否 1 是 IsReturn int `json:"is_return"` // 是否为驳回 示例 0 否 1 是
IsFilter string `json:"is_filter"` // 过滤字段 IsFilter string `json:"is_filter"` // 过滤字段
PageCount NumberOrString `json:"page_count"` // 页数
WordCount NumberOrString `json:"word_count"` // 字数
BookFormat NumberOrString `json:"book_format"` // 多少开
//CatId request.CatIdObject `json:"cat_id"` // 类目
Other map[string]interface{} `json:"other"` // 扩展字段，用于兼容未来新增字段
} }
// FlexibleString 处理可能是字符串或数组的字段 // FlexibleString 处理可能是字符串或数组的字段
@ -188,6 +193,11 @@ func (book *ESBook) ConvertToResponse() ESBookResponse {
IsIllegal: book.IsIllegal, IsIllegal: book.IsIllegal,
IsReturn: book.IsReturn, IsReturn: book.IsReturn,
IsFilter: book.IsFilter, IsFilter: book.IsFilter,
PageCount: book.PageCount,
WordCount: book.WordCount,
BookFormat: book.BookFormat,
//CatId: book.CatId,
Other: book.Other,
} }
} }
@ -250,6 +260,7 @@ type ESBook struct {
WordCount NumberOrString `json:"word_count"` // 字数 WordCount NumberOrString `json:"word_count"` // 字数
BookFormat NumberOrString `json:"book_format"` // 多少开 BookFormat NumberOrString `json:"book_format"` // 多少开
CatId request.CatIdObject `json:"cat_id"` // 类目 CatId request.CatIdObject `json:"cat_id"` // 类目
Other map[string]interface{} `json:"other"` // 扩展字段，用于兼容未来新增字段
} }
// AddBookRequest 用于 Service 方法的入参 // AddBookRequest 用于 Service 方法的入参
@ -459,7 +470,7 @@ type esHitsWrapper struct {
} }
// SearchBooks 搜索图书 // SearchBooks 搜索图书
func (svc *ESSearchService) SearchBooks(keyword string) ([]ESBook, error) { func (svc *ESSearchService) SearchBooks(keyword string, endpoint ...string) ([]ESBook, error) {
keyword = strings.TrimSpace(keyword) keyword = strings.TrimSpace(keyword)
if keyword == "" { if keyword == "" {
@ -494,7 +505,18 @@ func (svc *ESSearchService) SearchBooks(keyword string) ([]ESBook, error) {
TrackTotalHits: true, TrackTotalHits: true,
} }
res, err := req.Do(context.Background(), svc.ES.Client.Transport) // 如果有传入 endpoint，使用监控
var res *esapi.Response
var duration time.Duration
if len(endpoint) > 0 && endpoint[0] != "" {
monitoredES := monitor.NewMonitoredESClient(svc.ES.Client, endpoint[0])
res, duration, err = monitoredES.Search(context.Background(), &req)
log.Printf("[SearchBooks] ES 查询耗时：%dms", duration.Milliseconds())
} else {
// 原有逻辑，不带监控
res, err = req.Do(context.Background(), svc.ES.Client.Transport)
}
if err != nil { if err != nil {
return nil, fmt.Errorf("执行 ES 查询失败: %v", err) return nil, fmt.Errorf("执行 ES 查询失败: %v", err)
} }
@ -1918,20 +1940,41 @@ func (svc *ESSearchService) BatchGetBookBaseInfoES(c *gin.Context) ([]ESBook, in
fmt.Printf("[DEBUG] ES Query Body:\n%s\n", string(body)) fmt.Printf("[DEBUG] ES Query Body:\n%s\n", string(body))
// ========== 执行 ES 查询 ========== // ========== 执行 ES 查询 ==========
res, err := svc.ES.Client.Search( //res, err := svc.ES.Client.Search(
svc.ES.Client.Search.WithIndex(ESIndex), // svc.ES.Client.Search.WithIndex(ESIndex),
svc.ES.Client.Search.WithBody(bytes.NewReader(body)), // svc.ES.Client.Search.WithBody(bytes.NewReader(body)),
svc.ES.Client.Search.WithTrackTotalHits(true), // svc.ES.Client.Search.WithTrackTotalHits(true),
) //)
//
//fmt.Printf("[DEBUG] ES Query Response:\n%s\n", res)
//
//if err != nil {
// fmt.Printf("[ERROR] ES.Client.Search error: %v\n", err)
// return nil, 0, err
//}
//defer res.Body.Close()
endpoint := c.FullPath()
monitoredES := monitor.NewMonitoredESClient(svc.ES.Client, endpoint)
queryBody := string(body)
req := esapi.SearchRequest{
Index: []string{ESIndex},
Body: bytes.NewReader([]byte(queryBody)),
TrackTotalHits: true,
Pretty: true,
}
res, duration, err := monitoredES.Search(context.Background(), &req)
fmt.Printf("[DEBUG] ES Query Response:\n%s\n", res) fmt.Printf("[DEBUG] ES Query Response:\n%s\n", res)
fmt.Printf("[DEBUG] ES 查询耗时：%dms\n", duration.Milliseconds())
if err != nil { if err != nil {
fmt.Printf("[ERROR] ES.Client.Search error: %v\n", err) fmt.Printf("[ERROR] ES.Client.Search error: %v\n", err)
return nil, 0, err return nil, 0, err
} }
defer res.Body.Close() defer res.Body.Close()
// 读取响应 // 读取响应
var buf bytes.Buffer var buf bytes.Buffer
// 使用CopyBuffer可以重用缓冲区 // 使用CopyBuffer可以重用缓冲区
@ -2044,25 +2087,27 @@ func (svc *ESSearchService) BatchGetBookBaseInfoESHandler(c *gin.Context) {
"total": total, "total": total,
}) })
} }
func (svc *ESSearchService) SearchBooksHandler(c *gin.Context) { func (svc *ESSearchService) SearchBooksHandler(c *gin.Context) {
isbn := c.Query("isbn") isbn := c.Query("isbn")
if isbn == "" { if isbn == "" {
log.Printf("[SearchBooksHandler] 缺少 isbn 参数")
c.JSON(400, gin.H{"error": "缺少 isbn 参数"}) c.JSON(400, gin.H{"error": "缺少 isbn 参数"})
return return
} }
log.Printf("[SearchBooksHandler] ISBN 模糊搜索：%s", isbn)
ctx := context.Background() ctx := context.Background()
endpoint := c.FullPath()
db4Client, err := redisClient.GetClientByName("db1") // 获取监控的 Redis 客户端
db1Client, err := redisClient.GetClientByName("db1")
if err == nil { if err == nil {
val, err := db4Client.Get(ctx, isbn).Result() monitoredRedis := monitor.NewMonitoredRedisClient(db1Client, endpoint)
val, _, err := monitoredRedis.Get(ctx, isbn)
if err == nil && val != "" { if err == nil && val != "" {
log.Printf("[SearchBooksHandler] 从 Redis db1 查询到数据: %s", isbn) log.Printf("[SearchBooksHandler] 从 Redis db1 查询到数据：%s", isbn)
// 使用 RedisBookInfo 结构体解析
var redisBook request.BookInfo var redisBook request.BookInfo
if err := json.Unmarshal([]byte(val), &redisBook); err == nil { if err := json.Unmarshal([]byte(val), &redisBook); err == nil {
// 转换为 ESBook
esBook := ConvertRedisBookToESBook(&redisBook) esBook := ConvertRedisBookToESBook(&redisBook)
if esBook != nil { if esBook != nil {
responseData := esBook.ConvertToResponse() responseData := esBook.ConvertToResponse()
@ -2071,13 +2116,12 @@ func (svc *ESSearchService) SearchBooksHandler(c *gin.Context) {
}) })
return return
} }
} else {
log.Printf("[SearchBookByISBNHandler] Redis 数据解析失败：%v", err)
} }
} }
} }
result, err := svc.SearchBooks(isbn) // ES 查询
result, err := svc.SearchBooks(isbn, endpoint)
if err != nil { if err != nil {
c.JSON(500, gin.H{"error": "ES 查询失败", "details": err.Error()}) c.JSON(500, gin.H{"error": "ES 查询失败", "details": err.Error()})
return return
@ -2105,20 +2149,17 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) {
log.Printf("[SearchBookByISBNHandler] 查询 ISBN: %s", isbn) log.Printf("[SearchBookByISBNHandler] 查询 ISBN: %s", isbn)
ctx := context.Background() ctx := context.Background()
endpoint := c.FullPath()
db4Client, err := redisClient.GetClientByName("db1") // Redis 查询（使用监控）
fmt.Println(db4Client) db1Client, err := redisClient.GetClientByName("db1")
if err == nil {
if err != nil { monitoredRedis := monitor.NewMonitoredRedisClient(db1Client, endpoint)
log.Printf("[SearchBookByISBNHandler] 获取 Redis db1 客户端失败: %v", err) val, _, err := monitoredRedis.Get(ctx, isbn)
} else {
val, err := db4Client.Get(ctx, isbn).Result()
if err == nil && val != "" { if err == nil && val != "" {
log.Printf("[SearchBookByISBNHandler] 从 Redis db1 查询到数据: %s", isbn) log.Printf("[SearchBookByISBNHandler] 从 Redis db1 查询到数据：%s", isbn)
// 使用 RedisBookInfo 结构体解析
var redisBook request.BookInfo var redisBook request.BookInfo
if err := json.Unmarshal([]byte(val), &redisBook); err == nil { if err := json.Unmarshal([]byte(val), &redisBook); err == nil {
// 转换为 ESBook
esBook := ConvertRedisBookToESBook(&redisBook) esBook := ConvertRedisBookToESBook(&redisBook)
if esBook != nil { if esBook != nil {
responseData := esBook.ConvertToResponse() responseData := esBook.ConvertToResponse()
@ -2130,24 +2171,70 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) {
} else { } else {
log.Printf("[SearchBookByISBNHandler] Redis 数据解析失败：%v", err) log.Printf("[SearchBookByISBNHandler] Redis 数据解析失败：%v", err)
} }
} else {
log.Printf("[SearchBookByISBNHandler] Redis db1 中未找到 ISBN: %s", isbn)
} }
} }
result, err := svc.SearchBookByISBN(isbn) // ES 查询（使用监控）
query := map[string]interface{}{
"query": map[string]interface{}{
"term": map[string]interface{}{
"isbn": isbn,
},
},
"_source": true,
}
body, err := json.Marshal(query)
if err != nil { if err != nil {
log.Printf("[SearchBookByISBNHandler] ES 查询失败: %v", err) log.Printf("[SearchBookByISBNHandler] 构建查询 JSON 失败：%v", err)
c.JSON(500, gin.H{"error": err.Error()}) c.JSON(500, gin.H{"error": "构建查询失败"})
return return
} }
req := esapi.SearchRequest{
Index: []string{ESIndex},
Body: bytes.NewReader(body),
TrackTotalHits: true,
Pretty: true,
}
// 创建监控客户端并执行查询
monitoredES := monitor.NewMonitoredESClient(svc.ES.Client, endpoint)
resp, duration, err := monitoredES.Search(ctx, &req)
log.Printf("[SearchBookByISBNHandler] ES 查询耗时：%dms", duration.Milliseconds())
if err != nil {
log.Printf("[SearchBookByISBNHandler] ES 查询失败：%v", err)
c.JSON(500, gin.H{"error": "ES 查询失败：" + err.Error()})
return
}
defer resp.Body.Close()
if resp.IsError() {
log.Printf("[SearchBookByISBNHandler] ES 返回错误：%s", resp.String())
c.JSON(500, gin.H{"error": "ES 返回错误：" + resp.String()})
return
}
var parsed esHitsWrapper
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
log.Printf("[SearchBookByISBNHandler] 解析 ES 响应失败：%v", err)
c.JSON(500, gin.H{"error": "解析响应失败"})
return
}
var result *ESBook
if len(parsed.Hits.Hits) > 0 {
result = &parsed.Hits.Hits[0].Source
}
if result == nil { if result == nil {
log.Printf("[SearchBookByISBNHandler] ES 中未找到 ISBN: %s，从孔夫子抓取", isbn) log.Printf("[SearchBookByISBNHandler] ES 中未找到 ISBN: %s，从孔夫子抓取", isbn)
apiBook, err := kongfz.GetBookImageByISBN(isbn, "CALF_ELEPHANT_PROXY", "1297757178467602432", "QgQBvP7f") apiBook, err := kongfz.GetBookImageByISBN(isbn, "CALF_ELEPHANT_PROXY", "1297757178467602432", "QgQBvP7f")
if err != nil { if err != nil {
log.Printf("[SearchBookByISBNHandler] 孔夫子 API 查询失败: %v", err) log.Printf("[SearchBookByISBNHandler] 孔夫子 API 查询失败：%v", err)
c.JSON(500, gin.H{"error": err.Error()}) c.JSON(500, gin.H{"error": err.Error()})
return return
} }
@ -2157,16 +2244,16 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) {
return return
} }
log.Printf("[SearchBookByISBNHandler] 获取到图书信息: %+v", apiBook.Data) log.Printf("[SearchBookByISBNHandler] 获取到图书信息：%+v", apiBook.Data)
pddBookPicURL := "" pddBookPicURL := ""
if apiBook.Data.BookPic != "" { if apiBook.Data.BookPic != "" {
url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPic, isbn, "true", apiBook.Data.BookName, "true") url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPic, isbn, "true", apiBook.Data.BookName, "true")
if err != nil { if err != nil {
log.Printf("[SearchBookByISBNHandler] 上传 book_pic 失败: %v", err) log.Printf("[SearchBookByISBNHandler] 上传 book_pic 失败：%v", err)
} else { } else {
pddBookPicURL = url pddBookPicURL = url
log.Printf("[SearchBookByISBNHandler] 上传 book_pic 成功: %s", url) log.Printf("[SearchBookByISBNHandler] 上传 book_pic 成功：%s", url)
} }
} }
@ -2174,10 +2261,10 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) {
if apiBook.Data.BookPicS != "" { if apiBook.Data.BookPicS != "" {
url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPicS, isbn, "true", apiBook.Data.BookName, "true") url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPicS, isbn, "true", apiBook.Data.BookName, "true")
if err != nil { if err != nil {
log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 失败: %v", err) log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 失败：%v", err)
} else { } else {
pddBookPicSURL = url pddBookPicSURL = url
log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 成功: %s", url) log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 成功：%s", url)
} }
} }
@ -2185,18 +2272,15 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) {
esBook.BookPicS.PddResponse = pddBookPicSURL esBook.BookPicS.PddResponse = pddBookPicSURL
esBook.BookPic.PddPath = pddBookPicURL esBook.BookPic.PddPath = pddBookPicURL
//log.Printf("[SearchBookByISBNHandler] 写入 ES: %+v", esBook)
result, err = svc.AddBookToES(c.Request.Context(), esBook) result, err = svc.AddBookToES(c.Request.Context(), esBook)
if err != nil { if err != nil {
log.Printf("[SearchBookByISBNHandler] 写入 ES 失败: %v", err) log.Printf("[SearchBookByISBNHandler] 写入 ES 失败：%v", err)
c.JSON(500, gin.H{"error": err.Error()}) c.JSON(500, gin.H{"error": err.Error()})
return return
} }
log.Printf("[SearchBookByISBNHandler] 写入 ES 成功, ISBN: %s", isbn) log.Printf("[SearchBookByISBNHandler] 写入 ES 成功，ISBN: %s", isbn)
} else {
//log.Printf("[SearchBookByISBNHandler] 从 ES 查询到图书: %+v", result)
} }
responseData := result.ConvertToResponse() responseData := result.ConvertToResponse()
@ -2205,6 +2289,166 @@ func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) {
}) })
} }
// func (svc *ESSearchService) SearchBooksHandler(c *gin.Context) {
// isbn := c.Query("isbn")
// if isbn == "" {
// c.JSON(400, gin.H{"error": "缺少 isbn 参数"})
// return
// }
//
// ctx := context.Background()
//
// db4Client, err := redisClient.GetClientByName("db1")
// if err == nil {
// val, err := db4Client.Get(ctx, isbn).Result()
// if err == nil && val != "" {
// log.Printf("[SearchBooksHandler] 从 Redis db1 查询到数据: %s", isbn)
// // 使用 RedisBookInfo 结构体解析
// var redisBook request.BookInfo
// if err := json.Unmarshal([]byte(val), &redisBook); err == nil {
// // 转换为 ESBook
// esBook := ConvertRedisBookToESBook(&redisBook)
// if esBook != nil {
// responseData := esBook.ConvertToResponse()
// c.JSON(200, gin.H{
// "data": responseData,
// })
// return
// }
// } else {
// log.Printf("[SearchBookByISBNHandler] Redis 数据解析失败：%v", err)
// }
// }
// }
//
// result, err := svc.SearchBooks(isbn)
// if err != nil {
// c.JSON(500, gin.H{"error": "ES 查询失败", "details": err.Error()})
// return
// }
//
// responseList := make([]ESBookResponse, 0, len(result))
// for _, book := range result {
// responseList = append(responseList, book.ConvertToResponse())
// }
//
// c.JSON(200, gin.H{
// "count": len(result),
// "data": responseList,
// })
// }
//func (svc *ESSearchService) SearchBookByISBNHandler(c *gin.Context) {
//
// isbn := c.Query("isbn")
// if isbn == "" {
// log.Printf("[SearchBookByISBNHandler] 缺少 isbn 参数")
// c.JSON(400, gin.H{"error": "缺少 isbn 参数"})
// return
// }
//
// log.Printf("[SearchBookByISBNHandler] 查询 ISBN: %s", isbn)
//
// ctx := context.Background()
//
// db4Client, err := redisClient.GetClientByName("db1")
// fmt.Println(db4Client)
//
// if err != nil {
// log.Printf("[SearchBookByISBNHandler] 获取 Redis db1 客户端失败: %v", err)
// } else {
// val, err := db4Client.Get(ctx, isbn).Result()
// if err == nil && val != "" {
// log.Printf("[SearchBookByISBNHandler] 从 Redis db1 查询到数据: %s", isbn)
// // 使用 RedisBookInfo 结构体解析
// var redisBook request.BookInfo
// if err := json.Unmarshal([]byte(val), &redisBook); err == nil {
// // 转换为 ESBook
// esBook := ConvertRedisBookToESBook(&redisBook)
// if esBook != nil {
// responseData := esBook.ConvertToResponse()
// c.JSON(200, gin.H{
// "data": responseData,
// })
// return
// }
// } else {
// log.Printf("[SearchBookByISBNHandler] Redis 数据解析失败：%v", err)
// }
// } else {
// log.Printf("[SearchBookByISBNHandler] Redis db1 中未找到 ISBN: %s", isbn)
// }
// }
//
// result, err := svc.SearchBookByISBN(isbn)
// if err != nil {
// log.Printf("[SearchBookByISBNHandler] ES 查询失败: %v", err)
// c.JSON(500, gin.H{"error": err.Error()})
// return
// }
//
// if result == nil {
// log.Printf("[SearchBookByISBNHandler] ES 中未找到 ISBN: %s，从孔夫子抓取", isbn)
//
// apiBook, err := kongfz.GetBookImageByISBN(isbn, "CALF_ELEPHANT_PROXY", "1297757178467602432", "QgQBvP7f")
// if err != nil {
// log.Printf("[SearchBookByISBNHandler] 孔夫子 API 查询失败: %v", err)
// c.JSON(500, gin.H{"error": err.Error()})
// return
// }
// if apiBook == nil || apiBook.Data.ISBN == "" {
// log.Printf("[SearchBookByISBNHandler] 孔夫子 API 未找到图书信息 ISBN: %s", isbn)
// c.JSON(404, gin.H{"error": "未找到图书信息"})
// return
// }
//
// log.Printf("[SearchBookByISBNHandler] 获取到图书信息: %+v", apiBook.Data)
//
// pddBookPicURL := ""
// if apiBook.Data.BookPic != "" {
// url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPic, isbn, "true", apiBook.Data.BookName, "true")
// if err != nil {
// log.Printf("[SearchBookByISBNHandler] 上传 book_pic 失败: %v", err)
// } else {
// pddBookPicURL = url
// log.Printf("[SearchBookByISBNHandler] 上传 book_pic 成功: %s", url)
// }
// }
//
// pddBookPicSURL := ""
// if apiBook.Data.BookPicS != "" {
// url, err := image.DownloadAndUploadBookImage(apiBook.Data.BookPicS, isbn, "true", apiBook.Data.BookName, "true")
// if err != nil {
// log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 失败: %v", err)
// } else {
// pddBookPicSURL = url
// log.Printf("[SearchBookByISBNHandler] 上传 book_pic_s 成功: %s", url)
// }
// }
//
// esBook := ConvertKongfzToESBook(apiBook)
//
// esBook.BookPicS.PddResponse = pddBookPicSURL
// esBook.BookPic.PddPath = pddBookPicURL
// //log.Printf("[SearchBookByISBNHandler] 写入 ES: %+v", esBook)
//
// result, err = svc.AddBookToES(c.Request.Context(), esBook)
// if err != nil {
// log.Printf("[SearchBookByISBNHandler] 写入 ES 失败: %v", err)
// c.JSON(500, gin.H{"error": err.Error()})
// return
// }
//
// log.Printf("[SearchBookByISBNHandler] 写入 ES 成功, ISBN: %s", isbn)
// } else {
// //log.Printf("[SearchBookByISBNHandler] 从 ES 查询到图书: %+v", result)
// }
//
// responseData := result.ConvertToResponse()
// c.JSON(200, gin.H{
// "data": responseData,
// })
//}
// ConvertKongfzToESBook 将第三方接口返回的数据转换为 ESBook 结构 // ConvertKongfzToESBook 将第三方接口返回的数据转换为 ESBook 结构
func ConvertKongfzToESBook(apiBook *kongfz.BookResponse) *ESBook { func ConvertKongfzToESBook(apiBook *kongfz.BookResponse) *ESBook {
if apiBook == nil || apiBook.Data.ISBN == "" { if apiBook == nil || apiBook.Data.ISBN == "" {

3
health_api.go
View File

@ -1,11 +1,10 @@
package main package main
import ( import (
"github.com/gin-gonic/gin"
"net/http" "net/http"
"strconv" "strconv"
"time" "time"
"github.com/gin-gonic/gin"
) )
// SQLHealthController SQL健康监控控制器 // SQLHealthController SQL健康监控控制器

34
main.go
View File

@ -29,6 +29,7 @@ import (
"github.com/go-redis/redis/v8" "github.com/go-redis/redis/v8"
"centerBook/es" "centerBook/es"
"centerBook/monitor"
"centerBook/util/redisClient" "centerBook/util/redisClient"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
@ -284,20 +285,27 @@ func main() {
//r.GET("/ready", bookCenter.ReadyCheck) //r.GET("/ready", bookCenter.ReadyCheck)
// 7. SQL健康监控端点 // 7. SQL健康监控端点
sqlHealthController := NewSQLHealthController() //sqlHealthController := NewSQLHealthController()
r.GET("/api/sql-health/stats", sqlHealthController.GetSQLStats) //r.GET("/api/sql-health/stats", sqlHealthController.GetSQLStats)
r.GET("/api/sql-health/recent", sqlHealthController.GetRecentSQLRecords) //r.GET("/api/sql-health/recent", sqlHealthController.GetRecentSQLRecords)
r.GET("/api/sql-health/slow-queries", sqlHealthController.GetSlowQueries) //r.GET("/api/sql-health/slow-queries", sqlHealthController.GetSlowQueries)
r.GET("/api/sql-health/failed-queries", sqlHealthController.GetFailedQueries) //r.GET("/api/sql-health/failed-queries", sqlHealthController.GetFailedQueries)
r.POST("/api/sql-health/clear", sqlHealthController.ClearSQLRecords) //r.POST("/api/sql-health/clear", sqlHealthController.ClearSQLRecords)
r.GET("/api/sql-health/dashboard", sqlHealthController.GetSQLHealthDashboard) //r.GET("/api/sql-health/dashboard", sqlHealthController.GetSQLHealthDashboard)
// 8. API 监控端点（ES 和 Redis 调用监控）
apiMonitorController := monitor.NewAPIMonitorController()
r.GET("/api/api-monitor/stats", apiMonitorController.GetAPIStats)
r.GET("/api/api-monitor/all-stats", apiMonitorController.GetAllAPIStats)
r.GET("/api/api-monitor/es-calls", apiMonitorController.GetAPIESCalls)
r.GET("/api/api-monitor/redis-calls", apiMonitorController.GetAPIRedisCalls)
r.GET("/api/api-monitor/dashboard", apiMonitorController.GetAPIMonitorDashboard)
r.GET("/api/api-monitor/call-detail", apiMonitorController.GetAPICallDetail)
// =================== ⭐ ES ===================== // =================== ⭐ ES =====================
// ISBN 模糊搜索 // ISBN 模糊搜索
r.GET("/api/es/searchByISBNLike", esService.SearchBooksHandler) r.GET("/api/es/searchByISBNLike", esService.SearchBooksHandler) //监控
// ISBN 精确搜索 // ISBN 精确搜索
r.GET("/api/es/searchByISBN", esService.SearchBookByISBNHandler) //1 r.GET("/api/es/searchByISBN", esService.SearchBookByISBNHandler) //监控
// 书名搜索 // 书名搜索
r.GET("/api/es/searchByBookName", esService.SearchBookByBookNameHandler) r.GET("/api/es/searchByBookName", esService.SearchBookByBookNameHandler)
// 全字段搜索 // 全字段搜索
@ -312,7 +320,7 @@ func main() {
// 根据条件查询 ES 图书信息 // 根据条件查询 ES 图书信息
r.GET("/api/es/getBookBaseInfoES", bookController.SearchBookBaseInfoHandler) r.GET("/api/es/getBookBaseInfoES", bookController.SearchBookBaseInfoHandler)
// 新增：根据ISBN查询ES中是否存在，不存在则新增数据,存在则根据参数更新 // 新增：根据ISBN查询ES中是否存在，不存在则新增数据,存在则根据参数更新
//r.POST("/api/es/addBookToES", bookController.AddBookToESHandler) r.POST("/api/es/addBookToES", bookController.AddBookToESHandler)
// 更新：根据ISBN通用更新图书字段 // 更新：根据ISBN通用更新图书字段
r.POST("/api/es/updateBookFieldsByISBN", bookController.UpdateBookFieldsByISBNHandler) r.POST("/api/es/updateBookFieldsByISBN", bookController.UpdateBookFieldsByISBNHandler)
// 更新：根据ISBN通用更新图书字段 // 更新：根据ISBN通用更新图书字段
@ -324,7 +332,7 @@ func main() {
//------------------------------------------------------------------------ //------------------------------------------------------------------------
// 新：核价软件用批量获取 // 新：核价软件用批量获取
r.GET("/api/es/batchGetBookBaseInfoES", esService.BatchGetBookBaseInfoESHandler) r.GET("/api/es/batchGetBookBaseInfoES", esService.BatchGetBookBaseInfoESHandler) //监控
// 多条件高级搜索 // 多条件高级搜索
r.GET("/api/es/searchAdvanced", esService.SearchBooksByConditionsHandler) r.GET("/api/es/searchAdvanced", esService.SearchBooksByConditionsHandler)
// ID 范围计数 // ID 范围计数
@ -338,7 +346,7 @@ func main() {
// 新增：根据ISBN通用更新图书字段 // 新增：根据ISBN通用更新图书字段
//r.POST("/api/es/updateBookFieldsByISBN", esService.UpdateBookFieldsByISBNHandler) //1 //r.POST("/api/es/updateBookFieldsByISBN", esService.UpdateBookFieldsByISBNHandler) //1
// 新增：根据ISBN查询ES中是否存在，不存在则新增数据,存在则根据参数更新 // 新增：根据ISBN查询ES中是否存在，不存在则新增数据,存在则根据参数更新
r.POST("/api/es/addBookToES", esService.AddBookToESHandler) //1 //r.POST("/api/es/addBookToES", esService.AddBookToESHandler) //1
// 新增：完整插入接口，支持所有字段， // 新增：完整插入接口，支持所有字段，
r.POST("/api/es/addBookFullToES", esService.AddBookFullToESHandler) r.POST("/api/es/addBookFullToES", esService.AddBookFullToESHandler)
// 新增：批量插入接口，支持同时插入多本图书 // 新增：批量插入接口，支持同时插入多本图书

774
monitor/api_monitor.go Normal file
View File

@ -0,0 +1,774 @@
package monitor
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/elastic/go-elasticsearch/v8"
"github.com/elastic/go-elasticsearch/v8/esapi"
"io"
"strings"
"sync"
"time"
"github.com/go-redis/redis/v8"
)
// APIMonitor API 监控器 - 监控特定接口的 ES 和 Redis 调用
type APIMonitor struct {
endpoint string
esCalls []APICallRecord
redisCalls []APICallRecord
mutex sync.RWMutex
//maxRecords int
nextID int64
qpsWindow time.Duration // QPS 统计时间窗口
}
// APICallRecord API 调用记录
type APICallRecord struct {
ID int64 `json:"id"`
Timestamp time.Time `json:"timestamp"`
Duration int64 `json:"duration_ms"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
Operation string `json:"operation"` // ES: search/get, Redis: GET/SET/HGET
KeyOrIndex string `json:"key_or_index"` // ES 索引或 Redis Key
Query string `json:"query,omitempty"` // 查询内容
Request string `json:"request,omitempty"` // 请求内容
Response string `json:"response,omitempty"` // 响应内容
}
// APIStats API 统计数据
type APIStats struct {
Endpoint string `json:"endpoint"`
TotalCalls int64 `json:"total_calls"`
Escalls int64 `json:"es_calls"`
RedisCalls int64 `json:"redis_calls"`
ESQPS float64 `json:"es_qps"` // ES QPS
RedisQPS float64 `json:"redis_qps"` // Redis QPS
AvgDuration int64 `json:"avg_duration_ms"`
SuccessRate float64 `json:"success_rate"`
LastUpdate string `json:"last_update"`
}
// 全局监控器映射表：endpoint -> APIMonitor
var (
apiMonitors = make(map[string]*APIMonitor)
apiMonitorsMu sync.RWMutex
)
// GetOrCreateAPIMonitor 获取或创建 API 监控器
func GetOrCreateAPIMonitor(endpoint string, qpsWindow time.Duration) *APIMonitor {
apiMonitorsMu.Lock()
defer apiMonitorsMu.Unlock()
if monitor, exists := apiMonitors[endpoint]; exists {
return monitor
}
monitor := &APIMonitor{
endpoint: endpoint,
esCalls: make([]APICallRecord, 0),
redisCalls: make([]APICallRecord, 0),
//maxRecords: maxRecords,
nextID: 1,
qpsWindow: qpsWindow,
}
apiMonitors[endpoint] = monitor
return monitor
}
// RecordESCall 记录 ES 调用
func (am *APIMonitor) RecordESCall(operation, keyOrIndex, query string, duration time.Duration, err error) {
am.mutex.Lock()
defer am.mutex.Unlock()
record := APICallRecord{
ID: am.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: operation,
KeyOrIndex: keyOrIndex,
Query: query,
}
if err != nil {
record.Error = err.Error()
}
am.esCalls = append(am.esCalls, record)
//if len(am.esCalls) > am.maxRecords {
// am.esCalls = am.esCalls[1:]
//}
}
// RecordRedisCall 记录 Redis 调用
func (am *APIMonitor) RecordRedisCall(operation, keyOrIndex, query string, duration time.Duration, err error) {
am.mutex.Lock()
defer am.mutex.Unlock()
record := APICallRecord{
ID: am.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: operation,
KeyOrIndex: keyOrIndex,
Query: query,
}
if err != nil {
record.Error = err.Error()
}
am.redisCalls = append(am.redisCalls, record)
//if len(am.redisCalls) > am.maxRecords {
// am.redisCalls = am.redisCalls[1:]
//}
}
// getNextID 获取下一个 ID
func (am *APIMonitor) getNextID() int64 {
id := am.nextID
am.nextID++
return id
}
// GetStats 获取统计数据
func (am *APIMonitor) GetStats() *APIStats {
am.mutex.RLock()
defer am.mutex.RUnlock()
now := time.Now()
windowStart := now.Add(-am.qpsWindow)
// 统计 ES 调用
var esRecentCalls, esSuccessCalls int64
var esTotalDuration int64
for _, call := range am.esCalls {
if call.Timestamp.After(windowStart) {
esRecentCalls++
}
esTotalDuration += call.Duration
if call.Success {
esSuccessCalls++
}
}
// 统计 Redis 调用
var redisRecentCalls, redisSuccessCalls int64
var redisTotalDuration int64
for _, call := range am.redisCalls {
if call.Timestamp.After(windowStart) {
redisRecentCalls++
}
redisTotalDuration += call.Duration
if call.Success {
redisSuccessCalls++
}
}
totalCalls := int64(len(am.esCalls)) + int64(len(am.redisCalls))
successCalls := esSuccessCalls + redisSuccessCalls
totalDuration := esTotalDuration + redisTotalDuration
// 计算 QPS（每秒调用数）
windowSeconds := am.qpsWindow.Seconds()
esQPS := float64(esRecentCalls) / windowSeconds
redisQPS := float64(redisRecentCalls) / windowSeconds
var avgDuration int64
if totalCalls > 0 {
avgDuration = totalDuration / totalCalls
}
var successRate float64
if totalCalls > 0 {
successRate = float64(successCalls) / float64(totalCalls) * 100
}
return &APIStats{
Endpoint: am.endpoint,
TotalCalls: totalCalls,
Escalls: int64(len(am.esCalls)),
RedisCalls: int64(len(am.redisCalls)),
ESQPS: esQPS,
RedisQPS: redisQPS,
AvgDuration: avgDuration,
SuccessRate: successRate,
LastUpdate: now.Format("2006-01-02 15:04:05"),
}
}
// GetRecentESCalls 获取最近的 ES 调用记录（支持分页）
func (am *APIMonitor) GetRecentESCalls(page, pageSize int) ([]APICallRecord, int) {
am.mutex.RLock()
defer am.mutex.RUnlock()
total := len(am.esCalls)
if total == 0 {
return []APICallRecord{}, 0
}
if page <= 0 {
page = 1
}
if pageSize <= 0 {
pageSize = 50
}
if pageSize > 500 {
pageSize = 500
}
startIndex := (page - 1) * pageSize
endIndex := startIndex + pageSize
if startIndex >= total {
return []APICallRecord{}, total
}
if endIndex > total {
endIndex = total
}
result := make([]APICallRecord, endIndex-startIndex)
copy(result, am.esCalls[startIndex:endIndex])
for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 {
result[i], result[j] = result[j], result[i]
}
return result, total
}
// GetRecentRedisCalls 获取最近的 Redis 调用记录（支持分页）
func (am *APIMonitor) GetRecentRedisCalls(page, pageSize int) ([]APICallRecord, int) {
am.mutex.RLock()
defer am.mutex.RUnlock()
total := len(am.redisCalls)
if total == 0 {
return []APICallRecord{}, 0
}
if page <= 0 {
page = 1
}
if pageSize <= 0 {
pageSize = 50
}
if pageSize > 500 {
pageSize = 500
}
startIndex := (page - 1) * pageSize
endIndex := startIndex + pageSize
if startIndex >= total {
return []APICallRecord{}, total
}
if endIndex > total {
endIndex = total
}
result := make([]APICallRecord, endIndex-startIndex)
copy(result, am.redisCalls[startIndex:endIndex])
for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 {
result[i], result[j] = result[j], result[i]
}
return result, total
}
// GetRedisCallByID 根据 ID 获取 Redis 调用记录
func (am *APIMonitor) GetRedisCallByID(callID int64) *APICallRecord {
am.mutex.RLock()
defer am.mutex.RUnlock()
for i := len(am.redisCalls) - 1; i >= 0; i-- {
if am.redisCalls[i].ID == callID {
return &am.redisCalls[i]
}
}
return nil
}
/*----------------------------------------ES-----------------------------------*/
// MonitoredESClient 带监控的 ES 客户端
type MonitoredESClient struct {
client *elasticsearch.Client
monitor *APIMonitor
endpoint string
}
// NewMonitoredESClient 创建带监控的 ES 客户端
func NewMonitoredESClient(client *elasticsearch.Client, endpoint string) *MonitoredESClient {
//monitor := GetOrCreateAPIMonitor(endpoint, 1000, 1*time.Minute)
monitor := GetOrCreateAPIMonitor(endpoint, 1*time.Minute)
return &MonitoredESClient{
client: client,
monitor: monitor,
endpoint: endpoint,
}
}
// Search 带监控的 ES 搜索（使用 esapi.SearchRequest）
func (m *MonitoredESClient) Search(ctx context.Context, req *esapi.SearchRequest) (*esapi.Response, time.Duration, error) {
startTime := time.Now()
indexName := ""
if len(req.Index) > 0 {
indexName = strings.Join(req.Index, ",")
}
queryBody := ""
if req.Body != nil {
buf := new(bytes.Buffer)
buf.ReadFrom(req.Body)
queryBody = buf.String()
// 重新设置 Body，因为 ReadFrom 已经读取了
req.Body = io.NopCloser(buf)
}
req.Pretty = true
resp, err := req.Do(ctx, m.client)
duration := time.Since(startTime)
record := APICallRecord{
ID: m.monitor.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: "search",
KeyOrIndex: indexName,
Query: queryBody,
}
if err != nil {
record.Error = err.Error()
}
if err != nil {
record.Error = err.Error()
}
//m.monitor.RecordAPICall(record, "es")
m.monitor.RecordESCall("search", indexName, queryBody, duration, err)
return resp, duration, err
}
// RecordAPICall 通用记录方法
func (am *APIMonitor) RecordAPICall(record APICallRecord, callType string) {
am.mutex.Lock()
defer am.mutex.Unlock()
if callType == "es" {
am.esCalls = append(am.esCalls, record)
//if len(am.esCalls) > am.maxRecords {
// am.esCalls = am.esCalls[1:]
//}
} else if callType == "redis" {
am.redisCalls = append(am.redisCalls, record)
//if len(am.redisCalls) > am.maxRecords {
// am.redisCalls = am.redisCalls[1:]
//}
}
}
// SearchWithQuery 带监控的 ES 搜索（使用查询字符串）
func (m *MonitoredESClient) SearchWithQuery(ctx context.Context, index, query string) (*esapi.Response, time.Duration, error) {
startTime := time.Now()
req := esapi.SearchRequest{
Index: []string{index},
Body: strings.NewReader(query),
Pretty: true,
}
req.Pretty = true
resp, err := req.Do(ctx, m.client)
duration := time.Since(startTime)
m.monitor.RecordESCall("search", index, query, duration, err)
return resp, duration, err
}
// Get 带监控的 ES Get 文档
func (m *MonitoredESClient) Get(ctx context.Context, index, docID string) (*esapi.Response, time.Duration, error) {
startTime := time.Now()
req := esapi.GetRequest{
Index: index,
DocumentID: docID,
}
resp, err := req.Do(ctx, m.client)
duration := time.Since(startTime)
m.monitor.RecordESCall("get", index, "docID: "+docID, duration, err)
return resp, duration, err
}
// Index 带监控的 ES Index 文档
func (m *MonitoredESClient) Index(ctx context.Context, index, docID string, body io.Reader) (*esapi.Response, time.Duration, error) {
startTime := time.Now()
req := esapi.IndexRequest{
Index: index,
DocumentID: docID,
Body: body,
}
resp, err := req.Do(ctx, m.client)
duration := time.Since(startTime)
bodyStr := ""
if body != nil {
buf := new(bytes.Buffer)
buf.ReadFrom(body)
bodyStr = buf.String()
}
m.monitor.RecordESCall("index", index, bodyStr, duration, err)
return resp, duration, err
}
// Delete 带监控的 ES Delete 文档
func (m *MonitoredESClient) Delete(ctx context.Context, index, docID string) (*esapi.Response, time.Duration, error) {
startTime := time.Now()
req := esapi.DeleteRequest{
Index: index,
DocumentID: docID,
}
resp, err := req.Do(ctx, m.client)
duration := time.Since(startTime)
m.monitor.RecordESCall("delete", index, "docID: "+docID, duration, err)
return resp, duration, err
}
// Update 带监控的 ES Update 文档
func (m *MonitoredESClient) Update(ctx context.Context, index, docID string, body io.Reader) (*esapi.Response, time.Duration, error) {
startTime := time.Now()
req := esapi.UpdateRequest{
Index: index,
DocumentID: docID,
Body: body,
}
resp, err := req.Do(ctx, m.client)
duration := time.Since(startTime)
bodyStr := ""
if body != nil {
buf := new(bytes.Buffer)
buf.ReadFrom(body)
bodyStr = buf.String()
}
m.monitor.RecordESCall("update", index, bodyStr, duration, err)
return resp, duration, err
}
// Count 带监控的 ES Count
func (m *MonitoredESClient) Count(ctx context.Context, index string, query string) (*esapi.Response, time.Duration, error) {
startTime := time.Now()
req := esapi.CountRequest{
Index: []string{index},
}
if query != "" {
req.Body = strings.NewReader(query)
}
resp, err := req.Do(ctx, m.client)
duration := time.Since(startTime)
m.monitor.RecordESCall("count", index, query, duration, err)
return resp, duration, err
}
// GetESCallByID 根据 ID 获取 ES 调用记录
func (am *APIMonitor) GetESCallByID(callID int64) *APICallRecord {
am.mutex.RLock()
defer am.mutex.RUnlock()
for i := len(am.esCalls) - 1; i >= 0; i-- {
if am.esCalls[i].ID == callID {
return &am.esCalls[i]
}
}
return nil
}
/*-----------------------------------REDIS-----------------------------------*/
// MonitoredRedisClient 带监控的 Redis 客户端
type MonitoredRedisClient struct {
client *redis.Client
monitor *APIMonitor
endpoint string
}
// NewMonitoredRedisClient 创建带监控的 Redis 客户端
func NewMonitoredRedisClient(client *redis.Client, endpoint string) *MonitoredRedisClient {
//monitor := GetOrCreateAPIMonitor(endpoint, 1000, 1*time.Minute)
monitor := GetOrCreateAPIMonitor(endpoint, 1*time.Minute)
return &MonitoredRedisClient{
client: client,
monitor: monitor,
endpoint: endpoint,
}
}
// Get 带监控的 Redis Get 操作
func (m *MonitoredRedisClient) Get(ctx context.Context, key string) (string, time.Duration, error) {
startTime := time.Now()
val, err := m.client.Get(ctx, key).Result()
duration := time.Since(startTime)
record := APICallRecord{
ID: m.monitor.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: "GET",
KeyOrIndex: key,
Query: "",
Response: val,
}
if err != nil {
record.Error = err.Error()
}
//m.monitor.RecordAPICall(record, "redis")
m.monitor.RecordRedisCall("GET", key, "", duration, err)
return val, duration, err
}
// Set 带监控的 Redis Set 操作
func (m *MonitoredRedisClient) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) (string, time.Duration, error) {
startTime := time.Now()
err := m.client.Set(ctx, key, value, expiration).Err()
duration := time.Since(startTime)
query := ""
if str, ok := value.(string); ok {
query = str
} else if data, err := json.Marshal(value); err == nil {
query = string(data)
}
record := APICallRecord{
ID: m.monitor.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: "SET",
KeyOrIndex: key,
Query: query,
Response: "OK",
}
if err != nil {
record.Error = err.Error()
}
//m.monitor.RecordAPICall(record, "redis")
m.monitor.RecordRedisCall("SET", key, query, duration, err)
return "OK", duration, err
}
// ... existing code ...
// HGet 带监控的 Redis HGet 操作
func (m *MonitoredRedisClient) HGet(ctx context.Context, key, field string) (string, time.Duration, error) {
startTime := time.Now()
val, err := m.client.HGet(ctx, key, field).Result()
duration := time.Since(startTime)
record := APICallRecord{
ID: m.monitor.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: "HGET",
KeyOrIndex: key,
Query: "field: " + field,
Response: val,
}
if err != nil {
record.Error = err.Error()
}
m.monitor.RecordAPICall(record, "redis")
return val, duration, err
}
// ... existing code ...
// HSet 带监控的 Redis HSet 操作
func (m *MonitoredRedisClient) HSet(ctx context.Context, key, field string, value interface{}) (int64, time.Duration, error) {
startTime := time.Now()
result, err := m.client.HSet(ctx, key, field, value).Result()
duration := time.Since(startTime)
queryValue := ""
if str, ok := value.(string); ok {
queryValue = str
} else if data, err := json.Marshal(value); err == nil {
queryValue = string(data)
}
record := APICallRecord{
ID: m.monitor.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: "HSET",
KeyOrIndex: key,
Query: fmt.Sprintf("field: %s, value: %v", field, queryValue),
Response: fmt.Sprintf("%d", result),
}
if err != nil {
record.Error = err.Error()
}
m.monitor.RecordAPICall(record, "redis")
return result, duration, err
}
// ... existing code ...
// Exists 带监控的 Redis Exists 操作
func (m *MonitoredRedisClient) Exists(ctx context.Context, keys ...string) (int64, time.Duration, error) {
startTime := time.Now()
result, err := m.client.Exists(ctx, keys...).Result()
duration := time.Since(startTime)
record := APICallRecord{
ID: m.monitor.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: "EXISTS",
KeyOrIndex: strings.Join(keys, ", "),
Query: "",
Response: fmt.Sprintf("%d", result),
}
if err != nil {
record.Error = err.Error()
}
m.monitor.RecordAPICall(record, "redis")
return result, duration, err
}
// ... existing code ...
// Del 带监控的 Redis Del 操作
func (m *MonitoredRedisClient) Del(ctx context.Context, keys ...string) (int64, time.Duration, error) {
startTime := time.Now()
result, err := m.client.Del(ctx, keys...).Result()
duration := time.Since(startTime)
record := APICallRecord{
ID: m.monitor.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: "DEL",
KeyOrIndex: strings.Join(keys, ", "),
Query: "",
Response: fmt.Sprintf("%d", result),
}
if err != nil {
record.Error = err.Error()
}
m.monitor.RecordAPICall(record, "redis")
return result, duration, err
}
// ... existing code ...
// MGet 带监控的 Redis MGet 操作
func (m *MonitoredRedisClient) MGet(ctx context.Context, keys ...string) ([]interface{}, time.Duration, error) {
startTime := time.Now()
result, err := m.client.MGet(ctx, keys...).Result()
duration := time.Since(startTime)
resultJSON := ""
if data, marshalErr := json.Marshal(result); marshalErr == nil {
resultJSON = string(data)
}
record := APICallRecord{
ID: m.monitor.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: "MGET",
KeyOrIndex: strings.Join(keys, ", "),
Query: "",
Response: resultJSON,
}
if err != nil {
record.Error = err.Error()
}
m.monitor.RecordAPICall(record, "redis")
return result, duration, err
}
// ... existing code ...
// Ping 带监控的 Redis Ping 操作
func (m *MonitoredRedisClient) Ping(ctx context.Context) (string, time.Duration, error) {
startTime := time.Now()
val, err := m.client.Ping(ctx).Result()
duration := time.Since(startTime)
record := APICallRecord{
ID: m.monitor.getNextID(),
Timestamp: time.Now(),
Duration: duration.Milliseconds(),
Success: err == nil,
Operation: "PING",
KeyOrIndex: "",
Query: "PING",
Response: val,
}
if err != nil {
record.Error = err.Error()
}
m.monitor.RecordAPICall(record, "redis")
return val, duration, err
}

773
monitor/health_api.go Normal file
View File

@ -0,0 +1,773 @@
package monitor
import (
"net/http"
"sort"
"strconv"
"time"
"github.com/gin-gonic/gin"
)
// APIMonitorController API 监控控制器
type APIMonitorController struct{}
// NewAPIMonitorController 创建 API 监控控制器
func NewAPIMonitorController() *APIMonitorController {
return &APIMonitorController{}
}
// GetAPIStats 获取指定 API 的统计信息
func (amc *APIMonitorController) GetAPIStats(c *gin.Context) {
endpoint := c.Query("endpoint")
if endpoint == "" {
c.JSON(http.StatusBadRequest, gin.H{
"error": "缺少 endpoint 参数",
})
return
}
apiMonitorsMu.RLock()
monitor, exists := apiMonitors[endpoint]
apiMonitorsMu.RUnlock()
if !exists {
c.JSON(http.StatusNotFound, gin.H{
"error": "该接口暂无监控数据",
})
return
}
stats := monitor.GetStats()
c.JSON(http.StatusOK, gin.H{
"status": "success",
"data": stats,
"timestamp": time.Now().Format("2006-01-02 15:04:05"),
})
}
// GetAllAPIStats 获取所有 API 的统计信息
func (amc *APIMonitorController) GetAllAPIStats(c *gin.Context) {
apiMonitorsMu.RLock()
defer apiMonitorsMu.RUnlock()
allStats := make([]*APIStats, 0, len(apiMonitors))
endpoints := make([]string, 0, len(apiMonitors))
// 收集所有 endpoint
for endpoint := range apiMonitors {
endpoints = append(endpoints, endpoint)
}
// 按 endpoint 字母顺序排序
sort.Strings(endpoints)
// 按排序后的顺序获取统计信息
for _, endpoint := range endpoints {
allStats = append(allStats, apiMonitors[endpoint].GetStats())
}
c.JSON(http.StatusOK, gin.H{
"status": "success",
"data": allStats,
"count": len(allStats),
"timestamp": time.Now().Format("2006-01-02 15:04:05"),
})
}
// GetAPICallDetail 获取指定 API 调用的详细信息
func (amc *APIMonitorController) GetAPICallDetail(c *gin.Context) {
endpoint := c.Query("endpoint")
callIDStr := c.Query("call_id")
callType := c.Query("type") // es or redis
if endpoint == "" || callIDStr == "" {
c.JSON(http.StatusBadRequest, gin.H{
"error": "缺少 endpoint 或 call_id 参数",
})
return
}
callID, err := strconv.ParseInt(callIDStr, 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{
"error": "无效的 call_id",
})
return
}
apiMonitorsMu.RLock()
monitor, exists := apiMonitors[endpoint]
apiMonitorsMu.RUnlock()
if !exists {
c.JSON(http.StatusNotFound, gin.H{
"error": "该接口暂无监控数据",
})
return
}
// 查找对应的调用记录
var foundRecord *APICallRecord
var recordType string
if callType == "es" {
foundRecord = monitor.GetESCallByID(callID)
if foundRecord != nil {
recordType = "ES"
}
} else if callType == "redis" {
foundRecord = monitor.GetRedisCallByID(callID)
if foundRecord != nil {
recordType = "Redis"
}
} else {
c.JSON(http.StatusBadRequest, gin.H{
"error": "type 参数必须是 es 或 redis",
})
return
}
if foundRecord == nil {
c.JSON(http.StatusNotFound, gin.H{
"error": "未找到对应的调用记录",
})
return
}
c.JSON(http.StatusOK, gin.H{
"status": "success",
"data": gin.H{
"id": foundRecord.ID,
"type": recordType,
"timestamp": foundRecord.Timestamp.Format("2006-01-02 15:04:05"),
"duration_ms": foundRecord.Duration,
"success": foundRecord.Success,
"error": foundRecord.Error,
"operation": foundRecord.Operation,
"key_or_index": foundRecord.KeyOrIndex,
"query": foundRecord.Query,
"request": foundRecord.Request,
"response": foundRecord.Response,
},
})
}
// GetAPIESCalls 获取指定 API 的 ES 调用记录
func (amc *APIMonitorController) GetAPIESCalls(c *gin.Context) {
endpoint := c.Query("endpoint")
if endpoint == "" {
c.JSON(http.StatusBadRequest, gin.H{
"error": "缺少 endpoint 参数",
})
return
}
pageStr := c.DefaultQuery("page", "1")
pageSizeStr := c.DefaultQuery("page_size", "50")
page, err := strconv.Atoi(pageStr)
if err != nil || page <= 0 {
page = 1
}
pageSize, err := strconv.Atoi(pageSizeStr)
if err != nil || pageSize <= 0 {
pageSize = 50
}
if pageSize > 500 {
pageSize = 500
}
apiMonitorsMu.RLock()
monitor, exists := apiMonitors[endpoint]
apiMonitorsMu.RUnlock()
if !exists {
c.JSON(http.StatusNotFound, gin.H{
"error": "该接口暂无监控数据",
})
return
}
calls, total := monitor.GetRecentESCalls(page, pageSize)
totalPages := (total + pageSize - 1) / pageSize
c.JSON(http.StatusOK, gin.H{
"status": "success",
"data": gin.H{
"calls": calls,
"count": len(calls),
"total": total,
"page": page,
"page_size": pageSize,
"total_pages": totalPages,
},
"timestamp": time.Now().Format("2006-01-02 15:04:05"),
})
}
// GetAPIRedisCalls 获取指定 API 的 Redis 调用记录
func (amc *APIMonitorController) GetAPIRedisCalls(c *gin.Context) {
endpoint := c.Query("endpoint")
if endpoint == "" {
c.JSON(http.StatusBadRequest, gin.H{
"error": "缺少 endpoint 参数",
})
return
}
pageStr := c.DefaultQuery("page", "1")
pageSizeStr := c.DefaultQuery("page_size", "50")
page, err := strconv.Atoi(pageStr)
if err != nil || page <= 0 {
page = 1
}
pageSize, err := strconv.Atoi(pageSizeStr)
if err != nil || pageSize <= 0 {
pageSize = 50
}
if pageSize > 500 {
pageSize = 500
}
apiMonitorsMu.RLock()
monitor, exists := apiMonitors[endpoint]
apiMonitorsMu.RUnlock()
if !exists {
c.JSON(http.StatusNotFound, gin.H{
"error": "该接口暂无监控数据",
})
return
}
calls, total := monitor.GetRecentRedisCalls(page, pageSize)
totalPages := (total + pageSize - 1) / pageSize
c.JSON(http.StatusOK, gin.H{
"status": "success",
"data": gin.H{
"calls": calls,
"count": len(calls),
"total": total,
"page": page,
"page_size": pageSize,
"total_pages": totalPages,
},
"timestamp": time.Now().Format("2006-01-02 15:04:05"),
})
}
// GetAPIMonitorDashboard 获取 API 监控仪表板
func (amc *APIMonitorController) GetAPIMonitorDashboard(c *gin.Context) {
dashboardHTML := `<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>API 监控仪表板</title>
<style>
body { font-family: Arial, sans-serif; margin: 20px; background-color: #f5f5f5; }
.container { max-width: 1800px; margin: 0 auto; }
.card { background: white; border-radius: 8px; padding: 20px; margin: 20px 0; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }
.stats-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 15px; }
.stat-item { text-align: center; padding: 15px; background: #f8f9fa; border-radius: 6px; }
.stat-value { font-size: 24px; font-weight: bold; color: #007bff; }
.stat-label { font-size: 14px; color: #666; margin-top: 5px; }
.success { color: #28a745; }
.warning { color: #ffc107; }
.danger { color: #dc3545; }
.btn { padding: 8px 16px; background: #007bff; color: white; border: none; border-radius: 4px; cursor: pointer; margin: 5px; }
.btn:hover { background: #0056b3; }
.table { width: 100%; border-collapse: collapse; margin-top: 10px; font-size: 12px; }
.table th, .table td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; }
.table th { background-color: #f8f9fa; font-weight: bold; }
.query-text { max-width: 300px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-family: monospace; cursor: pointer; }
.query-text:hover { background-color: #e9ecef; }
.endpoint-card { border-left: 4px solid #007bff; }
.qps-badge { display: inline-block; padding: 4px 8px; border-radius: 4px; font-size: 12px; font-weight: bold; }
.qps-high { background: #dc3545; color: white; }
.qps-medium { background: #ffc107; color: black; }
.qps-low { background: #28a745; color: white; }
.tab { display: flex; gap: 10px; margin-bottom: 15px; }
.tab-btn { padding: 8px 16px; border: none; background: #e9ecef; cursor: pointer; border-radius: 4px; }
.tab-btn.active { background: #007bff; color: white; }
.tab-content { display: none; }
.tab-content.active { display: block; }
.modal { display: none; position: fixed; z-index: 1000; left: 0; top: 0; width: 100%; height: 100%; background-color: rgba(0,0,0,0.5); }
.modal-content { background-color: white; margin: 2% auto; padding: 20px; border-radius: 8px; width: 90%; max-height: 90vh; overflow-y: auto; }
.close { color: #aaa; float: right; font-size: 28px; font-weight: bold; cursor: pointer; }
.close:hover { color: black; }
.detail-section { margin: 15px 0; }
.detail-label { font-weight: bold; color: #333; margin-bottom: 5px; }
.detail-value { background: #f8f9fa; padding: 10px; border-radius: 4px; font-family: monospace; white-space: pre-wrap; word-break: break-word; max-height: 300px; overflow-y: auto; }
.json-viewer { background: #1e1e1e; color: #d4d4d4; padding: 15px; border-radius: 4px; font-family: 'Consolas', 'Monaco', monospace; font-size: 12px; line-height: 1.5; }
.loading { text-align: center; padding: 20px; color: #666; }
.pagination { display: flex; justify-content: center; align-items: center; gap: 8px; margin-top: 15px; }
.pagination button { padding: 6px 12px; border: 1px solid #007bff; background: white; color: #007bff; cursor: pointer; border-radius: 4px; }
.pagination button:hover:not(:disabled) { background: #007bff; color: white; }
.pagination button:disabled { border-color: #ccc; color: #ccc; cursor: not-allowed; }
.pagination button.active { background: #007bff; color: white; }
.pagination-info { margin: 0 10px; font-size: 13px; color: #666; }
.page-size-selector { margin-left: 10px; }
.page-size-selector select { padding: 6px; border: 1px solid #007bff; border-radius: 4px; color: #007bff; cursor: pointer; }
</style>
</head>
<body>
<div class="container">
<h1>🔍 API 监控仪表板 - ES & Redis 调用</h1>
<div class="controls">
<button class="btn" onclick="refreshData()">立即刷新</button>
<label>
<input type="checkbox" id="autoRefresh" checked> 自动刷新 (5 秒)
</label>
</div>
<div class="card">
<h2>所有接口概览</h2>
<div id="endpointsList"></div>
</div>
<div class="card endpoint-card">
<h2>详细监控数据</h2>
<div class="tab">
<button class="tab-btn active" onclick="switchTab('stats', event)">📊 统计信息</button>
<button class="tab-btn" onclick="switchTab('es-calls', event)">🔵 ES 调用</button>
<button class="tab-btn" onclick="switchTab('redis-calls', event)">🔴 Redis 调用</button>
</div>
<div id="statsTab" class="tab-content active">
<div class="stats-grid" id="statsGrid"></div>
</div>
<div id="es-callsTab" class="tab-content">
<div id="esCallsContent"></div>
<div id="esCallsPagination" class="pagination"></div>
</div>
<div id="redis-callsTab" class="tab-content">
<div id="redisCallsContent"></div>
<div id="redisCallsPagination" class="pagination"></div>
</div>
</div>
</div>
<!-- 详情弹窗 -->
<div id="detailModal" class="modal">
<div class="modal-content">
<span class="close" onclick="closeModal()">&times;</span>
<h2 id="modalTitle">调用详情</h2>
<div id="modalContent"></div>
</div>
</div>
<script>
let autoRefreshInterval;
let currentEndpoint = '';
let esCurrentPage = 1;
let esPageSize = 50;
let esTotalPages = 1;
let redisCurrentPage = 1;
let redisPageSize = 50;
let redisTotalPages = 1;
function refreshData() {
loadAllEndpoints();
if (currentEndpoint) {
loadEndpointDetails(currentEndpoint);
}
}
function loadAllEndpoints() {
fetch('/api/api-monitor/all-stats')
.then(response => response.json())
.then(data => {
const endpoints = data.data;
const container = document.getElementById('endpointsList');
if (!endpoints || endpoints.length === 0) {
container.innerHTML = '<p>暂无监控数据</p>';
return;
}
let html = '<table class="table"><thead><tr><th>接口</th><th>总调用</th><th>ES 调用</th><th>Redis 调用</th><th>ES QPS</th><th>Redis QPS</th><th>成功率</th><th>平均耗时</th></tr></thead><tbody>';
endpoints.forEach(function(stat) {
var esQpsClass = stat.es_qps > 10 ? 'qps-high' : stat.es_qps > 1 ? 'qps-medium' : 'qps-low';
var redisQpsClass = stat.redis_qps > 10 ? 'qps-high' : stat.redis_qps > 1 ? 'qps-medium' : 'qps-low';
var successClass = stat.success_rate >= 95 ? 'success' : stat.success_rate >= 90 ? 'warning' : 'danger';
html += '<tr onclick="selectEndpoint(\'' + stat.endpoint + '\')" style="cursor: pointer;">' +
'<td>' + stat.endpoint + '</td>' +
'<td>' + stat.total_calls + '</td>' +
'<td>' + stat.es_calls + '</td>' +
'<td>' + stat.redis_calls + '</td>' +
'<td><span class="qps-badge ' + esQpsClass + '">' + stat.es_qps.toFixed(2) + '</span></td>' +
'<td><span class="qps-badge ' + redisQpsClass + '">' + stat.redis_qps.toFixed(2) + '</span></td>' +
'<td class="' + successClass + '">' + stat.success_rate.toFixed(2) + '%</td>' +
'<td>' + stat.avg_duration_ms + 'ms</td>' +
'</tr>';
});
html += '</tbody></table>';
container.innerHTML = html;
})
.catch(function(error) { console.error('加载接口列表失败:', error); });
}
function selectEndpoint(endpoint) {
currentEndpoint = endpoint;
document.querySelectorAll('.tab-btn').forEach(function(btn) {
btn.classList.remove('active');
});
const firstTab = document.querySelector('.tab-btn');
if (firstTab) {
firstTab.classList.add('active');
}
loadEndpointDetails(endpoint);
}
function loadEndpointDetails(endpoint) {
fetch('/api/api-monitor/stats?endpoint=' + encodeURIComponent(endpoint))
.then(response => response.json())
.then(data => {
const stat = data.data;
const statsGrid = document.getElementById('statsGrid');
var successClass = stat.success_rate >= 95 ? 'success' : 'warning';
statsGrid.innerHTML = '' +
'<div class="stat-item">' +
'<div class="stat-value">' + stat.total_calls + '</div>' +
'<div class="stat-label">总调用次数</div>' +
'</div>' +
'<div class="stat-item">' +
'<div class="stat-value">' + stat.es_calls + '</div>' +
'<div class="stat-label">ES 调用</div>' +
'</div>' +
'<div class="stat-item">' +
'<div class="stat-value">' + stat.redis_calls + '</div>' +
'<div class="stat-label">Redis 调用</div>' +
'</div>' +
'<div class="stat-item">' +
'<div class="stat-value">' + stat.es_qps.toFixed(2) + '</div>' +
'<div class="stat-label">ES QPS</div>' +
'</div>' +
'<div class="stat-item">' +
'<div class="stat-value">' + stat.redis_qps.toFixed(2) + '</div>' +
'<div class="stat-label">Redis QPS</div>' +
'</div>' +
'<div class="stat-item">' +
'<div class="stat-value ' + successClass + '">' + stat.success_rate.toFixed(2) + '%</div>' +
'<div class="stat-label">成功率</div>' +
'</div>' +
'<div class="stat-item">' +
'<div class="stat-value">' + stat.avg_duration_ms + 'ms</div>' +
'<div class="stat-label">平均耗时</div>' +
'</div>' +
'<div class="stat-item">' +
'<div class="stat-value">' + stat.last_update + '</div>' +
'<div class="stat-label">最后更新</div>' +
'</div>';
})
.catch(function(error) { console.error('加载统计数据失败:', error); });
loadESCalls(endpoint);
loadRedisCalls(endpoint);
}
function loadESCalls(endpoint) {
fetch('/api/api-monitor/es-calls?endpoint=' + encodeURIComponent(endpoint) + '&page=' + esCurrentPage + '&page_size=' + esPageSize)
.then(response => response.json())
.then(data => {
if (!data.data || !data.data.calls) {
const container = document.getElementById('esCallsContent');
container.innerHTML = '<p>暂无 ES 调用记录</p>';
document.getElementById('esCallsPagination').innerHTML = '';
return;
}
const calls = data.data.calls;
const container = document.getElementById('esCallsContent');
if (!calls || calls.length === 0) {
container.innerHTML = '<p>暂无 ES 调用记录</p>';
document.getElementById('esCallsPagination').innerHTML = '';
return;
}
esTotalPages = data.data.total_pages || 1;
const total = data.data.total || 0;
let html = '<table class="table"><thead><tr><th>ID</th><th>操作</th><th>索引</th><th>查询</th><th>耗时 (ms)</th><th>时间</th><th>状态</th><th>详情</th></tr></thead><tbody>';
calls.forEach(function(call) {
var durationClass = call.duration_ms > 1000 ? 'danger' : call.duration_ms > 500 ? 'warning' : '';
var statusClass = call.success ? 'success' : 'danger';
html += '<tr>' +
'<td>' + call.id + '</td>' +
'<td>' + call.operation + '</td>' +
'<td>' + call.key_or_index + '</td>' +
'<td class="query-text" title="' + escapeHtml(call.query) + '" onclick="viewCallDetail(' + call.id + ', \'es\')">' + (call.query || '-') + '</td>' +
'<td class="' + durationClass + '">' + call.duration_ms + '</td>' +
'<td>' + new Date(call.timestamp).toLocaleString() + '</td>' +
'<td class="' + statusClass + '">' + (call.success ? '成功' : '失败') + '</td>' +
'<td><button class="btn" style="padding: 4px 8px; font-size: 11px;" onclick="viewCallDetail(' + call.id + ', \'es\')">查看详情</button></td>' +
'</tr>';
});
html += '</tbody></table>';
container.innerHTML = html;
renderESPagination(total);
})
.catch(function(error) { console.error('加载 ES 调用记录失败:', error); });
}
function renderESPagination(total) {
const container = document.getElementById('esCallsPagination');
if (esTotalPages <= 1) {
container.innerHTML = '';
return;
}
let html = '<button onclick="changeESPage(1)"' + (esCurrentPage === 1 ? ' disabled' : '') + '>首页</button>' +
'<button onclick="changeESPage(' + (esCurrentPage - 1) + ')"' + (esCurrentPage === 1 ? ' disabled' : '') + '>上一页</button>' +
'<span class="pagination-info">第 ' + esCurrentPage + ' / ' + esTotalPages + ' 页 (共 ' + total + ' 条)</span>' +
'<button onclick="changeESPage(' + (esCurrentPage + 1) + ')"' + (esCurrentPage === esTotalPages ? ' disabled' : '') + '>下一页</button>' +
'<button onclick="changeESPage(' + esTotalPages + ')"' + (esCurrentPage === esTotalPages ? ' disabled' : '') + '>末页</button>' +
'<div class="page-size-selector">' +
'<select onchange="changeESPageSize(this.value)">' +
'<option value="20"' + (esPageSize === 20 ? ' selected' : '') + '>20 条/页</option>' +
'<option value="50"' + (esPageSize === 50 ? ' selected' : '') + '>50 条/页</option>' +
'<option value="100"' + (esPageSize === 100 ? ' selected' : '') + '>100 条/页</option>' +
'<option value="200"' + (esPageSize === 200 ? ' selected' : '') + '>200 条/页</option>' +
'</select>' +
'</div>';
container.innerHTML = html;
}
function changeESPage(page) {
if (page < 1 || page > esTotalPages) return;
esCurrentPage = page;
loadESCalls(currentEndpoint);
}
function changeESPageSize(size) {
esPageSize = parseInt(size);
esCurrentPage = 1;
loadESCalls(currentEndpoint);
}
function loadRedisCalls(endpoint) {
fetch('/api/api-monitor/redis-calls?endpoint=' + encodeURIComponent(endpoint) + '&page=' + redisCurrentPage + '&page_size=' + redisPageSize)
.then(response => response.json())
.then(data => {
if (!data.data || !data.data.calls) {
const container = document.getElementById('redisCallsContent');
container.innerHTML = '<p>暂无 Redis 调用记录</p>';
document.getElementById('redisCallsPagination').innerHTML = '';
return;
}
const calls = data.data.calls;
const container = document.getElementById('redisCallsContent');
if (!calls || calls.length === 0) {
container.innerHTML = '<p>暂无 Redis 调用记录</p>';
document.getElementById('redisCallsPagination').innerHTML = '';
return;
}
redisTotalPages = data.data.total_pages || 1;
const total = data.data.total || 0;
let html = '<table class="table"><thead><tr><th>ID</th><th>操作</th><th>Key</th><th>值</th><th>耗时 (ms)</th><th>时间</th><th>状态</th><th>详情</th></tr></thead><tbody>';
calls.forEach(function(call) {
var durationClass = call.duration_ms > 100 ? 'danger' : call.duration_ms > 50 ? 'warning' : '';
var statusClass = call.success ? 'success' : 'danger';
html += '<tr>' +
'<td>' + call.id + '</td>' +
'<td>' + call.operation + '</td>' +
'<td>' + call.key_or_index + '</td>' +
'<td class="query-text" title="' + escapeHtml(call.query) + '" onclick="viewCallDetail(' + call.id + ', \'redis\')">' + (call.query || '-') + '</td>' +
'<td class="' + durationClass + '">' + call.duration_ms + '</td>' +
'<td>' + new Date(call.timestamp).toLocaleString() + '</td>' +
'<td class="' + statusClass + '">' + (call.success ? '成功' : '失败') + '</td>' +
'<td><button class="btn" style="padding: 4px 8px; font-size: 11px;" onclick="viewCallDetail(' + call.id + ', \'redis\')">查看详情</button></td>' +
'</tr>';
});
html += '</tbody></table>';
container.innerHTML = html;
renderRedisPagination(total);
})
.catch(function(error) { console.error('加载 Redis 调用记录失败:', error); });
}
function renderRedisPagination(total) {
const container = document.getElementById('redisCallsPagination');
if (redisTotalPages <= 1) {
container.innerHTML = '';
return;
}
let html = '<button onclick="changeRedisPage(1)"' + (redisCurrentPage === 1 ? ' disabled' : '') + '>首页</button>' +
'<button onclick="changeRedisPage(' + (redisCurrentPage - 1) + ')"' + (redisCurrentPage === 1 ? ' disabled' : '') + '>上一页</button>' +
'<span class="pagination-info">第 ' + redisCurrentPage + ' / ' + redisTotalPages + ' 页 (共 ' + total + ' 条)</span>' +
'<button onclick="changeRedisPage(' + (redisCurrentPage + 1) + ')"' + (redisCurrentPage === redisTotalPages ? ' disabled' : '') + '>下一页</button>' +
'<button onclick="changeRedisPage(' + redisTotalPages + ')"' + (redisCurrentPage === redisTotalPages ? ' disabled' : '') + '>末页</button>' +
'<div class="page-size-selector">' +
'<select onchange="changeRedisPageSize(this.value)">' +
'<option value="20"' + (redisPageSize === 20 ? ' selected' : '') + '>20 条/页</option>' +
'<option value="50"' + (redisPageSize === 50 ? ' selected' : '') + '>50 条/页</option>' +
'<option value="100"' + (redisPageSize === 100 ? ' selected' : '') + '>100 条/页</option>' +
'<option value="200"' + (redisPageSize === 200 ? ' selected' : '') + '>200 条/页</option>' +
'</select>' +
'</div>';
container.innerHTML = html;
}
function changeRedisPage(page) {
if (page < 1 || page > redisTotalPages) return;
redisCurrentPage = page;
loadRedisCalls(currentEndpoint);
}
function changeRedisPageSize(size) {
redisPageSize = parseInt(size);
redisCurrentPage = 1;
loadRedisCalls(currentEndpoint);
}
function viewCallDetail(callId, type) {
const modal = document.getElementById('detailModal');
const modalTitle = document.getElementById('modalTitle');
const modalContent = document.getElementById('modalContent');
modalTitle.innerText = (type === 'es' ? 'ES' : 'Redis') + ' 调用详情 - ID: ' + callId;
modalContent.innerHTML = '<div class="loading">加载中...</div>';
modal.style.display = 'block';
fetch('/api/api-monitor/call-detail?endpoint=' + encodeURIComponent(currentEndpoint) + '&call_id=' + callId + '&type=' + type)
.then(response => response.json())
.then(data => {
const detail = data.data;
let html = '' +
'<div class="detail-section">' +
'<div class="detail-label">基本信息</div>' +
'<table class="table">' +
'<tr><th width="150">调用 ID</th><td>' + detail.id + '</td></tr>' +
'<tr><th>类型</th><td>' + detail.type + '</td></tr>' +
'<tr><th>时间</th><td>' + detail.timestamp + '</td></tr>' +
'<tr><th>操作</th><td>' + detail.operation + '</td></tr>' +
'<tr><th>耗时</th><td class="' + (detail.duration_ms > 500 ? 'danger' : 'success') + '">' + detail.duration_ms + ' ms</td></tr>' +
'<tr><th>状态</th><td class="' + (detail.success ? 'success' : 'danger') + '">' + (detail.success ? '✓ 成功' : '✗ 失败') + '</td></tr>' +
(detail.error ? '<tr><th>错误信息</th><td class="danger">' + escapeHtml(detail.error) + '</td></tr>' : '') +
'</table>' +
'</div>' +
'<div class="detail-section">' +
'<div class="detail-label">查询信息</div>' +
'<table class="table">' +
'<tr><th>索引/Key</th><td>' + escapeHtml(detail.key_or_index) + '</td></tr>' +
'<tr><th>查询内容</th><td><div class="json-viewer">' + formatJSON(detail.query) + '</div></td></tr>' +
'</table>' +
'</div>';
if (detail.request) {
html += '<div class="detail-section">' +
'<div class="detail-label">请求内容</div>' +
'<div class="json-viewer">' + formatJSON(detail.request) + '</div>' +
'</div>';
}
if (detail.response) {
html += '<div class="detail-section">' +
'<div class="detail-label">响应内容</div>' +
'<div class="json-viewer">' + formatJSON(detail.response) + '</div>' +
'</div>';
}
modalContent.innerHTML = html;
})
.catch(function(error) {
modalContent.innerHTML = '<div class="danger">加载详情失败：' + error + '</div>';
});
}
function closeModal() {
document.getElementById('detailModal').style.display = 'none';
}
function switchTab(tabName, event) {
document.querySelectorAll('.tab-content').forEach(function(content) {
content.classList.remove('active');
});
document.querySelectorAll('.tab-btn').forEach(function(btn) {
btn.classList.remove('active');
});
var targetTab = document.getElementById(tabName + 'Tab');
if (targetTab) {
targetTab.classList.add('active');
}
if (event && event.target) {
event.target.classList.add('active');
}
}
function toggleAutoRefresh() {
const checkbox = document.getElementById('autoRefresh');
if (checkbox.checked) {
autoRefreshInterval = setInterval(refreshData, 5000);
} else {
clearInterval(autoRefreshInterval);
}
}
function escapeHtml(text) {
if (!text) return '';
return text.toString()
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#039;');
}
function formatJSON(jsonStr) {
if (!jsonStr) return '';
try {
if (typeof jsonStr === 'string') {
const obj = JSON.parse(jsonStr);
return JSON.stringify(obj, null, 2);
}
return JSON.stringify(jsonStr, null, 2);
} catch (e) {
return jsonStr;
}
}
window.onclick = function(event) {
const modal = document.getElementById('detailModal');
if (event.target == modal) {
closeModal();
}
}
document.getElementById('autoRefresh').addEventListener('change', toggleAutoRefresh);
refreshData();
toggleAutoRefresh();
</script>
</body>
</html>`
c.Header("Content-Type", "text/html; charset=utf-8")
c.String(http.StatusOK, dashboardHTML)
}

144
service/book.go
View File

@ -262,46 +262,12 @@ func (svc *BookService) UpdateBookFieldsByISBN(request *request.BookUpdateReques
if book == nil { if book == nil {
return nil, fmt.Errorf("未找到该 ISBN 对应的图书") return nil, fmt.Errorf("未找到该 ISBN 对应的图书")
} }
// 获取字段配置
fieldConfig := es.GetESFieldConfig()
// 构建更新脚本 // 构建更新脚本
var scriptParts []string var scriptParts []string
params := make(map[string]interface{}) params := make(map[string]interface{})
allowedFields := map[string]bool{
"book_name": true,
"book_pic": true,
"book_pic_s": true,
"book_pic_b": true,
"book_pic_w": true,
"author": true,
"category": true,
"publisher": true,
"publication_time": true,
"binding_layout": true,
"fix_price": true,
"content": true,
"is_suit": true,
"day_sale_7": true,
"day_sale_15": true,
"day_sale_30": true,
"day_sale_60": true,
"day_sale_90": true,
"day_sale_180": true,
"day_sale_365": true,
"this_year_sale": true,
"last_year_sale": true,
"total_sale": true,
"buy_counts": true,
"sell_counts": true,
"is_illegal": true,
"is_return": true,
"is_filter": true,
"update_time": true,
"page_count": true,
"word_count": true,
"book_format": true,
"cat_id": true,
}
// 判断 is_suit 是否已传递，如果没传则自动检测 // 判断 is_suit 是否已传递，如果没传则自动检测
if _, exists := request.Data["is_suit"]; !exists { if _, exists := request.Data["is_suit"]; !exists {
// 未传递 is_suit，自动检测并设置 // 未传递 is_suit，自动检测并设置
@ -310,7 +276,9 @@ func (svc *BookService) UpdateBookFieldsByISBN(request *request.BookUpdateReques
} }
for field, value := range request.Data { for field, value := range request.Data {
if !allowedFields[field] { // 使用配置检查字段是否允许更新
if !fieldConfig.IsAllowUpdate(field) {
log.Printf("[UpdateBookFieldsByISBN] 字段 %s 不允许更新，已跳过", field)
continue continue
} }
scriptParts = append(scriptParts, scriptParts = append(scriptParts,
@ -320,7 +288,6 @@ func (svc *BookService) UpdateBookFieldsByISBN(request *request.BookUpdateReques
if len(scriptParts) == 0 { if len(scriptParts) == 0 {
return nil, fmt.Errorf("没有有效的更新字段") return nil, fmt.Errorf("没有有效的更新字段")
} }
body := map[string]interface{}{ body := map[string]interface{}{
"script": map[string]interface{}{ "script": map[string]interface{}{
"source": strings.Join(scriptParts, " "), "source": strings.Join(scriptParts, " "),
@ -588,9 +555,9 @@ func (svc *BookService) buildUpdateData(existing, new *es.ESBook) map[string]int
{existing.PublicationTime == "" && new.PublicationTime != "", "publication_time", new.PublicationTime}, {existing.PublicationTime == "" && new.PublicationTime != "", "publication_time", new.PublicationTime},
{existing.BookName.Value == "" && new.BookName.Value != "", "book_name", new.BookName.Value}, {existing.BookName.Value == "" && new.BookName.Value != "", "book_name", new.BookName.Value},
{existing.Author == "" && new.Author != "", "author", new.Author}, {existing.Author == "" && new.Author != "", "author", new.Author},
{existing.BookPic.PddPath == "" && new.BookPic.PddPath != "", "book_pic", {existing.BookPic.PddPath == "" && new.BookPic.PddPath != "" && strings.Contains(new.BookPic.PddPath, "http"), "book_pic",
map[string]interface{}{"localPath": "", "pddPath": new.BookPic.PddPath}}, map[string]interface{}{"localPath": "", "pddPath": new.BookPic.PddPath}},
{existing.BookPicS.PddResponse == "" && new.BookPicS.PddResponse != "", "book_pic_s", {existing.BookPicS.PddResponse == "" && new.BookPicS.PddResponse != "" && strings.Contains(new.BookPicS.PddResponse, "http"), "book_pic_s",
map[string]interface{}{"localPath": "", "pddResponse": new.BookPicS.PddResponse}}, map[string]interface{}{"localPath": "", "pddResponse": new.BookPicS.PddResponse}},
} }
@ -620,55 +587,66 @@ func (svc *BookService) addBookToES(ctx context.Context, req *es.ESBook) (*es.ES
if req.PublicationTime != "" && req.PublicationTime != "0" { if req.PublicationTime != "" && req.PublicationTime != "0" {
publicationTimeTimestamp = strconv.FormatInt(util.ParsePublicationTime(publicationTimeTimestamp), 10) publicationTimeTimestamp = strconv.FormatInt(util.ParsePublicationTime(publicationTimeTimestamp), 10)
} }
// 准备数据映射（使用 Go 字段名）
dataMap := make(map[string]interface{})
// 构建 ES 文档 // 构建 ES 文档
doc := map[string]interface{}{ dataMap["ID"] = svc.generateNewID()
"id": svc.generateNewID(), dataMap["BookName"] = req.BookName.Value
"book_name": req.BookName.Value, dataMap["BookPic"] = es.BookPicObj{LocalPath: "", PddPath: req.BookPic.PddPath}
"book_pic": es.BookPicObj{LocalPath: "", PddPath: req.BookPic.PddPath}, dataMap["BookPicS"] = es.BookPicSObj{LocalPath: "", PddResponse: req.BookPicS.PddResponse}
"book_pic_s": es.BookPicSObj{LocalPath: "", PddResponse: req.BookPicS.PddResponse}, dataMap["BookPicB"] = req.BookPicB
"book_pic_b": req.BookPicB, dataMap["BookPicW"] = make(map[string]interface{})
"book_pic_w": make(map[string]interface{}), dataMap["ISBN"] = req.ISBN
"isbn": req.ISBN, dataMap["Author"] = req.Author
"author": req.Author, dataMap["Category"] = req.Category
"category": req.Category, dataMap["Publisher"] = req.Publisher
"publisher": req.Publisher, dataMap["PublicationTime"] = publicationTimeTimestamp
"publication_time": publicationTimeTimestamp, dataMap["BindingLayout"] = req.BindingLayout
"binding_layout": req.BindingLayout, dataMap["FixPrice"] = req.FixPrice
"fix_price": req.FixPrice, dataMap["Content"] = req.Content
"content": req.Content, dataMap["IsSuit"] = map[bool]int{true: 1, false: 0}[es.CheckBookSuit(req.BookName.Value)]
"is_suit": map[bool]int{true: 1, false: 0}[es.CheckBookSuit(req.BookName.Value)], // 销量字段
"day_sale_7": salesInfo.DaySale7, dataMap["DaySale7"] = salesInfo.DaySale7
"day_sale_15": salesInfo.DaySale15, dataMap["DaySale15"] = salesInfo.DaySale15
"day_sale_30": salesInfo.DaySale30, dataMap["DaySale30"] = salesInfo.DaySale30
"day_sale_60": salesInfo.DaySale60, dataMap["DaySale60"] = salesInfo.DaySale60
"day_sale_90": salesInfo.DaySale90, dataMap["DaySale90"] = salesInfo.DaySale90
"day_sale_180": salesInfo.DaySale180, dataMap["DaySale180"] = salesInfo.DaySale180
"day_sale_365": salesInfo.DaySale365, dataMap["DaySale365"] = salesInfo.DaySale365
"this_year_sale": salesInfo.ThisYearSale, dataMap["ThisYearSale"] = salesInfo.ThisYearSale
"last_year_sale": salesInfo.LastYearSale, dataMap["LastYearSale"] = salesInfo.LastYearSale
"total_sale": salesInfo.TotalSale, dataMap["TotalSale"] = salesInfo.TotalSale
"buy_counts": req.BuyCounts, dataMap["BuyCounts"] = req.BuyCounts
"sell_counts": req.SellCounts, dataMap["SellCounts"] = req.SellCounts
"book_pic_obj": req.BookPicObj, dataMap["BookPicObj"] = req.BookPicObj
"book_pic_obj_s": req.BookPicObjS, dataMap["BookPicObjS"] = req.BookPicObjS
"is_illegal": 0, dataMap["IsIllegal"] = 0
"is_return": 0, dataMap["IsReturn"] = 0
"is_filter": "000000", dataMap["IsFilter"] = "000000"
"update_time": es.NumberOrString(fmt.Sprintf("%d", time.Now().Unix())),
"page_count": req.PageCount, // 时间字段
"word_count": req.WordCount, dataMap["UpdateTime"] = es.NumberOrString(fmt.Sprintf("%d", time.Now().Unix()))
"book_format": req.BookFormat,
// 其他字段
dataMap["PageCount"] = req.PageCount
dataMap["WordCount"] = req.WordCount
dataMap["BookFormat"] = req.BookFormat
if req.Other != nil {
dataMap["Other"] = req.Other
} }
// 写入 ES fmt.Println("dataMap:", dataMap)
// 使用配置构建 ES 文档（自动转换字段名）
doc := es.GetESFieldConfig().BuildESDocument(dataMap)
//// 写入 ES
if err := svc.indexDocumentToES(ctx, doc, req.ISBN); err != nil { if err := svc.indexDocumentToES(ctx, doc, req.ISBN); err != nil {
return nil, err return nil, err
} }
// 同步 Redis // 同步 Redis
_ = svc.SyncRedisByISBN(req.ISBN, "update") _ = svc.SyncRedisByISBN(req.ISBN, "update")
// 建返回对象
// 构建返回对象 returnBook := &es.ESBook{
return &es.ESBook{
ID: doc["id"].(int64), ID: doc["id"].(int64),
BookName: es.FlexibleString{Value: req.BookName.Value}, BookName: es.FlexibleString{Value: req.BookName.Value},
BookPic: doc["book_pic"].(es.BookPicObj), BookPic: doc["book_pic"].(es.BookPicObj),
@ -705,7 +683,9 @@ func (svc *BookService) addBookToES(ctx context.Context, req *es.ESBook) (*es.ES
PageCount: req.PageCount, PageCount: req.PageCount,
WordCount: req.WordCount, WordCount: req.WordCount,
BookFormat: req.BookFormat, BookFormat: req.BookFormat,
}, nil Other: req.Other,
}
return returnBook, nil
} }
// SyncRedisByISBN 同步到Redis // SyncRedisByISBN 同步到Redis