diff --git a/csv/csv.go b/csv/csv.go index c965d73..7969910 100644 --- a/csv/csv.go +++ b/csv/csv.go @@ -15,79 +15,49 @@ import ( "sync" "sync/atomic" "time" + "unsafe" ) -// 修改操作类型 -type ModifyType int - -const ( - ModifyRow ModifyType = iota // 修改整行 - ModifyCell // 修改单元格 - InsertRow // 插入行 - DeleteRow // 删除行 -) - -// ModifyRequest 修改请求 -type ModifyRequest struct { - HandleID int64 // 句柄ID - RowNumber int64 // 行号（从1开始） - ModifyType ModifyType // 修改类型 - NewRow []string // 新行数据（整行修改） - NewCellValue string // 新单元格值 - ColumnIndex int // 列索引（单元格修改时使用） - ColumnName string // 列名（可选，单元格修改时使用） -} - -// ModifyResult 修改结果 -type ModifyResult struct { - Success bool // 是否成功 - Message string // 消息 - RowsAffected int64 // 影响的行数 - BackupFile string // 备份文件路径 - OperationID string // 操作ID（用于追踪） -} - -// RowPosition 行位置信息 -type RowPosition struct { - StartOffset int64 // 行开始偏移 - EndOffset int64 // 行结束偏移 - RowLength int // 行长度（字节数） -} +// ========================== 数据结构定义 ========================== // CSVHandle CSV文件句柄 type CSVHandle struct { - ID int64 // 句柄唯一ID - Filename string // 文件名 - Delimiter rune // 分隔符 - HasHeader bool // 是否有表头 - Header []string // 表头（如果有） - File *os.File // 底层文件句柄 - CSVReader *csv.Reader // CSV阅读器 - TotalRows int64 // 总行数（如果已计算） - IsOpen bool // 是否已打开 - OpenTime time.Time // 打开时间 - AccessTime time.Time // 最后访问时间 - AccessCount int64 // 访问计数 - mu sync.RWMutex // 读写锁（保护数据结构） - writeMu sync.Mutex // 写入专用锁（保证写入互斥） - autoCloseTimer *time.Timer // 自动关闭计时器 + ID int64 // 句柄唯一ID + Filename string // 文件名 + Delimiter rune // 分隔符 + HasHeader bool // 是否有表头 + Header []string // 表头（如果有） + File *os.File // 底层文件句柄 + CSVReader *csv.Reader // CSV阅读器 + CSVWriter *csv.Writer // CSV写入器 + TotalRows int64 // 总行数（如果已计算） + IsOpen bool // 是否已打开 + OpenTime time.Time // 打开时间 + AccessTime time.Time // 最后访问时间 + AccessCount int64 // 访问计数 + mu sync.RWMutex // 读写锁（保护数据结构） + autoCloseTimer *time.Timer // 自动关闭计时器 + cachedRowCount int64 // 缓存的行数（避免重复计算） + rowCountCached bool // 行数是否已缓存 + writeBuffer *bufio.Writer // 写入缓冲区 - // 修改相关 - rowIndex []RowPosition // 行索引（用于快速定位） - indexBuilt bool // 索引是否已构建 - tempDir string // 临时目录 - backupFiles []string // 备份文件列表 - operationLog []string // 操作日志 + // 新增：引用计数和状态管理 + refCount int64 // 引用计数 + refCountMu sync.RWMutex // 引用计数锁 + closing bool // 正在关闭中 + closed bool // 已关闭 + statusMu sync.RWMutex // 状态锁 + activeOps int64 // 正在进行的操作数 } // CSVManager CSV文件管理器 type CSVManager struct { handles sync.Map // map[int64]*CSVHandle + fileLocks sync.Map // map[string]*sync.RWMutex 文件级锁 nextHandle int64 // 下一个句柄ID maxOpen int // 最大打开文件数 semaphore chan struct{} // 信号量控制并发打开 config ManagerConfig // 管理器配置 - fileLocks *sync.Map // 文件级锁映射 } // ManagerConfig 管理器配置 @@ -97,9 +67,34 @@ type ManagerConfig struct { BufferSize int // 缓冲区大小 UseMMap bool // 是否使用内存映射（大文件） MMapThreshold int64 // 使用内存映射的阈值（字节） - CacheRows int // 缓存行数 - MaxWriteRetries int // 最大写入重试次数 - WriteTimeout time.Duration // 写入超时时间 +} + +// MergeOptions 合并选项 +type MergeOptions struct { + AppendMode bool // true=追加模式，false=覆盖模式 + SkipHeader bool // 是否跳过源文件的表头 + SkipDuplicates bool // 是否跳过重复行 + ColumnMapping map[int]int // 列映射：源文件列索引 -> 目标文件列索引 + ConflictResolution ConflictStrategy // 冲突解决策略 + TransformFunc func([]string) ([]string, error) // 数据转换函数 +} + +// ConflictStrategy 冲突解决策略 +type ConflictStrategy int + +const ( + Overwrite ConflictStrategy = iota // 覆盖目标文件的数据 + Skip // 跳过冲突的行 + KeepBoth // 保留两者 + UseSource // 使用源数据 + UseTarget // 使用目标数据 +) + +// CSVResponse CSV响应结构体 +type CSVResponse struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` + Data interface{} `json:"data,omitempty"` } // DefaultConfig 默认配置 @@ -109,9 +104,6 @@ var DefaultConfig = ManagerConfig{ BufferSize: 32 * 1024, // 32KB UseMMap: true, MMapThreshold: 10 * 1024 * 1024, // 10MB - CacheRows: 1000, - MaxWriteRetries: 3, - WriteTimeout: 30 * time.Second, } // 全局管理器实例 @@ -120,7 +112,17 @@ var ( managerInitOnce sync.Once ) -// GetManager 获取全局管理器 +// ========================== 辅助函数 ========================== + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// ========================== GetManager 获取全局管理器 ========================== + func GetManager() *CSVManager { managerInitOnce.Do(func() { globalManager = NewCSVManager(DefaultConfig) @@ -128,34 +130,260 @@ func GetManager() *CSVManager { return globalManager } -// NewCSVManager 创建新的CSV管理器 +// ========================== NewCSVManager 创建新的CSV管理器 ========================== + func NewCSVManager(config ManagerConfig) *CSVManager { if config.MaxOpenFiles <= 0 { config.MaxOpenFiles = DefaultConfig.MaxOpenFiles } return &CSVManager{ handles: sync.Map{}, + fileLocks: sync.Map{}, nextHandle: 1, maxOpen: config.MaxOpenFiles, semaphore: make(chan struct{}, config.MaxOpenFiles), config: config, - fileLocks: &sync.Map{}, } } -// getFileLock 获取文件级锁 +// ========================== CSVHandle 引用计数方法 ========================== + +// addRef 增加引用计数 +func (h *CSVHandle) addRef() { + atomic.AddInt64(&h.refCount, 1) + h.AccessTime = time.Now() + atomic.AddInt64(&h.AccessCount, 1) +} + +// releaseRef 减少引用计数，如果为0则返回true表示可以关闭 +func (h *CSVHandle) releaseRef() bool { + h.refCountMu.Lock() + defer h.refCountMu.Unlock() + + oldCount := atomic.AddInt64(&h.refCount, -1) + return oldCount <= 0 +} + +// getRefCount 获取引用计数 +func (h *CSVHandle) getRefCount() int64 { + return atomic.LoadInt64(&h.refCount) +} + +// isValid 检查句柄是否有效 +func (h *CSVHandle) isValid() bool { + h.statusMu.RLock() + defer h.statusMu.RUnlock() + return !h.closing && !h.closed && h.IsOpen +} + +// markClosing 标记为正在关闭 +func (h *CSVHandle) markClosing() { + h.statusMu.Lock() + defer h.statusMu.Unlock() + h.closing = true +} + +// markClosed 标记为已关闭 +func (h *CSVHandle) markClosed() { + h.statusMu.Lock() + defer h.statusMu.Unlock() + h.closing = false + h.closed = true + h.IsOpen = false +} + +// beginOperation 开始一个操作 +func (h *CSVHandle) beginOperation() bool { + h.statusMu.RLock() + if h.closing || h.closed || !h.IsOpen { + h.statusMu.RUnlock() + return false + } + atomic.AddInt64(&h.activeOps, 1) + h.statusMu.RUnlock() + return true +} + +// endOperation 结束一个操作 +func (h *CSVHandle) endOperation() { + atomic.AddInt64(&h.activeOps, -1) +} + +// waitForActiveOps 等待所有活动操作完成 +func (h *CSVHandle) waitForActiveOps(timeout time.Duration) bool { + start := time.Now() + for atomic.LoadInt64(&h.activeOps) > 0 { + if time.Since(start) > timeout { + return false + } + time.Sleep(10 * time.Millisecond) + } + return true +} + +// ========================== CSVHandle 文件操作方法 ========================== + +// close 关闭CSV句柄（内部方法） +func (h *CSVHandle) close() error { + h.mu.Lock() + defer h.mu.Unlock() + + if !h.IsOpen { + return nil + } + + // 停止自动关闭计时器 + if h.autoCloseTimer != nil { + h.autoCloseTimer.Stop() + h.autoCloseTimer = nil + } + + // 确保缓冲区数据写入文件 + if h.CSVWriter != nil { + h.CSVWriter.Flush() + } + if h.writeBuffer != nil { + h.writeBuffer.Flush() + } + + // 关闭文件 + if h.File != nil { + if err := h.File.Close(); err != nil { + return err + } + h.File = nil + } + + h.CSVReader = nil + h.CSVWriter = nil + h.writeBuffer = nil + h.IsOpen = false + h.markClosed() + + return nil +} + +// getHeader 获取表头 +func (h *CSVHandle) getHeader() []string { + h.mu.RLock() + defer h.mu.RUnlock() + return h.Header +} + +// calculateTotalRows 计算CSV文件的总行数 +func (h *CSVHandle) calculateTotalRows() (int64, error) { + h.mu.Lock() + defer h.mu.Unlock() + + // 如果已缓存，直接返回 + if h.rowCountCached { + return h.cachedRowCount, nil + } + + if !h.IsOpen { + return 0, fmt.Errorf("文件未打开") + } + + // 保存当前文件位置 + currentPos, err := h.File.Seek(0, 1) // 当前位置 + if err != nil { + return 0, fmt.Errorf("获取当前文件位置失败: %w", err) + } + + // 重置到文件开始 + if _, err := h.File.Seek(0, 0); err != nil { + return 0, fmt.Errorf("重置文件指针失败: %w", err) + } + + // 重置CSV阅读器 + reader := csv.NewReader(bufio.NewReader(h.File)) + reader.Comma = h.Delimiter + reader.LazyQuotes = true + reader.TrimLeadingSpace = true + reader.FieldsPerRecord = -1 // 允许字段数量可变（不检查每行字段数） + + // 统计行数 + var rowCount int64 = 0 + for { + _, err := reader.Read() + if err != nil { + if err == io.EOF { + break + } + // 恢复文件位置 + h.File.Seek(currentPos, 0) + return 0, fmt.Errorf("读取行失败: %w", err) + } + rowCount++ + } + + // 恢复文件位置 + if _, err := h.File.Seek(currentPos, 0); err != nil { + return 0, fmt.Errorf("恢复文件位置失败: %w", err) + } + + // 更新缓存 + h.cachedRowCount = rowCount + h.rowCountCached = true + h.TotalRows = rowCount + + // 如果有表头，需要减去表头行 + if h.HasHeader && rowCount > 0 { + h.cachedRowCount = rowCount - 1 + h.TotalRows = rowCount - 1 + } + + return h.cachedRowCount, nil +} + +// ========================== CSVManager 文件操作方法 ========================== + +// getFileLock 获取或创建文件锁 func (mgr *CSVManager) getFileLock(filename string) *sync.RWMutex { lock, _ := mgr.fileLocks.LoadOrStore(filename, &sync.RWMutex{}) return lock.(*sync.RWMutex) } -// ========================== 文件操作 ========================== - -// OpenCSVFile 打开CSV文件并返回句柄 +// OpenCSVFile 打开CSV文件并返回句柄（线程安全版本） func (mgr *CSVManager) OpenCSVFile(filename string, delimiter rune, hasHeader bool) (int64, error) { - // 检查文件是否存在 - if _, err := os.Stat(filename); os.IsNotExist(err) { - return -1, fmt.Errorf("文件不存在: %s", filename) + // 获取文件锁 + fileLock := mgr.getFileLock(filename) + fileLock.Lock() + defer fileLock.Unlock() + + // 首先检查文件是否已经有打开的句柄 + if existingHandleID := mgr.findHandleByFilename(filename); existingHandleID != -1 { + if handle, err := mgr.getHandle(existingHandleID); err == nil { + handle.addRef() + return existingHandleID, nil + } + } + + // 检查文件是否存在，如果不存在则创建 + fileInfo, err := os.Stat(filename) + fileExists := true + if os.IsNotExist(err) { + fileExists = false + // 创建文件 + file, createErr := os.Create(filename) + if createErr != nil { + return -1, fmt.Errorf("创建文件失败: %w", createErr) + } + file.Close() // 关闭文件，后续会重新打开 + + // 重新获取文件信息 + fileInfo, err = os.Stat(filename) + if err != nil { + return -1, fmt.Errorf("获取文件信息失败: %w", err) + } + } else if err != nil { + // 其他错误 + return -1, fmt.Errorf("检查文件状态失败: %w", err) + } + + // 如果是新创建的空文件，需要特殊处理 + if !fileExists && fileInfo.Size() == 0 { + return mgr.createEmptyCSVHandle(filename, delimiter, hasHeader) } // 限制并发打开文件数 @@ -167,12 +395,14 @@ func (mgr *CSVManager) OpenCSVFile(filename string, delimiter rune, hasHeader bo // 创建CSV句柄 csvHandle := &CSVHandle{ - ID: handleID, - Filename: filename, - Delimiter: delimiter, - HasHeader: hasHeader, - OpenTime: time.Now(), - AccessTime: time.Now(), + ID: handleID, + Filename: filename, + Delimiter: delimiter, + HasHeader: hasHeader, + OpenTime: time.Now(), + AccessTime: time.Now(), + AccessCount: 1, + refCount: 1, // 初始引用计数为1 } // 打开文件 @@ -180,21 +410,33 @@ func (mgr *CSVManager) OpenCSVFile(filename string, delimiter rune, hasHeader bo return -1, fmt.Errorf("打开文件失败: %w", err) } - // 获取总行数 - rows, err := csvHandle.getTotalRows() - if err != nil { - return -1, err - } - csvHandle.TotalRows = rows - - // 读取表头（如果需要） - if hasHeader { + // 如果是新创建的文件，可能需要写入表头 + if !fileExists && hasHeader { + // 创建空表头，用户后续可以写入实际表头 + csvHandle.Header = []string{} + } else if hasHeader && fileInfo.Size() > 0 { + // 读取现有文件的表头 if err := mgr.readHeader(csvHandle); err != nil { csvHandle.close() return -1, fmt.Errorf("读取表头失败: %w", err) } } + // 如果有数据行，计算总行数 + if fileInfo.Size() > 0 { + // 计算总行数 + rows, err := csvHandle.calculateTotalRows() + if err != nil { + csvHandle.close() + return -1, fmt.Errorf("计算总行数失败: %w", err) + } + csvHandle.TotalRows = rows + } else { + csvHandle.TotalRows = 0 + csvHandle.cachedRowCount = 0 + csvHandle.rowCountCached = true + } + // 注册到管理器 mgr.handles.Store(handleID, csvHandle) @@ -206,7 +448,297 @@ func (mgr *CSVManager) OpenCSVFile(filename string, delimiter rune, hasHeader bo return handleID, nil } -// 打开文件 +// findHandleByFilename 根据文件名查找已存在的句柄ID +func (mgr *CSVManager) findHandleByFilename(filename string) int64 { + var existingHandleID int64 = -1 + + // 遍历所有句柄，查找相同文件名的句柄 + mgr.handles.Range(func(key, value interface{}) bool { + handle := value.(*CSVHandle) + + // 检查文件名是否相同 + if handle.Filename == filename { + // 检查句柄是否有效 + if handle.isValid() { + existingHandleID = handle.ID + return false // 停止遍历 + } + } + return true // 继续遍历 + }) + + return existingHandleID +} + +// createEmptyCSVHandle 创建空的CSV文件句柄 +func (mgr *CSVManager) createEmptyCSVHandle(filename string, delimiter rune, hasHeader bool) (int64, error) { + // 限制并发打开文件数 + mgr.semaphore <- struct{}{} + defer func() { <-mgr.semaphore }() + + // 生成唯一句柄ID + handleID := atomic.AddInt64(&mgr.nextHandle, 1) + + // 创建CSV句柄 + csvHandle := &CSVHandle{ + ID: handleID, + Filename: filename, + Delimiter: delimiter, + HasHeader: hasHeader, + OpenTime: time.Now(), + AccessTime: time.Now(), + TotalRows: 0, + AccessCount: 1, + refCount: 1, // 初始引用计数为1 + } + + // 以读写模式打开文件 + file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return -1, fmt.Errorf("打开文件失败: %w", err) + } + + // 创建CSV写入器 + writer := csv.NewWriter(file) + writer.Comma = delimiter + + // 如果是新创建的文件且有表头，写入空表头占位 + if hasHeader { + // 写入空表头（用户后续需要设置实际表头） + if err := writer.Write([]string{}); err != nil { + file.Close() + return -1, fmt.Errorf("写入空表头失败: %w", err) + } + writer.Flush() + + // 重置到文件开始位置 + if _, err := file.Seek(0, 0); err != nil { + file.Close() + return -1, fmt.Errorf("重置文件指针失败: %w", err) + } + } + + // 创建CSV阅读器 + reader := csv.NewReader(bufio.NewReaderSize(file, mgr.config.BufferSize)) + reader.Comma = delimiter + reader.LazyQuotes = true + reader.TrimLeadingSpace = true + + // 创建写入缓冲区 + writeBuffer := bufio.NewWriterSize(file, mgr.config.BufferSize) + + csvHandle.File = file + csvHandle.CSVReader = reader + csvHandle.CSVWriter = csv.NewWriter(writeBuffer) + csvHandle.CSVWriter.Comma = delimiter + csvHandle.writeBuffer = writeBuffer + csvHandle.IsOpen = true + csvHandle.AccessTime = time.Now() + + // 如果有表头，初始化空表头数组 + if hasHeader { + csvHandle.Header = []string{} + } + + // 注册到管理器 + mgr.handles.Store(handleID, csvHandle) + + // 启动自动关闭计时器 + if mgr.config.AutoCloseTimeout > 0 { + csvHandle.startAutoClose(mgr.config.AutoCloseTimeout, mgr) + } + + return handleID, nil +} + +// getHandle 获取句柄对象（增加引用计数） +func (mgr *CSVManager) getHandle(handleID int64) (*CSVHandle, error) { + value, ok := mgr.handles.Load(handleID) + if !ok { + return nil, fmt.Errorf("句柄不存在: %d", handleID) + } + + handle := value.(*CSVHandle) + + // 检查句柄是否有效 + if !handle.isValid() { + return nil, fmt.Errorf("句柄无效: %d", handleID) + } + + // 增加引用计数 + handle.addRef() + + // 确保文件已打开 + handle.mu.RLock() + isOpen := handle.IsOpen + handle.mu.RUnlock() + + if !isOpen { + // 获取文件锁 + fileLock := mgr.getFileLock(handle.Filename) + fileLock.Lock() + defer fileLock.Unlock() + + // 重新检查，防止其他协程已经重新打开了 + handle.mu.RLock() + isOpen = handle.IsOpen + handle.mu.RUnlock() + + if !isOpen { + if err := mgr.openFile(handle); err != nil { + // 恢复引用计数 + handle.releaseRef() + return nil, fmt.Errorf("重新打开文件失败: %w", err) + } + } + } + + return handle, nil +} + +// releaseHandle 释放句柄引用 +func (mgr *CSVManager) releaseHandle(handleID int64) { + value, ok := mgr.handles.Load(handleID) + if !ok { + return + } + + handle := value.(*CSVHandle) + // 减少引用计数，如果为0则真正关闭 + if handle.releaseRef() { + mgr.closeHandleInternal(handleID, handle) + } +} + +// ========================== 句柄管理方法 ========================== + +// closeHandleInternal 内部关闭句柄方法 +func (mgr *CSVManager) closeHandleInternal(handleID int64, handle *CSVHandle) { + // 获取文件锁 + fileLock := mgr.getFileLock(handle.Filename) + fileLock.Lock() + defer fileLock.Unlock() + + // 再次检查引用计数，防止在获取锁期间有新的引用 + if handle.getRefCount() > 0 { + return + } + + // 标记为正在关闭 + handle.markClosing() + + // 等待所有活动操作完成 + if !handle.waitForActiveOps(30 * time.Second) { + // 超时，强制关闭 + fmt.Printf("警告：句柄 %d 关闭超时，强制关闭\n", handleID) + } + + // 真正关闭文件 + handle.close() + + // 从管理器移除 + mgr.handles.Delete(handleID) +} + +// CloseHandle 关闭指定句柄（外部调用） +func (mgr *CSVManager) CloseHandle(handleID int64) error { + value, ok := mgr.handles.Load(handleID) + if !ok { + return fmt.Errorf("句柄不存在: %d", handleID) + } + + handle := value.(*CSVHandle) + + // 减少引用计数，如果为0则真正关闭 + if handle.releaseRef() { + mgr.closeHandleInternal(handleID, handle) + } + + return nil +} + +// GracefulCloseHandle 优雅关闭句柄（等待所有操作完成） +func (mgr *CSVManager) GracefulCloseHandle(handleID int64, timeout time.Duration) error { + value, ok := mgr.handles.Load(handleID) + if !ok { + return fmt.Errorf("句柄不存在: %d", handleID) + } + + handle := value.(*CSVHandle) + + // 标记为正在关闭，阻止新操作 + handle.markClosing() + + // 等待现有操作完成 + done := make(chan bool, 1) + go func() { + // 等待引用计数降为1（只剩当前引用） + for handle.getRefCount() > 1 { + time.Sleep(100 * time.Millisecond) + } + done <- true + }() + + // 设置超时 + select { + case <-done: + // 真正关闭 + return mgr.ForceCloseHandle(handleID) + case <-time.After(timeout): + return fmt.Errorf("关闭句柄超时，仍有 %d 个引用", handle.getRefCount()) + } +} + +// ForceCloseHandle 强制关闭句柄（无论引用计数如何） +func (mgr *CSVManager) ForceCloseHandle(handleID int64) error { + value, ok := mgr.handles.Load(handleID) + if !ok { + return fmt.Errorf("句柄不存在: %d", handleID) + } + + handle := value.(*CSVHandle) + + // 强制设置引用计数为0 + handle.refCountMu.Lock() + atomic.StoreInt64(&handle.refCount, 0) + handle.refCountMu.Unlock() + + // 获取文件锁 + fileLock := mgr.getFileLock(handle.Filename) + fileLock.Lock() + defer fileLock.Unlock() + + // 标记为正在关闭 + handle.markClosing() + + // 等待所有活动操作完成 + handle.waitForActiveOps(5 * time.Second) + + // 真正关闭文件 + handle.close() + mgr.handles.Delete(handle.ID) + + return nil +} + +// closeAllHandles 关闭所有句柄 +func (mgr *CSVManager) closeAllHandles() { + // 收集所有句柄ID + var handleIDs []int64 + mgr.handles.Range(func(key, value interface{}) bool { + handleIDs = append(handleIDs, key.(int64)) + return true + }) + + // 关闭每个句柄 + for _, handleID := range handleIDs { + mgr.ForceCloseHandle(handleID) + } +} + +// ========================== 文件操作方法 ========================== + +// openFile 打开文件 func (mgr *CSVManager) openFile(handle *CSVHandle) error { handle.mu.Lock() defer handle.mu.Unlock() @@ -222,7 +754,6 @@ func (mgr *CSVManager) openFile(handle *CSVHandle) error { } fileSize := fileInfo.Size() - fmt.Println("文件大小：", fileSize) // 根据文件大小选择打开策略 if mgr.config.UseMMap && fileSize > mgr.config.MMapThreshold { @@ -234,7 +765,8 @@ func (mgr *CSVManager) openFile(handle *CSVHandle) error { // 正常打开文件 func (mgr *CSVManager) openFileNormal(handle *CSVHandle) error { - file, err := os.Open(handle.Filename) + // 以读写模式打开文件 + file, err := os.OpenFile(handle.Filename, os.O_RDWR, 0644) if err != nil { return err } @@ -245,27 +777,41 @@ func (mgr *CSVManager) openFileNormal(handle *CSVHandle) error { reader.LazyQuotes = true reader.TrimLeadingSpace = true + // 创建写入缓冲区 + writeBuffer := bufio.NewWriterSize(file, mgr.config.BufferSize) + handle.File = file handle.CSVReader = reader + handle.CSVWriter = csv.NewWriter(writeBuffer) + handle.CSVWriter.Comma = handle.Delimiter + handle.writeBuffer = writeBuffer handle.IsOpen = true handle.AccessTime = time.Now() + return nil } // 使用内存映射打开大文件 func (mgr *CSVManager) openFileWithMMap(handle *CSVHandle, fileSize int64) error { - file, err := os.Open(handle.Filename) + // 对于大文件，使用只读模式打开，写入需要特殊处理 + file, err := os.OpenFile(handle.Filename, os.O_RDWR, 0644) if err != nil { return err } // 对于大文件，我们可以先只打开，按需读取 - reader := csv.NewReader(file) + reader := csv.NewReader(bufio.NewReaderSize(file, mgr.config.BufferSize)) reader.Comma = handle.Delimiter reader.LazyQuotes = true + // 创建写入缓冲区 + writeBuffer := bufio.NewWriterSize(file, mgr.config.BufferSize) + handle.File = file handle.CSVReader = reader + handle.CSVWriter = csv.NewWriter(writeBuffer) + handle.CSVWriter.Comma = handle.Delimiter + handle.writeBuffer = writeBuffer handle.IsOpen = true handle.AccessTime = time.Now() @@ -287,7 +833,7 @@ func (mgr *CSVManager) readHeader(handle *CSVHandle) error { } // 重置CSV阅读器 - handle.CSVReader = csv.NewReader(handle.File) + handle.CSVReader = csv.NewReader(bufio.NewReader(handle.File)) handle.CSVReader.Comma = handle.Delimiter // 读取表头 @@ -300,34 +846,7 @@ func (mgr *CSVManager) readHeader(handle *CSVHandle) error { return nil } -// 获取句柄对象 -func (mgr *CSVManager) getHandle(handleID int64) (*CSVHandle, error) { - value, ok := mgr.handles.Load(handleID) - if !ok { - return nil, fmt.Errorf("句柄不存在: %d", handleID) - } - - handle := value.(*CSVHandle) - - // 确保文件已打开 - handle.mu.RLock() - isOpen := handle.IsOpen - handle.mu.RUnlock() - - if !isOpen { - if err := mgr.openFile(handle); err != nil { - return nil, fmt.Errorf("重新打开文件失败: %w", err) - } - } - - // 更新访问时间 - atomic.AddInt64(&handle.AccessCount, 1) - handle.AccessTime = time.Now() - - return handle, nil -} - -// 启动自动关闭计时器 +// startAutoClose 启动自动关闭计时器 func (h *CSVHandle) startAutoClose(timeout time.Duration, mgr *CSVManager) { h.autoCloseTimer = time.AfterFunc(timeout, func() { h.mu.Lock() @@ -335,1334 +854,1007 @@ func (h *CSVHandle) startAutoClose(timeout time.Duration, mgr *CSVManager) { // 检查是否长时间未访问 if h.IsOpen && time.Since(h.AccessTime) > timeout { - h.close() - // 从管理器移除 - mgr.handles.Delete(h.ID) + // 检查引用计数 + if h.getRefCount() == 0 { + h.close() + // 从管理器移除 + mgr.handles.Delete(h.ID) + } } }) } -// 获取表头 -func (h *CSVHandle) getHeader() []string { - h.mu.RLock() - defer h.mu.RUnlock() - return h.Header -} +// ========================== 写入功能 ========================== -//// 获取句柄信息 -//func (h *CSVHandle) getInfo() map[string]interface{} { -// h.mu.RLock() -// defer h.mu.RUnlock() -// -// return map[string]interface{}{ -// "id": h.ID, -// "filename": h.Filename, -// "delimiter": string(h.Delimiter), -// "has_header": h.HasHeader, -// "is_open": h.IsOpen, -// "open_time": h.OpenTime, -// "access_time": h.AccessTime, -// "access_count": h.AccessCount, -// "total_rows": h.TotalRows, -// "header": h.Header, -// } -//} - -// ========================== 句柄管理 ========================== - -// 关闭CSV句柄 -func (h *CSVHandle) close() error { - h.mu.Lock() - defer h.mu.Unlock() - - h.writeMu.Lock() - defer h.writeMu.Unlock() - - if !h.IsOpen { - return nil - } - - // 停止自动关闭计时器 - if h.autoCloseTimer != nil { - h.autoCloseTimer.Stop() - h.autoCloseTimer = nil - } - - // 关闭文件 - if h.File != nil { - if err := h.File.Close(); err != nil { - return err - } - h.File = nil - } - - h.CSVReader = nil - h.IsOpen = false - - return nil -} - -// 关闭指定句柄 -func (mgr *CSVManager) closeHandle(handleID int64) error { - value, ok := mgr.handles.Load(handleID) - if !ok { - return fmt.Errorf("句柄不存在: %d", handleID) - } - - handle := value.(*CSVHandle) - if err := handle.close(); err != nil { - return err - } - - // 清理文件锁 - mgr.fileLocks.Delete(handle.Filename) - - mgr.handles.Delete(handleID) - return nil -} - -// 关闭所有句柄 -func (mgr *CSVManager) closeAllHandles() { - mgr.handles.Range(func(key, value interface{}) bool { - handle := value.(*CSVHandle) - handle.close() - mgr.fileLocks.Delete(handle.Filename) - mgr.handles.Delete(key) - return true - }) -} - -// ========================== 读取操作 ========================== -// 读取一行数据 -func (h *CSVHandle) readRow() ([]string, error) { - h.mu.RLock() - defer h.mu.RUnlock() - - if !h.IsOpen { - return nil, fmt.Errorf("文件未打开") - } - - if h.CSVReader == nil { - return nil, fmt.Errorf("CSV阅读器未初始化") - } - - return h.CSVReader.Read() -} - -// 读取所有行 -func (h *CSVHandle) readAllRows() ([][]string, error) { - h.mu.Lock() - defer h.mu.Unlock() - - if !h.IsOpen { - return nil, fmt.Errorf("文件未打开") - } - - // 重置文件指针到数据开始位置 - if _, err := h.File.Seek(0, 0); err != nil { - return nil, err - } - - // 重新创建CSV阅读器 - h.CSVReader = csv.NewReader(h.File) - h.CSVReader.Comma = h.Delimiter - - // 跳过表头 - if h.HasHeader { - if _, err := h.CSVReader.Read(); err != nil { - return nil, err - } - } - - // 读取所有行 - return h.CSVReader.ReadAll() -} - -// 读取指定数量的行 -func (h *CSVHandle) readRows(count int) ([][]string, error) { - h.mu.Lock() - defer h.mu.Unlock() - - if !h.IsOpen { - return nil, fmt.Errorf("文件未打开") - } - - rows := make([][]string, 0, count) - - for i := 0; i < count; i++ { - row, err := h.CSVReader.Read() - if err == io.EOF { - break - } - if err != nil { - return rows, err - } - rows = append(rows, row) - } - - return rows, nil -} - -// 获取总行数 -func (h *CSVHandle) getTotalRows() (int64, error) { - h.mu.Lock() - defer h.mu.Unlock() - - if h.TotalRows > 0 { - return h.TotalRows, nil - } - - // 保存当前文件位置 - currentPos, err := h.File.Seek(0, io.SeekCurrent) - if err != nil { - return 0, err - } - defer h.File.Seek(currentPos, io.SeekStart) - - // 重置到文件开始 - if _, err := h.File.Seek(0, 0); err != nil { - return 0, err - } - - // 创建新的CSV阅读器进行计数 - reader := csv.NewReader(h.File) - reader.Comma = h.Delimiter - - var count int64 = 0 - - // 跳过表头 - if h.HasHeader { - if _, err := reader.Read(); err != nil { - return 0, err - } - } - - // 计数行数 - for { - _, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return count, err - } - count++ - } - - h.TotalRows = count - return count, nil -} - -// ========================== 修改操作 ========================== - -// 写入和覆盖CSV文件 -func (mgr *CSVManager) writeCSVFile(handleID int64, filename string, data [][]string) (int64, error) { - // 1. 获取句柄 +// WriteHeader 写入CSV文件表头 +func (mgr *CSVManager) WriteHeader(handleID int64, header []string) error { + // 获取句柄 handle, err := mgr.getHandle(handleID) if err != nil { - return -1, err + return fmt.Errorf("获取句柄失败: %w", err) } + defer mgr.releaseHandle(handleID) - if len(data) == 0 { - return -1, fmt.Errorf("写入的数据不能为空") + // 检查句柄状态 + if !handle.beginOperation() { + return fmt.Errorf("句柄已关闭或正在关闭: %d", handleID) } + defer handle.endOperation() - fileLock := mgr.getFileLock(handle.Filename) - fileLock.Lock() - defer fileLock.Unlock() - - var file *os.File - var fileMode int - - // 检查文件是否存在和有内容 - if info, err := os.Stat(handle.Filename); err == nil && info.Size() > 0 { - // 文件存在且有内容，追加模式 - fileMode = os.O_APPEND | os.O_WRONLY - } else { - // 文件不存在或为空，创建模式（会清空已有内容） - fileMode = os.O_CREATE | os.O_WRONLY | os.O_TRUNC - } - - // 打开或创建文件 - file, err = os.OpenFile(filename, fileMode, 0755) - if err != nil { - return -1, err - } - defer file.Close() - - // 写入CSV数据 - writer := csv.NewWriter(file) - if err := writer.WriteAll(data); err != nil { - return -1, err - } - writer.Flush() - - return -1, writer.Error() -} - -// 修改指定行 -func (mgr *CSVManager) modifyRow(handleID int64, rowNumber int64, newRow []string) (*ModifyResult, error) { - req := &ModifyRequest{ - HandleID: handleID, - RowNumber: rowNumber, - ModifyType: ModifyRow, - NewRow: newRow, - } - - return mgr.modifyCSV(req) -} - -// 通用的CSV修改函数 -func (mgr *CSVManager) modifyCSV(req *ModifyRequest) (*ModifyResult, error) { - // 1. 获取句柄 - handle, err := mgr.getHandle(req.HandleID) - if err != nil { - return &ModifyResult{ - Success: false, - Message: fmt.Sprintf("获取句柄失败: %v", err), - }, err - } - - // 2. 生成操作ID - operationID := fmt.Sprintf("%d_%d_%d", req.HandleID, time.Now().UnixNano(), req.RowNumber) - - // 3. 获取文件级写锁 - fileLock := mgr.getFileLock(handle.Filename) - fileLock.Lock() - defer fileLock.Unlock() - - // 4. 验证基本参数 - if err := mgr.validateModifyRequestBeforeLock(handle, req); err != nil { - return &ModifyResult{ - Success: false, - Message: fmt.Sprintf("参数验证失败: %v", err), - }, err - } - - // 5. 获取句柄写入锁 - handle.writeMu.Lock() - defer handle.writeMu.Unlock() - - // 6. 执行修改操作 - result, err := mgr.executeModifyOnce(handle, req, operationID) - if result != nil { - result.OperationID = operationID - } - - return result, err -} - -// validateModifyRequestBeforeLock 验证修改请求（无需句柄锁） -func (mgr *CSVManager) validateModifyRequestBeforeLock(handle *CSVHandle, req *ModifyRequest) error { - if req.RowNumber <= 0 { - return fmt.Errorf("行号必须大于0") - } - - // 验证列数 - handle.mu.RLock() - header := handle.Header - handle.mu.RUnlock() - - if req.ModifyType == ModifyRow { - expectedCols := len(header) - if len(req.NewRow) != expectedCols { - return fmt.Errorf("新行数据列数不匹配，期望: %d，实际: %d", expectedCols, len(req.NewRow)) - } - } - - return nil -} - -// executeModifyWithRetry 带重试的修改执行 -func (mgr *CSVManager) executeModifyWithRetry(handle *CSVHandle, req *ModifyRequest, operationID string) (*ModifyResult, error) { - var lastErr error - var result *ModifyResult - - for retry := 0; retry <= mgr.config.MaxWriteRetries; retry++ { - if retry > 0 { - // 重试前等待 - time.Sleep(time.Duration(retry*100) * time.Millisecond) - } - - result, lastErr = mgr.executeModifyOnce(handle, req, operationID) - if lastErr == nil { - // 记录操作日志 - handle.mu.Lock() - handle.operationLog = append(handle.operationLog, - fmt.Sprintf("%s: %s", operationID, result.Message)) - handle.mu.Unlock() - return result, nil - } - } - - return &ModifyResult{ - Success: false, - Message: fmt.Sprintf("修改失败，重试%d次后仍失败: %v", mgr.config.MaxWriteRetries, lastErr), - OperationID: operationID, - }, lastErr -} - -// executeModifyOnce 执行单次修改 -func (mgr *CSVManager) executeModifyOnce(handle *CSVHandle, req *ModifyRequest, operationID string) (*ModifyResult, error) { - // 1. 创建备份 - backupFile, err := mgr.createBackup(handle) - if err != nil { - return &ModifyResult{ - Success: false, - Message: fmt.Sprintf("创建备份失败: %v", err), - }, err - } - - // 2. 获取总行数（使用更高效的方式） - totalRows, err := mgr.getTotalRowsForModification(handle) - if err != nil { - return &ModifyResult{ - Success: false, - Message: fmt.Sprintf("获取总行数失败: %v", err), - }, err - } - - // 3. 验证行号范围 - if err := mgr.validateRowNumber(handle, req, totalRows); err != nil { - return &ModifyResult{ - Success: false, - Message: fmt.Sprintf("行号验证失败: %v", err), - }, err - } - - // 4. 执行修改逻辑 - startTime := time.Now() - var result *ModifyResult - var modifyErr error - - switch req.ModifyType { - case ModifyRow: - result, modifyErr = mgr.modifyRowInternal(handle, req, totalRows) - case ModifyCell: - result, modifyErr = mgr.modifyCellInternal(handle, req, totalRows) - case InsertRow: - result, modifyErr = mgr.insertRowInternal(handle, req, totalRows) - case DeleteRow: - result, modifyErr = mgr.deleteRowInternal(handle, req, totalRows) - default: - modifyErr = fmt.Errorf("不支持的修改类型: %v", req.ModifyType) - result = &ModifyResult{ - Success: false, - Message: fmt.Sprintf("不支持的修改类型: %v", req.ModifyType), - } - } - - // 记录执行时间 - elapsed := time.Since(startTime) - if elapsed > 1*time.Second { - fmt.Printf("警告：修改操作耗时较长: %v\n", elapsed) - } - // 5. 设置备份文件路径 - if result != nil { - result.BackupFile = backupFile - result.OperationID = operationID - } - - // 6. 记录备份文件 - if backupFile != "" { - handle.mu.Lock() - handle.backupFiles = append(handle.backupFiles, backupFile) - handle.mu.Unlock() - } - - return result, modifyErr -} - -// getTotalRowsForModification 为修改操作获取总行数（优化版本） -func (mgr *CSVManager) getTotalRowsForModification(handle *CSVHandle) (int64, error) { - // 先检查是否已有缓存 - if handle.TotalRows > 0 { - return handle.TotalRows, nil - } - - // 如果文件不大，直接快速计算 - fileInfo, err := os.Stat(handle.Filename) - if err != nil { - return 0, err - } - - // 对于小文件，使用快速计算方法 - if fileInfo.Size() < 10*1024*1024 { // 10MB以下 - return mgr.fastCountRows(handle) - } - - // 大文件使用原来的方法 - return mgr.calculateTotalRowsWithLock(handle) -} - -// fastCountRows 快速计算行数（无需复杂锁） -func (mgr *CSVManager) fastCountRows(handle *CSVHandle) (int64, error) { - file, err := os.Open(handle.Filename) - if err != nil { - return 0, err - } - defer file.Close() - - reader := csv.NewReader(file) - reader.Comma = handle.Delimiter - - var count int64 = 0 - for { - _, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return count, err - } - count++ - } - - // 减去表头行 - if handle.HasHeader { - count-- - } - - // 缓存结果 - handle.mu.Lock() - handle.TotalRows = count - handle.mu.Unlock() - - return count, nil -} - -// calculateTotalRowsWithLock 计算总行数（带锁） -func (mgr *CSVManager) calculateTotalRowsWithLock(handle *CSVHandle) (int64, error) { handle.mu.Lock() defer handle.mu.Unlock() + if !handle.IsOpen { + return fmt.Errorf("文件未打开") + } + + // 检查文件是否已经有内容 if handle.TotalRows > 0 { - return handle.TotalRows, nil + return fmt.Errorf("文件已有数据，无法修改表头") } - // 保存当前文件位置 - currentPos, err := handle.File.Seek(0, io.SeekCurrent) - if err != nil { - return 0, err - } - defer handle.File.Seek(currentPos, io.SeekStart) - - // 重置到文件开始 + // 移动到文件开头 if _, err := handle.File.Seek(0, 0); err != nil { - return 0, err + return fmt.Errorf("移动文件指针失败: %w", err) } - // 创建新的CSV阅读器进行计数 - reader := csv.NewReader(handle.File) - reader.Comma = handle.Delimiter - - var count int64 = 0 - - // 跳过表头 - if handle.HasHeader { - if _, err := reader.Read(); err != nil { - return 0, err - } + // 清空文件内容 + if err := handle.File.Truncate(0); err != nil { + return fmt.Errorf("清空文件失败: %w", err) } - // 计数行数 - for { - _, err := reader.Read() - if err == io.EOF { - break - } - if err != nil { - return count, err - } - count++ + // 写入表头 + if err := handle.CSVWriter.Write(header); err != nil { + return fmt.Errorf("写入表头失败: %w", err) } - handle.TotalRows = count - return count, nil -} - -// validateRowNumber 验证行号 -func (mgr *CSVManager) validateRowNumber(handle *CSVHandle, req *ModifyRequest, totalRows int64) error { - // 调整行号（考虑表头） - adjustedRowNumber := req.RowNumber - if handle.HasHeader { - adjustedRowNumber = req.RowNumber + 1 // 表头占第1行 + handle.CSVWriter.Flush() + if handle.writeBuffer != nil { + handle.writeBuffer.Flush() } - // 验证行号范围 - if req.ModifyType == ModifyRow || req.ModifyType == ModifyCell { - if adjustedRowNumber > totalRows { - return fmt.Errorf("行号超出范围，最大行数为: %d", totalRows-1) - } - } else if req.ModifyType == InsertRow { - // 插入行可以在末尾 - if adjustedRowNumber > totalRows+1 { - return fmt.Errorf("插入行号超出范围，最大行数为: %d", totalRows) - } - } else if req.ModifyType == DeleteRow { - if adjustedRowNumber > totalRows { - return fmt.Errorf("删除行号超出范围，最大行数为: %d", totalRows-1) - } - } + // 更新句柄状态 + handle.Header = header + handle.HasHeader = true return nil } -// createBackup 创建备份文件 -func (mgr *CSVManager) createBackup(handle *CSVHandle) (string, error) { - handle.mu.RLock() - defer handle.mu.RUnlock() - - // 创建临时目录 - if handle.tempDir == "" { - tempDir, err := os.MkdirTemp("", fmt.Sprintf("csv_backup_%d_", handle.ID)) - if err != nil { - return "", fmt.Errorf("创建临时目录失败: %v", err) - } - handle.tempDir = tempDir - } - fmt.Println("handle.tempDir", handle.tempDir) - - // 生成备份文件名 - backupFile := filepath.Join(handle.tempDir, - fmt.Sprintf("backup_%d_%s.csv", time.Now().UnixNano(), - filepath.Base(handle.Filename))) - - // 复制文件 - src, err := os.Open(handle.Filename) +// WriteRow 写入单行数据到CSV文件 +func (mgr *CSVManager) WriteRow(handleID int64, row []string) error { + // 获取句柄 + handle, err := mgr.getHandle(handleID) if err != nil { - return "", err + return fmt.Errorf("获取句柄失败: %w", err) } - defer src.Close() + defer mgr.releaseHandle(handleID) - dst, err := os.Create(backupFile) - if err != nil { - return "", err + // 检查句柄状态 + if !handle.beginOperation() { + return fmt.Errorf("句柄已关闭或正在关闭: %d", handleID) } - defer dst.Close() + defer handle.endOperation() - _, err = io.Copy(dst, src) - if err != nil { - os.Remove(backupFile) // 删除失败的备份文件 - return "", err - } - - return backupFile, nil -} - -// modifyRowInternal 修改整行 -func (mgr *CSVManager) modifyRowInternal(handle *CSVHandle, req *ModifyRequest, totalRows int64) (*ModifyResult, error) { - // 1. 创建临时文件 - tempFile, err := os.CreateTemp(handle.tempDir, "temp_*.csv") - if err != nil { - return nil, fmt.Errorf("创建临时文件失败: %v", err) - } - defer func() { - tempFile.Close() - os.Remove(tempFile.Name()) - }() - - // 2. 复制并修改文件 - modifiedCount, err := mgr.copyAndModifyRow(handle, tempFile, req) - if err != nil { - return nil, fmt.Errorf("复制修改文件失败: %v", err) - } - - // 3. 关闭临时文件以确保数据写入磁盘 - if err := tempFile.Close(); err != nil { - return nil, fmt.Errorf("关闭临时文件失败: %v", err) - } - - // 4. 原子替换原文件 - if err := mgr.atomicReplaceFile(tempFile.Name(), handle.Filename); err != nil { - // 尝试恢复备份 - mgr.restoreFromBackup(handle) - return nil, fmt.Errorf("替换文件失败: %v", err) - } - - // 5. 重新打开文件 handle.mu.Lock() - if handle.File != nil { - handle.File.Close() - handle.File = nil - handle.IsOpen = false + defer handle.mu.Unlock() + + if !handle.IsOpen { + return fmt.Errorf("文件未打开") } - if err := mgr.openFile(handle); err != nil { - handle.mu.Unlock() - return nil, fmt.Errorf("重新打开文件失败: %v", err) + // 检查是否有表头且表头长度是否匹配 + if handle.HasHeader && len(handle.Header) > 0 && len(row) != len(handle.Header) { + return fmt.Errorf("行数据列数(%d)与表头列数(%d)不匹配", len(row), len(handle.Header)) } - // 更新总行数 - handle.TotalRows = totalRows - handle.mu.Unlock() + // 移动到文件末尾 + if _, err := handle.File.Seek(0, 2); err != nil { + return fmt.Errorf("移动到文件末尾失败: %w", err) + } - return &ModifyResult{ - Success: true, - Message: fmt.Sprintf("修改第%d行成功", req.RowNumber), - RowsAffected: modifiedCount, - }, nil + // 写入行数据 + if err := handle.CSVWriter.Write(row); err != nil { + return fmt.Errorf("写入行失败: %w", err) + } + + // 更新行数统计 + handle.TotalRows++ + handle.cachedRowCount = handle.TotalRows + handle.rowCountCached = true + + return nil } -// copyAndModifyRow 复制文件并修改指定行 -func (mgr *CSVManager) copyAndModifyRow(handle *CSVHandle, dst *os.File, req *ModifyRequest) (int64, error) { - // 打开源文件 - src, err := os.Open(handle.Filename) +// WriteRows 批量写入多行数据到CSV文件 +func (mgr *CSVManager) WriteRows(handleID int64, rows [][]string) (int64, error) { + // 首先记录日志 + if err := mgr.logWriteRows(handleID, rows); err != nil { + // 日志记录失败不影响主流程，但可以打印警告 + fmt.Printf("警告：记录日志失败: %v\n", err) + } + + // 获取句柄 + handle, err := mgr.getHandle(handleID) if err != nil { - return 0, err + return -1, fmt.Errorf("WriteRows 获取句柄失败: %w", err) } - defer src.Close() + defer mgr.releaseHandle(handleID) - // 创建CSV读写器 - csvReader := csv.NewReader(src) - csvReader.Comma = handle.Delimiter - csvReader.LazyQuotes = true + // 检查句柄状态 + if !handle.beginOperation() { + return -1, fmt.Errorf("句柄已关闭或正在关闭: %d", handleID) + } + defer handle.endOperation() - csvWriter := csv.NewWriter(dst) - csvWriter.Comma = handle.Delimiter + handle.mu.Lock() + defer handle.mu.Unlock() - var rowCount int64 = 0 - var modifiedCount int64 = 0 - - for { - row, err := csvReader.Read() - if err == io.EOF { - break - } - if err != nil { - csvWriter.Flush() - return modifiedCount, fmt.Errorf("读取第%d行失败: %v", rowCount+1, err) - } - - rowCount++ - - // 检查是否是目标行（考虑表头） - targetRowNumber := req.RowNumber - if handle.HasHeader { - targetRowNumber = req.RowNumber + 1 - } - - if rowCount == targetRowNumber { - // 修改这一行 - row = req.NewRow - modifiedCount++ - } - - // 写入行 - if err := csvWriter.Write(row); err != nil { - csvWriter.Flush() - return modifiedCount, fmt.Errorf("写入第%d行失败: %v", rowCount, err) - } + if !handle.IsOpen { + return -1, fmt.Errorf("文件未打开") } - // 确保所有数据写入文件 - if err := csvWriter.Error(); err != nil { - return modifiedCount, fmt.Errorf("CSV写入错误: %v", err) - } - - csvWriter.Flush() - return modifiedCount, nil -} - -// modifyCellInternal 修改单元格 -func (mgr *CSVManager) modifyCellInternal(handle *CSVHandle, req *ModifyRequest, totalRows int64) (*ModifyResult, error) { - // 1. 读取目标行 - targetRow, err := mgr.readSpecificRow(handle, req.RowNumber) - if err != nil { - return nil, fmt.Errorf("读取目标行失败: %v", err) - } - - // 2. 修改单元格 - if req.ColumnIndex >= 0 && req.ColumnIndex < len(targetRow) { - targetRow[req.ColumnIndex] = req.NewCellValue - } else { - return nil, fmt.Errorf("列索引超出范围: %d", req.ColumnIndex) - } - - // 3. 构建修改请求（转换为整行修改） - rowReq := &ModifyRequest{ - HandleID: req.HandleID, - RowNumber: req.RowNumber, - ModifyType: ModifyRow, - NewRow: targetRow, - } - - // 4. 调用整行修改 - return mgr.modifyRowInternal(handle, rowReq, totalRows) -} - -// insertRowInternal 插入行 -func (mgr *CSVManager) insertRowInternal(handle *CSVHandle, req *ModifyRequest, totalRows int64) (*ModifyResult, error) { - // 创建临时文件 - tempFile, err := os.CreateTemp(handle.tempDir, "insert_*.csv") - if err != nil { - return nil, fmt.Errorf("创建临时文件失败: %v", err) - } - defer func() { - tempFile.Close() - os.Remove(tempFile.Name()) - }() - - // 打开源文件 - src, err := os.Open(handle.Filename) - if err != nil { - return nil, err - } - defer src.Close() - - // 创建CSV读写器 - csvReader := csv.NewReader(src) - csvReader.Comma = handle.Delimiter - - csvWriter := csv.NewWriter(tempFile) - csvWriter.Comma = handle.Delimiter - - var rowCount int64 = 0 - - for { - row, err := csvReader.Read() - if err == io.EOF { - break - } - if err != nil { - csvWriter.Flush() - return nil, fmt.Errorf("读取第%d行失败: %v", rowCount+1, err) - } - - rowCount++ - - // 写入当前行 - if err := csvWriter.Write(row); err != nil { - csvWriter.Flush() - return nil, fmt.Errorf("写入第%d行失败: %v", rowCount, err) - } - - // 检查是否到达插入位置（考虑表头） - insertPosition := req.RowNumber - if handle.HasHeader { - insertPosition = req.RowNumber + 1 - } - - if rowCount == insertPosition { - // 插入新行 - if err := csvWriter.Write(req.NewRow); err != nil { - csvWriter.Flush() - return nil, fmt.Errorf("插入新行失败: %v", err) + // 检查是否有表头 + if handle.HasHeader && len(handle.Header) > 0 { + // 验证每行的列数 + for i, row := range rows { + if len(row) != len(handle.Header) { + return -1, fmt.Errorf("第%d行列数(%d)与表头列数(%d)不匹配", i+1, len(row), len(handle.Header)) } } } - // 如果是最后一行之后插入 - adjustedTotalRows := totalRows - if handle.HasHeader { - adjustedTotalRows = totalRows - 1 + // 移动到文件末尾 + if _, err := handle.File.Seek(0, 2); err != nil { + return -1, fmt.Errorf("移动到文件末尾失败: %w", err) } - if req.RowNumber == adjustedTotalRows+1 { - // 在文件末尾插入 - if err := csvWriter.Write(req.NewRow); err != nil { - csvWriter.Flush() - return nil, fmt.Errorf("在末尾插入新行失败: %v", err) + // 批量写入行数据 + for _, row := range rows { + if err := handle.CSVWriter.Write(row); err != nil { + return -1, fmt.Errorf("写入行失败: %w", err) } } - // 确保所有数据写入 - csvWriter.Flush() - if err := csvWriter.Error(); err != nil { - return nil, fmt.Errorf("CSV写入错误: %v", err) - } + // 更新行数统计 + handle.TotalRows += int64(len(rows)) + handle.cachedRowCount = handle.TotalRows + handle.rowCountCached = true - // 关闭临时文件 - if err := tempFile.Close(); err != nil { - return nil, fmt.Errorf("关闭临时文件失败: %v", err) - } - - // 原子替换原文件 - handle.mu.Lock() - if handle.File != nil { - handle.File.Close() - handle.File = nil - handle.IsOpen = false - } - - if err := mgr.atomicReplaceFile(tempFile.Name(), handle.Filename); err != nil { - handle.mu.Unlock() - mgr.restoreFromBackup(handle) - return nil, fmt.Errorf("替换文件失败: %v", err) - } - - // 重新打开文件 - if err := mgr.openFile(handle); err != nil { - handle.mu.Unlock() - return nil, fmt.Errorf("重新打开文件失败: %v", err) - } - - // 更新总行数 - handle.TotalRows = totalRows + 1 - handle.mu.Unlock() - - return &ModifyResult{ - Success: true, - Message: fmt.Sprintf("在第%d行后插入成功", req.RowNumber), - RowsAffected: 1, - }, nil + return handle.TotalRows, nil } -// deleteRowInternal 删除行 -func (mgr *CSVManager) deleteRowInternal(handle *CSVHandle, req *ModifyRequest, totalRows int64) (*ModifyResult, error) { - // 创建临时文件 - tempFile, err := os.CreateTemp(handle.tempDir, "delete_*.csv") +// logWriteRows 记录行数据到日志文件 +func (mgr *CSVManager) logWriteRows(handleID int64, rows [][]string) error { + // 获取句柄信息（但不锁定，因为我们只是读取元数据） + handle, err := mgr.getHandle(handleID) if err != nil { - return nil, fmt.Errorf("创建临时文件失败: %v", err) + return fmt.Errorf("获取句柄信息失败: %w", err) } - defer func() { - tempFile.Close() - os.Remove(tempFile.Name()) - }() + defer mgr.releaseHandle(handleID) - // 打开源文件 - src, err := os.Open(handle.Filename) + // 创建日志目录 + logDir := filepath.Join(filepath.Dir("csv"), "logs") + if err := os.MkdirAll(logDir, 0755); err != nil { + return fmt.Errorf("创建日志目录失败: %w", err) + } + + // 生成日志文件名（基于CSV文件名和时间） + baseName := filepath.Base("csv") + logFileName := fmt.Sprintf("%s_%s_write.log", + baseName[:len(baseName)-len(filepath.Ext(baseName))], + time.Now().Format("20060102")) + logFilePath := filepath.Join(logDir, logFileName) + + // 打开或创建日志文件 + logFile, err := os.OpenFile(logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { - return nil, err + return fmt.Errorf("打开日志文件失败: %w", err) } - defer src.Close() + defer logFile.Close() - // 创建CSV读写器 - csvReader := csv.NewReader(src) - csvReader.Comma = handle.Delimiter - - csvWriter := csv.NewWriter(tempFile) - csvWriter.Comma = handle.Delimiter - - var rowCount int64 = 0 - var deletedCount int64 = 0 - - for { - row, err := csvReader.Read() - if err == io.EOF { - break - } - if err != nil { - csvWriter.Flush() - return nil, fmt.Errorf("读取第%d行失败: %v", rowCount+1, err) - } - - rowCount++ - - // 检查是否是要删除的行（考虑表头） - deleteRowNumber := req.RowNumber - if handle.HasHeader { - deleteRowNumber = req.RowNumber + 1 - } - - if rowCount == deleteRowNumber { - // 跳过这一行（不写入） - deletedCount++ - continue - } - - // 写入行 - if err := csvWriter.Write(row); err != nil { - csvWriter.Flush() - return nil, fmt.Errorf("写入第%d行失败: %v", rowCount, err) - } + // 创建日志条目 + logEntry := map[string]interface{}{ + "timestamp": time.Now().Format("2006-01-02 15:04:05.000"), + "handle_id": handleID, + "filename": "csv", + "operation": "write_rows", + "row_count": len(rows), + "total_rows": handle.TotalRows + int64(len(rows)), // 预计的总行数 + "data": rows, } - // 确保所有数据写入 - csvWriter.Flush() - if err := csvWriter.Error(); err != nil { - return nil, fmt.Errorf("CSV写入错误: %v", err) + // 序列化为JSON + logData, err := json.Marshal(logEntry) + if err != nil { + return fmt.Errorf("序列化日志数据失败: %w", err) } - // 关闭临时文件 - if err := tempFile.Close(); err != nil { - return nil, fmt.Errorf("关闭临时文件失败: %v", err) + // 写入日志（每行一个JSON对象） + if _, err := logFile.Write(append(logData, '\n')); err != nil { + return fmt.Errorf("写入日志文件失败: %w", err) } - // 原子替换原文件 - handle.mu.Lock() - if handle.File != nil { - handle.File.Close() - handle.File = nil - handle.IsOpen = false - } - - if err := mgr.atomicReplaceFile(tempFile.Name(), handle.Filename); err != nil { - handle.mu.Unlock() - mgr.restoreFromBackup(handle) - return nil, fmt.Errorf("替换文件失败: %v", err) - } - - // 重新打开文件 - if err := mgr.openFile(handle); err != nil { - handle.mu.Unlock() - return nil, fmt.Errorf("重新打开文件失败: %v", err) - } - - // 更新总行数 - handle.TotalRows = totalRows - deletedCount - handle.mu.Unlock() - - return &ModifyResult{ - Success: true, - Message: fmt.Sprintf("删除第%d行成功", req.RowNumber), - RowsAffected: deletedCount, - }, nil + return nil } -// readSpecificRow 读取指定行 -func (mgr *CSVManager) readSpecificRow(handle *CSVHandle, rowNumber int64) ([]string, error) { - // 保存当前位置 - currentPos, err := handle.File.Seek(0, io.SeekCurrent) +// Flush 将缓冲区数据写入文件 +func (mgr *CSVManager) Flush(handleID int64) error { + // 获取句柄 + handle, err := mgr.getHandle(handleID) if err != nil { - return nil, err + return fmt.Errorf("Flush 获取句柄失败: %w", err) + } + defer mgr.releaseHandle(handleID) + + // 检查句柄状态 + if !handle.beginOperation() { + return fmt.Errorf("句柄已关闭或正在关闭: %d", handleID) + } + defer handle.endOperation() + + handle.mu.Lock() + defer handle.mu.Unlock() + + if !handle.IsOpen { + return fmt.Errorf("文件未打开") + } + + if handle.CSVWriter == nil { + return fmt.Errorf("CSV写入器未初始化") + } + + // 刷新缓冲区 + handle.CSVWriter.Flush() + if handle.writeBuffer != nil { + if err := handle.writeBuffer.Flush(); err != nil { + return fmt.Errorf("刷新写入缓冲区失败: %w", err) + } + } + + return nil +} + +// AppendRow 追加行数据（自动Flush） +func (mgr *CSVManager) AppendRow(handleID int64, row []string) error { + if err := mgr.WriteRow(handleID, row); err != nil { + return err + } + return mgr.Flush(handleID) +} + +// AppendRows 批量追加行数据（自动Flush） +func (mgr *CSVManager) AppendRows(handleID int64, rows [][]string) (int64, error) { + totalRows, err := mgr.WriteRows(handleID, rows) + if err != nil { + return -1, err + } + err = mgr.Flush(handleID) + if err != nil { + return -1, err + } + return totalRows, nil +} + +// GetHeader 获取CSV文件表头 +func (mgr *CSVManager) GetHeader(handleID int64) ([]string, error) { + // 获取句柄 + handle, err := mgr.getHandle(handleID) + if err != nil { + return nil, fmt.Errorf("获取句柄失败: %w", err) + } + defer mgr.releaseHandle(handleID) + + // 检查句柄状态 + if !handle.beginOperation() { + return nil, fmt.Errorf("句柄已关闭或正在关闭: %d", handleID) + } + defer handle.endOperation() + + handle.mu.RLock() + defer handle.mu.RUnlock() + + if !handle.HasHeader { + return nil, fmt.Errorf("文件没有表头") + } + + return handle.Header, nil +} + +// ========================== 获取总行数 ========================== + +// GetTotalRows 快速获取总行数（如果已计算过则直接返回，否则重新计算） +func (mgr *CSVManager) GetTotalRows(handleID int64) (int64, error) { + // 获取句柄 + handle, err := mgr.getHandle(handleID) + if err != nil { + return 0, fmt.Errorf("获取句柄失败: %w", err) + } + defer mgr.releaseHandle(handleID) + + // 检查句柄状态 + if !handle.beginOperation() { + return 0, fmt.Errorf("句柄已关闭或正在关闭: %d", handleID) + } + defer handle.endOperation() + + handle.mu.RLock() + // 如果已缓存，直接返回缓存值 + if handle.rowCountCached { + count := handle.cachedRowCount + handle.mu.RUnlock() + return count, nil + } + handle.mu.RUnlock() + + // 否则计算总行数 + return handle.calculateTotalRows() +} + +// ========================== 修改指定行功能 ========================== + +// UpdateRow 修改指定行的数据 +func (mgr *CSVManager) UpdateRow(handleID int64, rowNumber int64, newData []string) error { + // 获取句柄 + handle, err := mgr.getHandle(handleID) + if err != nil { + return fmt.Errorf("获取句柄失败: %w", err) + } + defer mgr.releaseHandle(handleID) + + // 检查句柄状态 + if !handle.beginOperation() { + return fmt.Errorf("句柄已关闭或正在关闭: %d", handleID) + } + defer handle.endOperation() + + //handle.mu.Lock() + //defer handle.mu.Unlock() + + if !handle.IsOpen { + return fmt.Errorf("文件未打开") + } + + // 检查行号是否有效 + if rowNumber < 0 { + return fmt.Errorf("行号不能为负数: %d", rowNumber) + } + + // 获取总行数 + totalRows, err := handle.calculateTotalRows() + if err != nil { + return fmt.Errorf("获取总行数失败: %w", err) + } + + if rowNumber >= totalRows { + return fmt.Errorf("行号超出范围: %d, 总行数: %d", rowNumber, totalRows) + } + + // 检查新数据列数是否匹配 + if handle.HasHeader && len(handle.Header) > 0 { + if len(newData) != len(handle.Header) { + return fmt.Errorf("新数据列数(%d)与表头列数(%d)不匹配", len(newData), len(handle.Header)) + } + } + + // 读取整个文件到内存 + allRows, err := mgr.readAllRows(handle) + if err != nil { + return fmt.Errorf("读取文件失败: %w", err) + } + + // 修改指定行的数据 + if handle.HasHeader && len(allRows) > 0 { + // 如果有表头，第一行是表头，实际数据从第二行开始 + if rowNumber+1 >= int64(len(allRows)) { + return fmt.Errorf("行号超出范围: %d, 总数据行数: %d", rowNumber, len(allRows)-1) + } + allRows[rowNumber+1] = newData + } else { + if rowNumber >= int64(len(allRows)) { + return fmt.Errorf("行号超出范围: %d, 总行数: %d", rowNumber, len(allRows)) + } + allRows[rowNumber] = newData + } + + // 写回文件 + if err := mgr.writeAllRows(handle, allRows); err != nil { + return fmt.Errorf("写回文件失败: %w", err) + } + + // 更新缓存的行数 + handle.rowCountCached = false + + // 重新计算总行数 + if _, err := handle.calculateTotalRows(); err != nil { + return fmt.Errorf("重新计算总行数失败: %w", err) + } + + return nil +} + +// UpdateRows 批量修改多行数据 +func (mgr *CSVManager) UpdateRows(handleID int64, updates map[int64][]string) (int, error) { + // 获取句柄 + handle, err := mgr.getHandle(handleID) + if err != nil { + return 0, fmt.Errorf("获取句柄失败: %w", err) + } + defer mgr.releaseHandle(handleID) + + // 检查句柄状态 + if !handle.beginOperation() { + return 0, fmt.Errorf("句柄已关闭或正在关闭: %d", handleID) + } + defer handle.endOperation() + + handle.mu.Lock() + defer handle.mu.Unlock() + + if !handle.IsOpen { + return 0, fmt.Errorf("文件未打开") + } + + // 检查行号是否有效 + for rowNumber := range updates { + if rowNumber < 0 { + return 0, fmt.Errorf("行号不能为负数: %d", rowNumber) + } + } + + // 获取总行数 + totalRows, err := handle.calculateTotalRows() + if err != nil { + return 0, fmt.Errorf("获取总行数失败: %w", err) + } + + // 检查新数据列数是否匹配 + if handle.HasHeader && len(handle.Header) > 0 { + for rowNumber, newData := range updates { + if rowNumber >= totalRows { + return 0, fmt.Errorf("行号超出范围: %d, 总行数: %d", rowNumber, totalRows) + } + if len(newData) != len(handle.Header) { + return 0, fmt.Errorf("行%d的新数据列数(%d)与表头列数(%d)不匹配", rowNumber, len(newData), len(handle.Header)) + } + } + } + + // 读取整个文件到内存 + allRows, err := mgr.readAllRows(handle) + if err != nil { + return 0, fmt.Errorf("读取文件失败: %w", err) + } + + // 修改指定行的数据 + updatedCount := 0 + for rowNumber, newData := range updates { + if handle.HasHeader && len(allRows) > 0 { + // 如果有表头，第一行是表头，实际数据从第二行开始 + if rowNumber+1 >= int64(len(allRows)) { + return 0, fmt.Errorf("行号超出范围: %d, 总数据行数: %d", rowNumber, len(allRows)-1) + } + allRows[rowNumber+1] = newData + updatedCount++ + } else { + if rowNumber >= int64(len(allRows)) { + return 0, fmt.Errorf("行号超出范围: %d, 总行数: %d", rowNumber, len(allRows)) + } + allRows[rowNumber] = newData + updatedCount++ + } + } + + // 写回文件 + if err := mgr.writeAllRows(handle, allRows); err != nil { + return 0, fmt.Errorf("写回文件失败: %w", err) + } + + // 更新缓存的行数 + handle.rowCountCached = false + + // 重新计算总行数 + if _, err := handle.calculateTotalRows(); err != nil { + return 0, fmt.Errorf("重新计算总行数失败: %w", err) + } + + return updatedCount, nil +} + +// readAllRows 读取文件中的所有行 +func (mgr *CSVManager) readAllRows(handle *CSVHandle) ([][]string, error) { + // 保存当前文件位置 + currentPos, err := handle.File.Seek(0, 1) // 当前位置 + if err != nil { + return nil, fmt.Errorf("获取当前文件位置失败: %w", err) } - defer handle.File.Seek(currentPos, io.SeekStart) // 重置到文件开始 if _, err := handle.File.Seek(0, 0); err != nil { - return nil, err + return nil, fmt.Errorf("重置文件指针失败: %w", err) } - // 创建新的CSV阅读器 - reader := csv.NewReader(handle.File) + // 重置CSV阅读器 + reader := csv.NewReader(bufio.NewReader(handle.File)) reader.Comma = handle.Delimiter + reader.LazyQuotes = true + reader.TrimLeadingSpace = true - var rowCount int64 = 0 - targetRowNumber := rowNumber - if handle.HasHeader { - targetRowNumber = rowNumber + 1 - } - + // 读取所有行 + var allRows [][]string for { row, err := reader.Read() - if err == io.EOF { - break - } if err != nil { - return nil, err - } - - rowCount++ - if rowCount == targetRowNumber { - return row, nil + if err == io.EOF { + break + } + // 恢复文件位置 + handle.File.Seek(currentPos, 0) + return nil, fmt.Errorf("读取行失败: %w", err) } + allRows = append(allRows, row) } - return nil, fmt.Errorf("行号超出范围: %d", rowNumber) + // 恢复文件位置 + if _, err := handle.File.Seek(currentPos, 0); err != nil { + return nil, fmt.Errorf("恢复文件位置失败: %w", err) + } + + return allRows, nil } -// ========================== 工具方法 ========================== - -// atomicReplaceFile 原子替换文件 -func (mgr *CSVManager) atomicReplaceFile(src, dst string) error { - // 先尝试原子重命名（Unix/Linux上这是原子的） - if err := os.Rename(src, dst); err == nil { - return nil +// writeAllRows 将所有行写回文件 +func (mgr *CSVManager) writeAllRows(handle *CSVHandle, rows [][]string) error { + // 清空文件 + if err := handle.File.Truncate(0); err != nil { + return fmt.Errorf("清空文件失败: %w", err) } - // 如果重命名失败（比如Windows或跨分区），使用复制方式 - // 创建临时目标文件 - tmpDst := dst + ".tmp" - if err := mgr.copyFile(src, tmpDst); err != nil { - return err + // 移动到文件开头 + if _, err := handle.File.Seek(0, 0); err != nil { + return fmt.Errorf("移动文件指针失败: %w", err) } - // 重命名为目标文件 - if err := os.Rename(tmpDst, dst); err != nil { - os.Remove(tmpDst) - return err + // 写入所有行 + for _, row := range rows { + if err := handle.CSVWriter.Write(row); err != nil { + return fmt.Errorf("写入行失败: %w", err) + } + } + + // 刷新缓冲区 + handle.CSVWriter.Flush() + if handle.writeBuffer != nil { + if err := handle.writeBuffer.Flush(); err != nil { + return fmt.Errorf("刷新写入缓冲区失败: %w", err) + } } return nil } -// copyFile 复制文件 -func (mgr *CSVManager) copyFile(src, dst string) error { - source, err := os.Open(src) - if err != nil { - return err - } - defer source.Close() - - destination, err := os.Create(dst) - if err != nil { - return err - } - defer destination.Close() - - _, err = io.Copy(destination, source) - return err -} - -// restoreFromBackup 从备份恢复 -func (mgr *CSVManager) restoreFromBackup(handle *CSVHandle) error { - handle.mu.Lock() - defer handle.mu.Unlock() - - if len(handle.backupFiles) == 0 { - return fmt.Errorf("没有可用的备份文件") - } - - // 使用最新的备份 - latestBackup := handle.backupFiles[len(handle.backupFiles)-1] - - // 关闭当前文件 - if handle.File != nil { - handle.File.Close() - handle.File = nil - handle.IsOpen = false - } - - // 恢复备份 - if err := mgr.atomicReplaceFile(latestBackup, handle.Filename); err != nil { - return fmt.Errorf("恢复备份失败: %v", err) - } - - // 重新打开文件 - return mgr.openFile(handle) -} - -// ========================== 其他操作 ========================== - -// GetRow 获取指定行数据 +// GetRow 获取指定行的数据 func (mgr *CSVManager) GetRow(handleID int64, rowNumber int64) ([]string, error) { + // 获取句柄 handle, err := mgr.getHandle(handleID) if err != nil { - return nil, err + return nil, fmt.Errorf("获取句柄失败: %w", err) + } + defer mgr.releaseHandle(handleID) + + // 检查句柄状态 + if !handle.beginOperation() { + return nil, fmt.Errorf("句柄已关闭或正在关闭: %d", handleID) + } + defer handle.endOperation() + + //handle.mu.RLock() + //defer handle.mu.RUnlock() + + if !handle.IsOpen { + return nil, fmt.Errorf("文件未打开") } - // 获取文件级读锁 - fileLock := mgr.getFileLock(handle.Filename) - fileLock.RLock() - defer fileLock.RUnlock() + // 检查行号是否有效 + if rowNumber < 0 { + return nil, fmt.Errorf("行号不能为负数: %d", rowNumber) + } - return mgr.readSpecificRow(handle, rowNumber) -} - -// UndoLastModify 撤销最后一次修改 -func (mgr *CSVManager) UndoLastModify(handleID int64) (*ModifyResult, error) { - handle, err := mgr.getHandle(handleID) + // 获取总行数 + totalRows, err := handle.calculateTotalRows() if err != nil { - return nil, err + return nil, fmt.Errorf("获取总行数失败: %w", err) } - // 获取文件级写锁 - fileLock := mgr.getFileLock(handle.Filename) - fileLock.Lock() - defer fileLock.Unlock() - - handle.mu.Lock() - defer handle.mu.Unlock() - - if len(handle.backupFiles) == 0 { - return &ModifyResult{ - Success: false, - Message: "没有可撤销的修改", - }, nil + if rowNumber >= totalRows { + return nil, fmt.Errorf("行号超出范围: %d, 总行数: %d", rowNumber, totalRows) } - // 关闭当前文件 - if handle.File != nil { - handle.File.Close() - handle.File = nil - handle.IsOpen = false - } - - // 获取最新的备份 - latestBackup := handle.backupFiles[len(handle.backupFiles)-1] - - // 恢复文件 - if err := mgr.atomicReplaceFile(latestBackup, handle.Filename); err != nil { - return &ModifyResult{ - Success: false, - Message: fmt.Sprintf("恢复文件失败: %v", err), - }, err - } - - // 重新打开文件 - if err := mgr.openFile(handle); err != nil { - return &ModifyResult{ - Success: false, - Message: fmt.Sprintf("重新打开文件失败: %v", err), - }, err - } - - // 移除已使用的备份 - handle.backupFiles = handle.backupFiles[:len(handle.backupFiles)-1] - - // 重新计算总行数 - totalRows, _ := mgr.calculateTotalRowsWithLock(handle) - fmt.Printf("总行数: %d\n", totalRows) - - return &ModifyResult{ - Success: true, - Message: "撤销成功", - RowsAffected: 1, - BackupFile: latestBackup, - }, nil -} - -// CleanupBackups 清理备份文件 -func (mgr *CSVManager) CleanupBackups(handleID int64) error { - handle, err := mgr.getHandle(handleID) + // 保存当前文件位置 + currentPos, err := handle.File.Seek(0, 1) // 当前位置 if err != nil { - return err + return nil, fmt.Errorf("获取当前文件位置失败: %w", err) } - handle.mu.Lock() - defer handle.mu.Unlock() - - // 清理所有备份文件 - for _, backupFile := range handle.backupFiles { - os.Remove(backupFile) - } - handle.backupFiles = nil - - // 清理临时目录 - if handle.tempDir != "" { - os.RemoveAll(handle.tempDir) - handle.tempDir = "" + // 重置到文件开始 + if _, err := handle.File.Seek(0, 0); err != nil { + return nil, fmt.Errorf("重置文件指针失败: %w", err) } - return nil -} + // 重置CSV阅读器 + reader := csv.NewReader(bufio.NewReader(handle.File)) + reader.Comma = handle.Delimiter + reader.LazyQuotes = true + reader.TrimLeadingSpace = true -// UpdateCSVRowSafe 修改csv文件行数据 -func (mgr *CSVManager) UpdateCSVRowSafe(handleID int64, rowNum int, newRow []string) error { - getHandle, err := mgr.getHandle(handleID) - if err != nil { - return fmt.Errorf("获取句柄信息失败: %v", err) - } - - // 1. 创建临时文件 - tempDir := filepath.Dir(getHandle.Filename) - if tempDir == "" { - tempDir = "." - } - - tempFile, err := os.CreateTemp(tempDir, "temp_*.csv") - if err != nil { - return fmt.Errorf("创建临时文件失败: %v", err) - } - tempFileName := tempFile.Name() - - // 确保临时文件被关闭和清理 - defer func() { - tempFile.Close() - // 如果临时文件还存在（替换失败），则清理它 - if _, err := os.Stat(tempFileName); err == nil { - os.Remove(tempFileName) + // 定位到指定行 + var currentRow int64 = 0 + var targetRow []string + for { + row, err := reader.Read() + if err != nil { + if err == io.EOF { + break + } + // 恢复文件位置 + handle.File.Seek(currentPos, 0) + return nil, fmt.Errorf("读取行失败: %w", err) } + + // 判断是否到达目标行 + if handle.HasHeader { + // 如果有表头，跳过第一行 + if currentRow == rowNumber+1 { + targetRow = row + break + } + } else { + // 如果没有表头，直接匹配 + if currentRow == rowNumber { + targetRow = row + break + } + } + currentRow++ + } + + // 恢复文件位置 + if _, err := handle.File.Seek(currentPos, 0); err != nil { + return nil, fmt.Errorf("恢复文件位置失败: %w", err) + } + + if targetRow == nil { + return nil, fmt.Errorf("未找到行号 %d", rowNumber) + } + + return targetRow, nil +} + +// ========================== 文件合并功能 ========================== + +// MergeCSVFiles 合并两个CSV文件 +func (mgr *CSVManager) MergeCSVFiles(srcHandleID, dstHandleID int64, options MergeOptions) (int64, error) { + // 获取源文件句柄 + srcHandle, err := mgr.getHandle(srcHandleID) + if err != nil { + return 0, fmt.Errorf("获取源文件句柄失败: %w", err) + } + defer mgr.releaseHandle(srcHandleID) + + // 获取目标文件句柄 + dstHandle, err := mgr.getHandle(dstHandleID) + if err != nil { + return 0, fmt.Errorf("获取目标文件句柄失败: %w", err) + } + defer mgr.releaseHandle(dstHandleID) + + // 检查句柄状态 + if !srcHandle.beginOperation() { + return 0, fmt.Errorf("源文件句柄已关闭或正在关闭: %d", srcHandleID) + } + defer srcHandle.endOperation() + + if !dstHandle.beginOperation() { + return 0, fmt.Errorf("目标文件句柄已关闭或正在关闭: %d", dstHandleID) + } + defer dstHandle.endOperation() + + srcHandle.mu.RLock() + dstHandle.mu.Lock() + defer func() { + srcHandle.mu.RUnlock() + dstHandle.mu.Unlock() }() - // 2. 读取原始文件 - sourceFile, err := os.Open(getHandle.Filename) + if !srcHandle.IsOpen { + return 0, fmt.Errorf("源文件未打开") + } + if !dstHandle.IsOpen { + return 0, fmt.Errorf("目标文件未打开") + } + + // 1. 检查表头兼容性 + if err := mgr.checkHeaderCompatibility(srcHandle, dstHandle, options); err != nil { + return 0, fmt.Errorf("表头不兼容: %w", err) + } + + // 2. 读取源文件的所有行 + srcRows, err := mgr.readAllRows(srcHandle) if err != nil { - return fmt.Errorf("打开源文件失败: %v", err) + return 0, fmt.Errorf("读取源文件失败: %w", err) } - defer sourceFile.Close() - // 3. 读取并处理数据 - reader := csv.NewReader(sourceFile) - allRows, err := reader.ReadAll() + // 3. 合并数据 + mergedRows, err := mgr.mergeDataRows(srcRows, srcHandle, dstHandle, options) if err != nil { - return fmt.Errorf("读取CSV失败: %v", err) + return 0, fmt.Errorf("合并数据失败: %w", err) } - // 4. 验证并更新 - if rowNum < 1 || rowNum > len(allRows) { - return fmt.Errorf("行号 %d 超出范围 (1-%d)", rowNum, len(allRows)) + // 4. 写入目标文件 + if options.AppendMode { + // 追加模式：写入到文件末尾 + if err := mgr.appendRowsToFile(dstHandle, mergedRows); err != nil { + return 0, fmt.Errorf("追加数据失败: %w", err) + } + } else { + // 覆盖模式：先读取目标文件的所有行，然后合并后写回 + dstRows, err := mgr.readAllRows(dstHandle) + if err != nil { + return 0, fmt.Errorf("读取目标文件失败: %w", err) + } + + // 如果是第一次合并且目标文件为空，可能需要保留表头 + allRows := mgr.mergeAllRows(dstRows, mergedRows, dstHandle, srcHandle, options) + if err := mgr.writeAllRows(dstHandle, allRows); err != nil { + return 0, fmt.Errorf("写入合并数据失败: %w", err) + } } - // 显示修改前后的对比 - oldRow := allRows[rowNum-1] - fmt.Printf("修改前第 %d 行: %v\n", rowNum, oldRow) - fmt.Printf("修改后第 %d 行: %v\n", rowNum, newRow) - - allRows[rowNum-1] = newRow - - // 5. 写入临时文件 - writer := csv.NewWriter(tempFile) - if err := writer.WriteAll(allRows); err != nil { - return fmt.Errorf("写入临时文件失败: %v", err) - } - writer.Flush() - - if err := writer.Error(); err != nil { - return fmt.Errorf("刷新写入失败: %v", err) + // 5. 更新目标文件的行数缓存 + dstHandle.rowCountCached = false + newTotalRows, err := dstHandle.calculateTotalRows() + if err != nil { + return 0, fmt.Errorf("重新计算行数失败: %w", err) } - // 6. 确保数据写入磁盘 - if err := tempFile.Sync(); err != nil { - return fmt.Errorf("同步文件失败: %v", err) - } - tempFile.Close() - - // 7. 使用复制而不是重命名（解决Windows文件占用问题） - if err := copyFile(tempFileName, getHandle.Filename); err != nil { - return fmt.Errorf("复制文件失败: %v", err) - } - - fmt.Printf("✅ 成功更新第 %d 行，文件已直接更新\n", rowNum) - return nil + return newTotalRows, nil } -// copyFile 复制文件内容 -func copyFile(src, dst string) error { - // 打开源文件 - source, err := os.Open(src) - if err != nil { - return err - } - defer source.Close() +// 检查表头兼容性 +func (mgr *CSVManager) checkHeaderCompatibility(srcHandle, dstHandle *CSVHandle, options MergeOptions) error { + // 如果两个文件都有表头 + if srcHandle.HasHeader && dstHandle.HasHeader { + srcHeader := srcHandle.Header + dstHeader := dstHandle.Header - // 创建目标文件 - destination, err := os.Create(dst) - if err != nil { - return err - } - defer destination.Close() + // 如果指定了列映射，跳过表头检查 + if len(options.ColumnMapping) > 0 { + return nil + } - // 复制内容 - _, err = io.Copy(destination, source) - if err != nil { - return err - } + // 检查列数是否相同 + if len(srcHeader) != len(dstHeader) && !options.SkipHeader { + return fmt.Errorf("源文件列数(%d)与目标文件列数(%d)不一致", len(srcHeader), len(dstHeader)) + } - // 确保数据写入磁盘 - err = destination.Sync() - if err != nil { - return err + // 检查列名是否相同（可选） + for i := 0; i < min(len(srcHeader), len(dstHeader)); i++ { + if srcHeader[i] != dstHeader[i] { + fmt.Printf("警告: 第%d列列名不同: 源='%s', 目标='%s'\n", i+1, srcHeader[i], dstHeader[i]) + } + } + } else if srcHandle.HasHeader != dstHandle.HasHeader { + // 一个有表头，一个没有表头 + if !options.SkipHeader { + return fmt.Errorf("源文件与目标文件的表头设置不一致") + } } return nil } -// CSV响应结构体 -type CSVResponse struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` - Data interface{} `json:"data,omitempty"` +// 合并数据行 +func (mgr *CSVManager) mergeDataRows(srcRows [][]string, srcHandle, dstHandle *CSVHandle, options MergeOptions) ([][]string, error) { + var result [][]string + + startIndex := 0 + // 如果源文件有表头且需要跳过表头 + if srcHandle.HasHeader && options.SkipHeader { + if len(srcRows) > 0 { + startIndex = 1 + } + } + + // 处理列映射 + for i := startIndex; i < len(srcRows); i++ { + srcRow := srcRows[i] + transformedRow, err := mgr.transformRow(srcRow, srcHandle, dstHandle, options) + if err != nil { + return nil, fmt.Errorf("转换第%d行数据失败: %w", i+1, err) + } + + // 跳过空行 + if transformedRow == nil { + continue + } + + // 检查重复行 + if options.SkipDuplicates && mgr.isDuplicateRow(result, transformedRow) { + continue + } + + result = append(result, transformedRow) + } + + return result, nil } -// ===================== C 函数导入 ================== -// OpenCSVFile 打开/创建CSV文件 +// 转换行数据 +func (mgr *CSVManager) transformRow(srcRow []string, srcHandle, dstHandle *CSVHandle, options MergeOptions) ([]string, error) { + // 如果有自定义转换函数，优先使用 + if options.TransformFunc != nil { + return options.TransformFunc(srcRow) + } + + // 如果有列映射，按照映射转换 + if len(options.ColumnMapping) > 0 { + // 确定目标行的最大列数 + maxCol := 0 + for _, dstIndex := range options.ColumnMapping { + if dstIndex > maxCol { + maxCol = dstIndex + } + } + + // 创建目标行 + dstRow := make([]string, maxCol+1) + for srcIndex, dstIndex := range options.ColumnMapping { + if srcIndex < len(srcRow) && dstIndex >= 0 && dstIndex < len(dstRow) { + dstRow[dstIndex] = srcRow[srcIndex] + } + } + return dstRow, nil + } + + // 如果没有列映射，直接复制 + dstRow := make([]string, len(srcRow)) + copy(dstRow, srcRow) + + // 如果目标文件有表头且列数不同，进行调整 + if dstHandle.HasHeader && len(dstRow) != len(dstHandle.Header) { + if len(dstRow) < len(dstHandle.Header) { + // 源列数少，用空字符串填充 + for i := len(dstRow); i < len(dstHandle.Header); i++ { + dstRow = append(dstRow, "") + } + } else { + // 源列数多，截断 + dstRow = dstRow[:len(dstHandle.Header)] + } + } + + return dstRow, nil +} + +// 检查是否为重复行 +func (mgr *CSVManager) isDuplicateRow(rows [][]string, newRow []string) bool { + for _, row := range rows { + if mgr.compareRows(row, newRow) { + return true + } + } + return false +} + +// 比较两行是否相同 +func (mgr *CSVManager) compareRows(row1, row2 []string) bool { + if len(row1) != len(row2) { + return false + } + for i := range row1 { + if row1[i] != row2[i] { + return false + } + } + return true +} + +// 追加行到文件末尾 +func (mgr *CSVManager) appendRowsToFile(handle *CSVHandle, rows [][]string) error { + // 移动到文件末尾 + if _, err := handle.File.Seek(0, 2); err != nil { + return fmt.Errorf("移动到文件末尾失败: %w", err) + } + + // 批量写入行数据 + for _, row := range rows { + if err := handle.CSVWriter.Write(row); err != nil { + return fmt.Errorf("写入行失败: %w", err) + } + } + + // 刷新缓冲区 + handle.CSVWriter.Flush() + if handle.writeBuffer != nil { + if err := handle.writeBuffer.Flush(); err != nil { + return fmt.Errorf("刷新写入缓冲区失败: %w", err) + } + } + + // 更新行数统计 + handle.TotalRows += int64(len(rows)) + handle.cachedRowCount = handle.TotalRows + handle.rowCountCached = true + + return nil +} + +// 合并所有行（用于覆盖模式） +func (mgr *CSVManager) mergeAllRows(dstRows, srcRows [][]string, dstHandle, srcHandle *CSVHandle, options MergeOptions) [][]string { + var result [][]string + + // 处理表头 + if dstHandle.HasHeader && len(dstRows) > 0 { + // 保留目标文件的表头 + result = append(result, dstRows[0]) + dstRows = dstRows[1:] + } + + // 根据冲突解决策略合并数据 + switch options.ConflictResolution { + case Overwrite: + // 覆盖模式：源数据完全替换目标数据 + result = append(result, srcRows...) + case Skip: + // 跳过模式：只保留目标数据中不冲突的行 + result = append(result, dstRows...) + for _, srcRow := range srcRows { + if !mgr.isDuplicateRow(result, srcRow) { + result = append(result, srcRow) + } + } + case KeepBoth: + // 保留两者：先写目标数据，再写源数据 + result = append(result, dstRows...) + result = append(result, srcRows...) + case UseSource: + // 使用源数据：源数据优先级更高 + result = append(result, srcRows...) + case UseTarget: + // 使用目标数据：目标数据优先级更高 + result = append(result, dstRows...) + default: + // 默认：追加模式 + result = append(result, dstRows...) + result = append(result, srcRows...) + } + + return result +} + +// MergeCSVFilesSimple 简单合并CSV文件（默认选项） +func (mgr *CSVManager) MergeCSVFilesSimple(srcHandleID, dstHandleID int64, appendMode bool) (int64, error) { + options := MergeOptions{ + AppendMode: appendMode, + SkipHeader: true, // 跳过源文件表头 + SkipDuplicates: false, // 不跳过重复行 + ConflictResolution: KeepBoth, // 保留两者 + } + return mgr.MergeCSVFiles(srcHandleID, dstHandleID, options) +} + +// ========================== C 导出函数 ========================== + +// OpenCSVFile 打开CSV文件 // //export OpenCSVFile func OpenCSVFile(filename *C.char, delimiter C.char, hasHeader C.int) *C.char { - goFilename := C.GoString(filename) - goDelimiter := rune(delimiter) - var hasHeaderBool bool - if hasHeader != 0 { - hasHeaderBool = true - } - hasHeaderBool = false - handle, err := GetManager().OpenCSVFile(goFilename, goDelimiter, hasHeaderBool) + filenameStr := C.GoString(filename) + delimiterStr := rune(delimiter) + handleID, err := GetManager().OpenCSVFile(filenameStr, delimiterStr, hasHeader != 0) response := struct { - Handle int64 `json:"handle"` + HandleID int64 `json:"handleID"` }{ - Handle: handle, + HandleID: handleID, } var csvResponse CSVResponse if err != nil { @@ -1676,153 +1868,222 @@ func OpenCSVFile(filename *C.char, delimiter C.char, hasHeader C.int) *C.char { Data: response, } } - // 转换为JSON字符串 - jsonData, err := json.Marshal(csvResponse) - if err != nil { - // 如果JSON序列化失败，返回错误信息 - csvResponse = CSVResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", err), - } - errorJson, _ := json.Marshal(csvResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) } -// UpdateCSVRowSafe 修改csv文件行数据 +// WriteHeader 写入表头 // -//export UpdateCSVRowSafe -func UpdateCSVRowSafe(handleID C.longlong, rowNum C.int, newRow *C.char) *C.char { - goHandle := int64(handleID) - goRowNum := int(rowNum) - goNewRow := C.GoString(newRow) - var row []string +//export WriteHeader +func WriteHeader(handleID C.int, header *C.char) *C.char { var csvResponse CSVResponse - // 解析JSON - err := json.Unmarshal([]byte(goNewRow), &row) + goHandleID := int64(handleID) + goHeader := C.GoString(header) + var data []string + err := json.Unmarshal([]byte(goHeader), &data) if err != nil { csvResponse = CSVResponse{ Success: false, - Message: fmt.Sprintf("解析JSON失败: %v", err), + Message: fmt.Sprintf("header JSON解析失败: %v", err), } - errorJson, _ := json.Marshal(csvResponse) - return C.CString(string(errorJson)) + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) } - // 修改csv行数据 - err = GetManager().UpdateCSVRowSafe(goHandle, goRowNum, row) - if err != nil { + errs := GetManager().WriteHeader(goHandleID, data) + if errs != nil { csvResponse = CSVResponse{ Success: false, - Message: fmt.Sprintf(err.Error()), + Message: errs.Error(), } - errorJson, _ := json.Marshal(csvResponse) - return C.CString(string(errorJson)) } else { csvResponse = CSVResponse{ Success: true, } } - // 转换为JSON字符串 - jsonData, err := json.Marshal(csvResponse) + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) +} + +// WriteRows 写入/覆盖行数据 +// +//export WriteRows +func WriteRows(handleID C.int, rowsData *C.char) *C.char { + var csvResponse CSVResponse + goHandleID := int64(handleID) + goRowsData := C.GoString(rowsData) + // json解析 + var data [][]string + err := json.Unmarshal([]byte(goRowsData), &data) if err != nil { - // 如果JSON序列化失败，返回错误信息 csvResponse = CSVResponse{ Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", err), + Message: fmt.Sprintf("rowsData JSON解析失败: %v", err), } - errorJson, _ := json.Marshal(csvResponse) - return C.CString(string(errorJson)) + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) } - return C.CString(string(jsonData)) + // 写入数据 + rows, err := GetManager().AppendRows(goHandleID, data) + response := struct { + TotalRows int64 `json:"totalRows"` + }{ + TotalRows: rows, + } + if err != nil { + csvResponse = CSVResponse{ + Success: false, + Message: err.Error(), + } + } else { + csvResponse = CSVResponse{ + Success: true, + Data: response, + } + } + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) +} + +// UpdateRow C导出函数 - 修改指定行数据 +// +//export UpdateRow +func UpdateRow(handleID C.int, rowNumber C.int, rowData *C.char) *C.char { + var csvResponse CSVResponse + goHandleID := int64(handleID) + goRowNumber := int64(rowNumber) + goRowData := C.GoString(rowData) + + // JSON解析行数据 + var data []string + err := json.Unmarshal([]byte(goRowData), &data) + if err != nil { + csvResponse = CSVResponse{ + Success: false, + Message: fmt.Sprintf("rowsData JSON解析失败: %v", err), + } + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) + } + // 修改行数据 + err = GetManager().UpdateRow(goHandleID, goRowNumber, data) + if err != nil { + csvResponse = CSVResponse{ + Success: false, + Message: err.Error(), + } + } else { + csvResponse = CSVResponse{ + Success: true, + Message: fmt.Sprintf("成功修改第%d行数据", goRowNumber), + } + } + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) +} + +// GetRow C导出函数 - 获取指定行数据 +// +//export GetRow +func GetRow(handleID C.int, rowNumber C.int) *C.char { + goHandleID := int64(handleID) + goRowNumber := int64(rowNumber) + + // 获取行数据 + rowData, err := GetManager().GetRow(goHandleID, goRowNumber) + var csvResponse CSVResponse + if err != nil { + csvResponse = CSVResponse{ + Success: false, + Message: err.Error(), + } + } else { + csvResponse = CSVResponse{ + Success: true, + Data: rowData, + } + } + + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) +} + +// MergeCSVFilesSimple 合并两个csv文件 +// +//export MergeCSVFilesSimple +func MergeCSVFilesSimple(srcHandleID, dstHandleID, appendMode C.int) *C.char { + var csvResponse CSVResponse + goSrcHandleID := int64(srcHandleID) + goDstHandleID := int64(dstHandleID) + goAppendMode := int(appendMode) + rows, err := GetManager().MergeCSVFilesSimple(goSrcHandleID, goDstHandleID, goAppendMode == 0) + response := struct { + TotalRows int64 `json:"totalRows"` + }{ + TotalRows: rows, + } + if err != nil { + csvResponse = CSVResponse{ + Success: false, + Message: err.Error(), + } + } else { + csvResponse = CSVResponse{ + Success: true, + Data: response, + } + } + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) +} + +// CloseHandles 关闭指定句柄（使用优雅关闭） +// +//export CloseHandles +func CloseHandles(handleID C.int) *C.char { + goHandleID := int64(handleID) + + // 使用优雅关闭 + err := GetManager().CloseHandle(goHandleID) + + var csvResponse CSVResponse + if err != nil { + csvResponse = CSVResponse{ + Success: false, + Message: err.Error(), + } + } else { + csvResponse = CSVResponse{ + Success: true, + Message: fmt.Sprintf("成功关闭句柄 %d", goHandleID), + } + } + + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) +} + +// CloseAllHandles 关闭所有句柄 +// +//export CloseAllHandles +func CloseAllHandles() *C.char { + GetManager().closeAllHandles() + + csvResponse := CSVResponse{ + Success: true, + Message: "成功关闭所有句柄", + } + + csvResponseStr, _ := json.Marshal(csvResponse) + return C.CString(string(csvResponseStr)) +} + +// FreeCString 释放C字符串内存 +// +//export FreeCString +func FreeCString(str *C.char) { + C.free(unsafe.Pointer(str)) } // 主函数 func main() { - fmt.Println("=== CSV句柄管理器测试 ===") - - filename := "csv/taskLog1.csv" - fmt.Printf("1. 创建测试文件: %s\n", filename) - - // 2. 打开CSV文件 - handleID, err := GetManager().OpenCSVFile(filename, ',', true) - if err != nil { - fmt.Printf("打开文件失败: %v\n", err) - return - } - defer GetManager().closeHandle(handleID) - - newStr := []string{"9787115524539", "20.00", "2", "上传成功", ""} - - row, err := GetManager().modifyRow(handleID, 3, newStr) - if err != nil { - fmt.Println(err) - } - fmt.Println(row) - - //fmt.Printf("2. 打开文件成功，句柄ID: %d\n", handleID) - // - //// 3. 获取句柄信息 - //handle, err := GetCSVHandle(handleID) - //if err != nil { - // fmt.Printf("获取句柄失败: %v\n", err) - // return - //} - // - //info := handle.GetInfo() - //fmt.Printf("3. 文件信息:\n") - //fmt.Printf(" - 文件名: %s\n", info["filename"]) - //fmt.Printf(" - 总行数: %v\n", info["total_rows"]) - //fmt.Printf(" - 表头: %v\n", info["header"]) - // - //// 准备新的行数据 - //newData := []string{"9787115524539", "20.00", "20", "上传成功", ""} - //err = UpdateCSVRowSafes(handleID, 6, newData) - //if err != nil { - // fmt.Printf("错误: %v\n", err) - //} else { - // fmt.Println("CSV文件更新成功！") - //} - // - ////4. 读取数据 - //fmt.Println("4. 读取测试:") - //row3, err := GetCSVRow(handleID, 100000) - //if err != nil { - // fmt.Printf(" 读取第100000行失败: %v\n", err) - //} else { - // fmt.Printf(" 第100000行数据: %v\n", row3) - //} - // - //// 5. 修改测试 - //fmt.Println("5. 修改测试:") - //newRow := []string{"9787115524539", "20.00", "1", "上传成功", ""} - //result, err := ModifyCSVRow(handleID, 3, newRow) - //if err != nil { - // fmt.Printf(" 修改失败: %v\n", err) - //} else { - // fmt.Printf(" 修改结果: %s (影响行数: %d)\n", result.Message, result.RowsAffected) - // - // // 验证修改 - // modifiedRow, _ := GetCSVRow(handleID, 3) - // fmt.Printf(" 修改后的第3行: %v\n", modifiedRow) - //} - // - //// 6. 单元格修改测试 - //fmt.Println("6. 单元格修改测试:") - //cellResult, err := ModifyCSVCell(handleID, 5, 1, "35") - //if err != nil { - // fmt.Printf(" 单元格修改失败: %v\n", err) - //} else { - // fmt.Printf(" 单元格修改结果: %s\n", cellResult.Message) - // - // // 验证修改 - // cellModifiedRow, _ := GetCSVRow(handleID, 5) - // fmt.Printf(" 修改后的第5行: %v\n", cellModifiedRow) - //} - // - //// 7. 并发测试 - //fmt.Println("7. 并发测试:") - //TestConcurrentOperations(handleID) - // - //fmt.Println("=== 测试完成 ===") } diff --git a/csv/csvDll.go b/csv/csvDll.go new file mode 100644 index 0000000..17b4599 --- /dev/null +++ b/csv/csvDll.go @@ -0,0 +1,843 @@ +package main + +import ( + "context" + "fmt" + "github.com/redis/go-redis/v9" + "log" +) + +// +//var ( +// headerOnce sync.Once +// fileHandle int64 = -1 +//) +// +//// testwrite 函数 - 使用sync.Once确保表头只写入一次 +//func testwrite(mgr *CSVManager, batchIndex int, wg *sync.WaitGroup) { +// defer wg.Done() +// +// fmt.Printf("协程 %d 开始执行...\n", batchIndex) +// +// // 1. 打开CSV文件 - 所有协程操作同一个文件 +// filename := "csv/test_concurrent.csv" +// delimiter := ',' +// hasHeader := true +// +// handleID, err := mgr.OpenCSVFile(filename, delimiter, hasHeader) +// if err != nil { +// fmt.Printf("协程 %d 打开文件失败: %v\n", batchIndex, err) +// return +// } +// +// fmt.Printf("协程 %d 成功打开文件，句柄ID: %d\n", batchIndex, handleID) +// +// // 2. 使用sync.Once确保表头只写入一次 +// headerOnce.Do(func() { +// header := []string{ +// "ID", "Name", +// } +// +// err = mgr.WriteHeader(handleID, header) +// if err != nil { +// fmt.Printf("写入表头失败: %v\n", err) +// } else { +// fmt.Println("表头写入成功（仅第一次）") +// } +// }) +// +// // 3. 生成并写入1000条数据 +// batchSize := 1000 +// //testData := generateTestData(batchIndex, batchSize) +// +// var rows [][]string +// +// for i := 0; i < batchSize; i++ { +// //rowIndex := batchIndex*batchSize + i + 1 +// row := []string{ +// fmt.Sprintf("%d", i), +// fmt.Sprintf("用户_%d", i), +// } +// rows = append(rows, row) +// // 批量写入数据 +// _, err = mgr.AppendRows(handleID, rows) +// if err != nil { +// fmt.Printf("协程 %d 写入数据失败: %v\n", batchIndex, err) +// mgr.CloseHandle(handleID) +// return +// } +// +// fmt.Printf("协程 %d 成功写入 %d 条数据\n", batchIndex, i) +// rows = nil +// } +// +// //return rows +// +// // 4. 关闭句柄 +// err = mgr.CloseHandle(handleID) +// if err != nil { +// fmt.Printf("协程 %d 关闭句柄失败: %v\n", batchIndex, err) +// } else { +// fmt.Printf("协程 %d 关闭句柄成功: %d\n", batchIndex, handleID) +// } +// +// fmt.Printf("协程 %d 执行完成\n", batchIndex) +//} +// +//// 生成测试数据 +//func generateTestData(batchIndex, batchSize int) [][]string { +// var rows [][]string +// +// for i := 0; i < batchSize; i++ { +// rowIndex := batchIndex*batchSize + i + 1 +// row := []string{ +// fmt.Sprintf("%d", rowIndex), +// fmt.Sprintf("用户_%d", rowIndex), +// fmt.Sprintf("user%d@example.com", rowIndex), +// fmt.Sprintf("%d", 20+(rowIndex%50)), +// fmt.Sprintf("地址_%d", rowIndex), +// fmt.Sprintf("13800138%03d", rowIndex%1000), +// []string{"active", "inactive", "pending"}[rowIndex%3], +// fmt.Sprintf("%d", 50+(rowIndex%50)), +// []string{"A", "B", "C", "D"}[rowIndex%4], +// fmt.Sprintf("tag%d,tag%d", rowIndex, rowIndex+1), +// } +// rows = append(rows, row) +// } +// +// return rows +//} +// +//func main() { +// fmt.Println("开始多协程CSV写入测试...") +// fmt.Println("所有协程同时操作同一个文件：test_concurrent.csv") +// fmt.Println("每个协程执行：打开文件 → 写入表头（仅第一次）→ 写入1000条数据 → 关闭文件") +// fmt.Println("启动10个协程，总共写入10000条数据") +// +// // 获取CSV管理器 +// mgr := GetManager() +// +// // 创建等待组 +// var wg sync.WaitGroup +// +// // 启动10个协程，每个协程执行完整的操作流程 +// totalGoroutines := 4 +// +// for i := 0; i < totalGoroutines; i++ { +// wg.Add(1) +// go testwrite(mgr, i, &wg) +// } +// +// // 等待所有协程完成 +// wg.Wait() +// +// fmt.Println("\n所有协程执行完成！") +// fmt.Println("测试文件：test_concurrent.csv") +//} + +//import "C" +//import ( +// "bytes" +// "encoding/csv" +// "encoding/json" +// "fmt" +// "io" +// "net/http" +// "os" +// "path/filepath" +// "syscall" +// "unsafe" +//) +// +//// CsvDLL 代理DLL结构 +//type csvDLL struct { +// dll *syscall.DLL +// openCSVFile *syscall.Proc // 打开/创建CSV文件 +// readRows *syscall.Proc // 读取多行数据 +// writeRows *syscall.Proc // 写入/覆盖行数据 +// appendRows *syscall.Proc // 追加行数据 +// getRowCount *syscall.Proc // 获取总行数 +// findRows *syscall.Proc // 搜索行 +// closeCSVFile *syscall.Proc // 关闭CSV文件 +// mergeCSVFiles *syscall.Proc // 合并多个CSV文件 +// getError *syscall.Proc // 获取错误信息 +// updateCSVRowSafe *syscall.Proc +// createOpenCSVFile *syscall.Proc +// freeCString *syscall.Proc // 释放C字符串 +//} +// +//// 初始化csvDLL +//func InitCsvDLL() (*csvDLL, error) { +// dllPath := filepath.Join("csv/dll", "csv.dll") +// if _, err := os.Stat(dllPath); os.IsNotExist(err) { +// return nil, fmt.Errorf("csv DLL 不存在: %s", dllPath) +// } +// if dll, err := syscall.LoadDLL(dllPath); err != nil { +// return nil, fmt.Errorf("加载csv DLL 失败: %s", err) +// } else { +// return &csvDLL{ +// dll: dll, +// openCSVFile: dll.MustFindProc("OpenCSVFile"), +// updateCSVRowSafe: dll.MustFindProc("UpdateCSVRowSafe"), +// readRows: dll.MustFindProc("ReadRows"), +// writeRows: dll.MustFindProc("WriteRows"), +// appendRows: dll.MustFindProc("AppendRows"), +// getRowCount: dll.MustFindProc("GetRowCount"), +// findRows: dll.MustFindProc("FindRows"), +// closeCSVFile: dll.MustFindProc("CloseCSVFile"), +// mergeCSVFiles: dll.MustFindProc("MergeCSVFiles"), +// createOpenCSVFile: dll.MustFindProc("CreateOpenCSVFile"), +// getError: dll.MustFindProc("GetError"), +// freeCString: dll.MustFindProc("FreeCString"), +// }, nil +// } +//} +// +//// cStr 获取C字符串 +//func (m *csvDLL) cStr(p uintptr) string { +// if p == 0 { +// return "" +// } +// b := []byte{} +// for i := uintptr(0); ; i++ { +// c := *(*byte)(unsafe.Pointer(p + i)) +// if c == 0 { +// break +// } +// b = append(b, c) +// } +// s := string(b) +// if m.freeCString != nil { +// m.freeCString.Call(p) +// } +// return s +//} +// +//// 打开/创建CSV文件 +//func (m *csvDLL) OpenCSVFile(filePath string, delimiter byte, hasHeader bool) (int64, error) { +// proc, err := m.dll.FindProc("OpenCSVFile") +// if err != nil { +// return -1, fmt.Errorf("找不到函数 OpenCSVFile: %v", err) +// } +// filePathPtr, _ := syscall.BytePtrFromString(filePath) +// hasHeaderInt := 0 +// if hasHeader { +// hasHeaderInt = 1 +// } +// info, _, _ := proc.Call( +// uintptr(unsafe.Pointer(filePathPtr)), +// uintptr(delimiter), +// uintptr(hasHeaderInt)) +// +// return int64(info), nil +//} +// +//// CSV响应结构体 +//type CSVResponses struct { +// Success bool `json:"success"` +// Message string `json:"message,omitempty"` +// Data CSVData `json:"data,omitempty"` +//} +// +//type CSVData struct { +// HandleID int64 `json:"handleID"` +//} +// +//func (m *csvDLL) CreateOpenCSVFile(filePath string, delimiter byte, hasHeader bool) (*CSVResponses, error) { +// proc, err := m.dll.FindProc("CreateOpenCSVFile") +// if err != nil { +// return nil, fmt.Errorf("找不到函数 CreateOpenCSVFile: %v", err) +// } +// filePathPtr, _ := syscall.BytePtrFromString(filePath) +// hasHeaderInt := 0 +// if hasHeader { +// hasHeaderInt = 1 +// } +// info, _, _ := proc.Call( +// uintptr(unsafe.Pointer(filePathPtr)), +// uintptr(delimiter), +// uintptr(hasHeaderInt)) +// var csvResponse CSVResponses +// str := m.cStr(info) +// if err := json.Unmarshal([]byte(str), &csvResponse); err != nil { +// return nil, err +// } +// return &csvResponse, nil +//} +// +//func (m *csvDLL) UpdateCSVRowSafe(handleID int64, rowNum int, newRow []string) (*CSVResponses, error) { +// proc, err := m.dll.FindProc("UpdateCSVRowSafe") +// if err != nil { +// return nil, fmt.Errorf("找不到函数 UpdateCSVRowSafe: %v", err) +// } +// jsonString, _ := json.Marshal(newRow) +// newRowPtr, _ := syscall.BytePtrFromString(string(jsonString)) +// info, _, _ := proc.Call( +// uintptr(handleID), +// uintptr(rowNum), +// uintptr(unsafe.Pointer(newRowPtr))) +// var csvResponse CSVResponses +// str := m.cStr(info) +// if err := json.Unmarshal([]byte(str), &csvResponse); err != nil { +// return nil, err +// } +// return &csvResponse, nil +//} +// +////// 读取多行数据 +////func (m *csvDLL) ReadRows(handle int64) ([]string, error) { +//// proc, err := m.dll.FindProc("ReadRows") +//// if err != nil { +//// return nil, fmt.Errorf("找不到函数 ReadRows: %v", err) +//// } +//// +////} +// +//// 写入/覆盖行数据 +//func (m *csvDLL) WriteRows(handle int64, rowsData [][]string, header int) (int, error) { +// proc, err := m.dll.FindProc("WriteRows") +// if err != nil { +// return -1, fmt.Errorf("找不到函数 WriteRows: %v", err) +// } +// var buffer bytes.Buffer +// writer := csv.NewWriter(&buffer) +// // 写入所有行 +// if err := writer.WriteAll(rowsData); err != nil { +// return -1, fmt.Errorf("序列化 CSV 数据失败: %v", err) +// } +// writer.Flush() +// +// // 转换为 C 字符串 +// cStr := C.CString(buffer.String()) +// +// ret, _, _ := proc.Call( +// uintptr(handle), +// uintptr(unsafe.Pointer(cStr)), +// uintptr(header)) +// +// freeProc, _ := m.dll.FindProc("FreeCString") +// if freeProc != nil { +// defer freeProc.Call(uintptr(unsafe.Pointer(cStr))) +// } +// return int(ret), nil +//} +// +//// 定义请求结构体 +//type OpenCSVRequest struct { +// FilePath string `json:"filePath"` +// Delimiter string `json:"delimiter"` // 可以是逗号、分号等字符 +// HasHeader bool `json:"hasHeader"` +//} +// +//// 定义响应结构体 +//type OpenCSVResponse struct { +// Success bool `json:"success"` +// Message string `json:"message"` +// Handle int64 `json:"handle,omitempty"` +// Error string `json:"error,omitempty"` +//} +// +//func handleOpenCSVFile(w http.ResponseWriter, r *http.Request) { +// // 设置响应头 +// w.Header().Set("Content-Type", "application/json; charset=utf-8") +// +// // 只允许 POST 请求 +// if r.Method != http.MethodPost { +// response := OpenCSVResponse{ +// Success: false, +// Message: "只支持POST请求", +// } +// w.WriteHeader(http.StatusMethodNotAllowed) +// json.NewEncoder(w).Encode(response) +// return +// } +// +// // 1.初始化DLL管理器 +// dll, err := InitCsvDLL() +// if err != nil { +// response := OpenCSVResponse{ +// Success: false, +// Message: "初始化DLL失败", +// Error: err.Error(), +// } +// w.WriteHeader(http.StatusInternalServerError) +// json.NewEncoder(w).Encode(response) +// return +// } +// +// // 2.读取请求体 +// body, err := io.ReadAll(r.Body) +// if err != nil { +// response := OpenCSVResponse{ +// Success: false, +// Message: "读取请求体失败", +// Error: err.Error(), +// } +// w.WriteHeader(http.StatusInternalServerError) +// json.NewEncoder(w).Encode(response) +// return +// } +// defer r.Body.Close() +// +// // 3.解析JSON请求 +// var req OpenCSVRequest +// if err := json.Unmarshal(body, &req); err != nil { +// response := OpenCSVResponse{ +// Success: false, +// Message: "JSON解析失败", +// Error: err.Error(), +// } +// w.WriteHeader(http.StatusInternalServerError) +// json.NewEncoder(w).Encode(response) +// return +// } +// +// // 4.验证必填参数 +// if req.FilePath == "" { +// response := OpenCSVResponse{ +// Success: false, +// Message: "filePath参数不能为空", +// } +// w.WriteHeader(http.StatusBadRequest) +// json.NewEncoder(w).Encode(response) +// return +// } +// +// // 5. 处理分隔符参数 +// delimiter := ',' +// if req.Delimiter != "" { +// // 确保分隔符是单个字符 +// if len(req.Delimiter) == 1 { +// delimiter = rune(req.Delimiter[0]) +// } else { +// // 尝试解析常见分隔符的字符串表示 +// switch req.Delimiter { +// case "comma", ",": +// delimiter = ',' +// case "semicolon", ";": +// delimiter = ';' +// case "tab", "\t": +// delimiter = '\t' +// case "pipe", "|": +// delimiter = '|' +// default: +// response := OpenCSVResponse{ +// Success: false, +// Message: "无效的分隔符，请使用单个字符或预定义的分隔符名称", +// } +// w.WriteHeader(http.StatusBadRequest) +// json.NewEncoder(w).Encode(response) +// return +// } +// } +// } +// +// // 6. 调用DLL函数 +// handle, err := dll.OpenCSVFile(req.FilePath, byte(delimiter), req.HasHeader) +// if err != nil { +// response := OpenCSVResponse{ +// Success: false, +// Message: "打开CSV文件失败", +// Error: err.Error(), +// } +// w.WriteHeader(http.StatusInternalServerError) +// json.NewEncoder(w).Encode(response) +// return +// } +// +// // 7. 返回成功响应 +// response := OpenCSVResponse{ +// Success: true, +// Message: "CSV文件打开成功", +// Handle: handle, +// } +// w.WriteHeader(http.StatusOK) +// json.NewEncoder(w).Encode(response) +//} +// +//// 解码从DLL读取的数据 +//func decodeRowData(buffer []byte, maxBytes int) [][]string { +// var rows [][]string +// var currentRow []string +// +// offset := 0 +// for offset < maxBytes { +// if offset+4 > maxBytes { +// break +// } +// +// // 读取单元格长度 +// cellLen := int(uint32(buffer[offset]) | +// uint32(buffer[offset+1])<<8 | +// uint32(buffer[offset+2])<<16 | +// uint32(buffer[offset+3])<<24) +// offset += 4 +// +// if cellLen == 0 { +// // 行结束 +// if len(currentRow) > 0 { +// rows = append(rows, currentRow) +// currentRow = nil +// } +// continue +// } +// +// if offset+cellLen > maxBytes { +// break +// } +// +// // 读取单元格数据 +// cell := string(buffer[offset : offset+cellLen]) +// offset += cellLen +// currentRow = append(currentRow, cell) +// } +// +// return rows +//} + +//func main() { +// 使用dll文件 +//dll, err := InitCsvDLL() +//if err != nil { +// +//} +//file, err := dll.CreateOpenCSVFile("csv/taskLog.csv", ',', true) +//if err != nil { +// +//} +//newRow := []string{"9787115524539", "100.00", "1", "上传成功", ""} +//safe, err := dll.UpdateCSVRowSafe(file.Data.HandleID, 9, newRow) +//if err != nil { +// +//} +//marshal, _ := json.Marshal(safe) +//fmt.Println(string(marshal)) + +//file, err := GetManager().OpenCSVFile("csv/taskLog1.csv", ',', true) +//if err != nil { +// fmt.Println(err) +//} +//fmt.Println(file) + +//handle, err := GetManager().getHandle(2) +//if err != nil { +// fmt.Println(err) +//} +// 获取指定数量的行 +//row, err := handle.readRows(100) +//if err != nil { +// fmt.Println(err) +//} +//for _, i := range row { +// fmt.Println(i) +//} + +//// 获取总行数 +//row, err := handle.getTotalRows() +//if err != nil { +// fmt.Println(err) +//} +//fmt.Println(row) +//newRow := []string{ +// "9787107267505", "20.00", "10", "上传成功", "877133619369", +//} +// +//row, err := GetManager().modifyRow(file, 1, newRow) +//if err != nil { +// fmt.Println(err) +//} +//fmt.Println(row) + +//http.HandleFunc("/csv/openCSVFile", handleOpenCSVFile) +//port := "8080" +//server := &http.Server{ +// Addr: ":" + port, +// Handler: nil, +//} +// +//// 4. 优雅关闭设置 +//done := make(chan bool, 1) +//quit := make(chan os.Signal, 1) +//signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) +// +//// 5. 优雅关闭协程 +//go func() { +// <-quit +// fmt.Println("\n服务器正在关闭...") +// +// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) +// defer cancel() +// +// if err := server.Shutdown(ctx); err != nil { +// fmt.Printf("强制关闭服务器: %v\n", err) +// } +// close(done) +//}() +// +//// 启动服务器 +//if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { +// fmt.Printf("服务器启动失败: %s\n", err) +//} +//// 7. 等待关闭完成 +//<-done +//fmt.Println("服务器已关闭") +//} + +// 主函数 - 测试代码 +func main() { + fmt.Println("=== CSV句柄管理器测试 ===") //7984 + + filename := "csv/test1.csv" + ////dstFile := "csv/taskLog3.csv" + //fmt.Printf("1. 创建测试文件: %s\n", filename) + // + // 创建 Redis 客户端 + rdb := redis.NewClient(&redis.Options{ + Addr: "192.168.101.209:6379", // Redis 地址 + Password: "", // 密码，没有则为空 + DB: 0, // 使用的数据库编号 + }) + + // 创建上下文 + ctx := context.Background() + + // 测试连接 + pong, err := rdb.Ping(ctx).Result() + if err != nil { + log.Fatal("连接 Redis 失败:", err) + } + fmt.Println("连接成功:", pong) + + key := "2006557053525397505_2010589232836288513_20260112_1" + + //// 获取值 + //val, err := rdb.Get(ctx, "1995373681100910593_2010521216979214337_20260112_1").Result() + //if err != nil { + // log.Fatal("获取键值失败:", err) + //} + //fmt.Println("key:", val) + + handleID, err := GetManager().OpenCSVFile(filename, ',', true) + if err != nil { + fmt.Printf("打开文件失败: %v\n", err) + } + + fmt.Printf("2. 打开文件成功，句柄ID: %d\n", handleID) + + // 测试写入表头 + header := []string{"ID", "data"} + err = GetManager().WriteHeader(handleID, header) + if err != nil { + fmt.Printf("写入表头失败: %v\n", err) + } else { + fmt.Println("写入表头成功") + } + + // 获取列表所有元素 + listValues, err := rdb.LRange(ctx, key, 0, -1).Result() + if err != nil { + log.Fatal("获取列表失败:", err) + } + + fmt.Printf("列表包含 %d 个元素:\n", len(listValues)) + for i, value := range listValues { + + fmt.Printf(" [%d] %s\n", i, value) + + rowsData := [][]string{ + {fmt.Sprintf("%d", i), value}, + } + + totalRows, err := GetManager().WriteRows(handleID, rowsData) + if err != nil { + fmt.Printf("批量写入数据失败: %v\n", err) + } else { + fmt.Printf("批量写入数据成功,行数: %d\n", totalRows) + } + + } + + // 3. 也可以检查键是否存在 + exists, err := rdb.Exists(ctx, key).Result() + if err != nil { + log.Fatal("检查键是否存在失败:", err) + } + if exists == 0 { + fmt.Printf("键 '%s' 不存在\n", key) + } else { + fmt.Printf("键 '%s' 存在\n", key) + } + + // 关闭连接 + defer rdb.Close() + + // + //// 2. 创建目标文件（部分数据） + //dstHandle, err := GetManager().OpenCSVFile(dstFile, ',', true) + //if err != nil { + // fmt.Printf("创建目标文件失败: %v", err) + //} + //// 合并 + //totalRows, err := GetManager().MergeCSVFilesSimple(handleID, dstHandle, true) + //if err != nil { + // fmt.Printf("合并文件失败: %v", err) + //} + //fmt.Println(totalRows) + + //for i := 0; i < 10000; i++ { + // // 打开CSV文件 + // handleID, err := GetManager().OpenCSVFile(filename, ',', true) + // if err != nil { + // fmt.Printf("打开文件失败: %v\n", err) + // } + // + // fmt.Printf("2. 打开文件成功，句柄ID: %d\n", handleID) + // + // // 测试写入表头 + // header := []string{"ID", "Name", "Age", "Email"} + // err = GetManager().WriteHeader(handleID, header) + // if err != nil { + // fmt.Printf("写入表头失败: %v\n", err) + // } else { + // fmt.Println("写入表头成功") + // } + // + // //// 写入测试数据 + // //rowsData := [][]string{ + // // {"1", fmt.Sprintf("张三", i), "25", "zhangsan@example.com"}, + // // {"2", "李四", "30", "lisi@example.com"}, + // // {"3", "王五", "28", "wangwu@example.com"}, + // // {"4", "赵六", "35", "zhaoliu@example.com"}, + // //} + // + // // 写入测试数据 + // rowsData := [][]string{ + // {fmt.Sprintf("%d", i+100), "张三", fmt.Sprintf("%d", i+1000), fmt.Sprintf("%d", i+10000)}, + // } + // + // totalRows, err := GetManager().WriteRows(handleID, rowsData) + // if err != nil { + // fmt.Printf("批量写入数据失败: %v\n", err) + // } else { + // fmt.Printf("批量写入数据成功,行数: %d\n", totalRows) + // } + // + // GetManager().CloseHandle(handleID) + //} + + //// 测试写入表头 + //header := []string{"ID", "Name", "Age", "Email"} + //err = GetManager().WriteHeader(handleID, header) + //if err != nil { + // fmt.Printf("写入表头失败: %v\n", err) + //} else { + // fmt.Println("写入表头成功") + //} + + //// 写入测试数据 + //rowsData := [][]string{ + // {"1", "张三", "25", "zhangsan@example.com"}, + // {"2", "李四", "30", "lisi@example.com"}, + // {"3", "王五", "28", "wangwu@example.com"}, + // {"4", "赵六", "35", "zhaoliu@example.com"}, + //} + // + //totalRows, err := GetManager().WriteRows(handleID, rowsData) + //if err != nil { + // fmt.Printf("批量写入数据失败: %v\n", err) + //} else { + // fmt.Printf("批量写入数据成功,行数: %d\n", totalRows) + //} + // + //// 获取写入后的总行数 + //totalRows, err = GetManager().GetTotalRows(handleID) + //if err != nil { + // fmt.Printf("获取总行数失败: %v\n", err) + //} else { + // fmt.Printf("写入后总行数: %d\n", totalRows) + //} + + //// ============ 测试修改行功能 ============ + // + //fmt.Println("\n3. 测试修改行功能") + // + //// 修改第1行数据（索引0） + //newRow1 := []string{"1", "张三（已修改1）", "26", "zhangsan_updated@example.com"} + //err = GetManager().UpdateRow(handleID, 0, newRow1) + //if err != nil { + // fmt.Printf("修改第1行数据失败: %v\n", err) + //} else { + // fmt.Println("修改第1行数据成功") + //} + // + //// 修改第3行数据（索引2） + //newRow3 := []string{"3", "王五（已修改）", "29", "wangwu_updated@example.com"} + //err = GetManager().UpdateRow(handleID, 2, newRow3) + //if err != nil { + // fmt.Printf("修改第3行数据失败: %v\n", err) + //} else { + // fmt.Println("修改第3行数据成功") + //} + // + //// 获取修改后的行数据 + //fmt.Println("\n4. 获取修改后的行数据") + + //获取第1行 + //row1, err := GetManager().GetRow(handleID, 10) + //if err != nil { + // fmt.Printf("获取第1行失败: %v\n", err) + //} else { + // fmt.Printf("第1行数据: %v\n", row1) + //} + + //// 获取第3行 + //row3, err := GetManager().GetRow(handleID, 2) + //if err != nil { + // fmt.Printf("获取第3行失败: %v\n", err) + //} else { + // fmt.Printf("第3行数据: %v\n", row3) + //} + + //// ============ 测试批量修改功能 ============ + // + //fmt.Println("\n5. 测试批量修改功能") + // + //updates := map[int64][]string{ + // 1: {"2", "李四（批量修改）", "31", "lisi_batch@example.com"}, + // 3: {"4", "赵六（批量修改）", "36", "zhaoliu_batch@example.com"}, + //} + // + //updatedCount, err := GetManager().UpdateRows(handleID, updates) + //if err != nil { + // fmt.Printf("批量修改失败: %v\n", err) + //} else { + // fmt.Printf("批量修改成功，修改了%d行\n", updatedCount) + //} + // + //// 验证批量修改结果 + //row2, err := GetManager().GetRow(handleID, 1) + //if err != nil { + // fmt.Printf("获取第2行失败: %v\n", err) + //} else { + // fmt.Printf("第2行数据（批量修改后）: %v\n", row2) + //} + // + //row4, err := GetManager().GetRow(handleID, 3) + //if err != nil { + // fmt.Printf("获取第4行失败: %v\n", err) + //} else { + // fmt.Printf("第4行数据（批量修改后）: %v\n", row4) + //} + // + //// 获取最终总行数 + //finalTotalRows, err := GetManager().GetTotalRows(handleID) + //if err != nil { + // fmt.Printf("获取最终总行数失败: %v\n", err) + //} else { + // fmt.Printf("最终总行数: %d\n", finalTotalRows) + //} + + // 清理 + //GetManager().closeAllHandles() + fmt.Println("\n测试完成！") +} diff --git a/csv/csvDllTest.go b/csv/csvDllTest.go deleted file mode 100644 index 679de38..0000000 --- a/csv/csvDllTest.go +++ /dev/null @@ -1,449 +0,0 @@ -package main - -import "C" -import ( - "bytes" - "encoding/csv" - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "syscall" - "unsafe" -) - -// CsvDLL 代理DLL结构 -type csvDLL struct { - dll *syscall.DLL - openCSVFile *syscall.Proc // 打开/创建CSV文件 - readRows *syscall.Proc // 读取多行数据 - writeRows *syscall.Proc // 写入/覆盖行数据 - appendRows *syscall.Proc // 追加行数据 - getRowCount *syscall.Proc // 获取总行数 - findRows *syscall.Proc // 搜索行 - closeCSVFile *syscall.Proc // 关闭CSV文件 - mergeCSVFiles *syscall.Proc // 合并多个CSV文件 - getError *syscall.Proc // 获取错误信息 - updateCSVRowSafe *syscall.Proc - createOpenCSVFile *syscall.Proc - freeCString *syscall.Proc // 释放C字符串 -} - -// 初始化csvDLL -func InitCsvDLL() (*csvDLL, error) { - dllPath := filepath.Join("csv/dll", "csv.dll") - if _, err := os.Stat(dllPath); os.IsNotExist(err) { - return nil, fmt.Errorf("csv DLL 不存在: %s", dllPath) - } - if dll, err := syscall.LoadDLL(dllPath); err != nil { - return nil, fmt.Errorf("加载csv DLL 失败: %s", err) - } else { - return &csvDLL{ - dll: dll, - openCSVFile: dll.MustFindProc("OpenCSVFile"), - updateCSVRowSafe: dll.MustFindProc("UpdateCSVRowSafe"), - readRows: dll.MustFindProc("ReadRows"), - writeRows: dll.MustFindProc("WriteRows"), - appendRows: dll.MustFindProc("AppendRows"), - getRowCount: dll.MustFindProc("GetRowCount"), - findRows: dll.MustFindProc("FindRows"), - closeCSVFile: dll.MustFindProc("CloseCSVFile"), - mergeCSVFiles: dll.MustFindProc("MergeCSVFiles"), - createOpenCSVFile: dll.MustFindProc("CreateOpenCSVFile"), - getError: dll.MustFindProc("GetError"), - freeCString: dll.MustFindProc("FreeCString"), - }, nil - } -} - -// cStr 获取C字符串 -func (m *csvDLL) cStr(p uintptr) string { - if p == 0 { - return "" - } - b := []byte{} - for i := uintptr(0); ; i++ { - c := *(*byte)(unsafe.Pointer(p + i)) - if c == 0 { - break - } - b = append(b, c) - } - s := string(b) - if m.freeCString != nil { - m.freeCString.Call(p) - } - return s -} - -// 打开/创建CSV文件 -func (m *csvDLL) OpenCSVFile(filePath string, delimiter byte, hasHeader bool) (int64, error) { - proc, err := m.dll.FindProc("OpenCSVFile") - if err != nil { - return -1, fmt.Errorf("找不到函数 OpenCSVFile: %v", err) - } - filePathPtr, _ := syscall.BytePtrFromString(filePath) - hasHeaderInt := 0 - if hasHeader { - hasHeaderInt = 1 - } - info, _, _ := proc.Call( - uintptr(unsafe.Pointer(filePathPtr)), - uintptr(delimiter), - uintptr(hasHeaderInt)) - - return int64(info), nil -} - -// CSV响应结构体 -type CSVResponses struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` - Data CSVData `json:"data,omitempty"` -} - -type CSVData struct { - HandleID int64 `json:"handleID"` -} - -func (m *csvDLL) CreateOpenCSVFile(filePath string, delimiter byte, hasHeader bool) (*CSVResponses, error) { - proc, err := m.dll.FindProc("CreateOpenCSVFile") - if err != nil { - return nil, fmt.Errorf("找不到函数 CreateOpenCSVFile: %v", err) - } - filePathPtr, _ := syscall.BytePtrFromString(filePath) - hasHeaderInt := 0 - if hasHeader { - hasHeaderInt = 1 - } - info, _, _ := proc.Call( - uintptr(unsafe.Pointer(filePathPtr)), - uintptr(delimiter), - uintptr(hasHeaderInt)) - var csvResponse CSVResponses - str := m.cStr(info) - if err := json.Unmarshal([]byte(str), &csvResponse); err != nil { - return nil, err - } - return &csvResponse, nil -} - -func (m *csvDLL) UpdateCSVRowSafe(handleID int64, rowNum int, newRow []string) (*CSVResponses, error) { - proc, err := m.dll.FindProc("UpdateCSVRowSafe") - if err != nil { - return nil, fmt.Errorf("找不到函数 UpdateCSVRowSafe: %v", err) - } - jsonString, _ := json.Marshal(newRow) - newRowPtr, _ := syscall.BytePtrFromString(string(jsonString)) - info, _, _ := proc.Call( - uintptr(handleID), - uintptr(rowNum), - uintptr(unsafe.Pointer(newRowPtr))) - var csvResponse CSVResponses - str := m.cStr(info) - if err := json.Unmarshal([]byte(str), &csvResponse); err != nil { - return nil, err - } - return &csvResponse, nil -} - -//// 读取多行数据 -//func (m *csvDLL) ReadRows(handle int64) ([]string, error) { -// proc, err := m.dll.FindProc("ReadRows") -// if err != nil { -// return nil, fmt.Errorf("找不到函数 ReadRows: %v", err) -// } -// -//} - -// 写入/覆盖行数据 -func (m *csvDLL) WriteRows(handle int64, rowsData [][]string, header int) (int, error) { - proc, err := m.dll.FindProc("WriteRows") - if err != nil { - return -1, fmt.Errorf("找不到函数 WriteRows: %v", err) - } - var buffer bytes.Buffer - writer := csv.NewWriter(&buffer) - // 写入所有行 - if err := writer.WriteAll(rowsData); err != nil { - return -1, fmt.Errorf("序列化 CSV 数据失败: %v", err) - } - writer.Flush() - - // 转换为 C 字符串 - cStr := C.CString(buffer.String()) - - ret, _, _ := proc.Call( - uintptr(handle), - uintptr(unsafe.Pointer(cStr)), - uintptr(header)) - - freeProc, _ := m.dll.FindProc("FreeCString") - if freeProc != nil { - defer freeProc.Call(uintptr(unsafe.Pointer(cStr))) - } - return int(ret), nil -} - -// 定义请求结构体 -type OpenCSVRequest struct { - FilePath string `json:"filePath"` - Delimiter string `json:"delimiter"` // 可以是逗号、分号等字符 - HasHeader bool `json:"hasHeader"` -} - -// 定义响应结构体 -type OpenCSVResponse struct { - Success bool `json:"success"` - Message string `json:"message"` - Handle int64 `json:"handle,omitempty"` - Error string `json:"error,omitempty"` -} - -func handleOpenCSVFile(w http.ResponseWriter, r *http.Request) { - // 设置响应头 - w.Header().Set("Content-Type", "application/json; charset=utf-8") - - // 只允许 POST 请求 - if r.Method != http.MethodPost { - response := OpenCSVResponse{ - Success: false, - Message: "只支持POST请求", - } - w.WriteHeader(http.StatusMethodNotAllowed) - json.NewEncoder(w).Encode(response) - return - } - - // 1.初始化DLL管理器 - dll, err := InitCsvDLL() - if err != nil { - response := OpenCSVResponse{ - Success: false, - Message: "初始化DLL失败", - Error: err.Error(), - } - w.WriteHeader(http.StatusInternalServerError) - json.NewEncoder(w).Encode(response) - return - } - - // 2.读取请求体 - body, err := io.ReadAll(r.Body) - if err != nil { - response := OpenCSVResponse{ - Success: false, - Message: "读取请求体失败", - Error: err.Error(), - } - w.WriteHeader(http.StatusInternalServerError) - json.NewEncoder(w).Encode(response) - return - } - defer r.Body.Close() - - // 3.解析JSON请求 - var req OpenCSVRequest - if err := json.Unmarshal(body, &req); err != nil { - response := OpenCSVResponse{ - Success: false, - Message: "JSON解析失败", - Error: err.Error(), - } - w.WriteHeader(http.StatusInternalServerError) - json.NewEncoder(w).Encode(response) - return - } - - // 4.验证必填参数 - if req.FilePath == "" { - response := OpenCSVResponse{ - Success: false, - Message: "filePath参数不能为空", - } - w.WriteHeader(http.StatusBadRequest) - json.NewEncoder(w).Encode(response) - return - } - - // 5. 处理分隔符参数 - delimiter := ',' - if req.Delimiter != "" { - // 确保分隔符是单个字符 - if len(req.Delimiter) == 1 { - delimiter = rune(req.Delimiter[0]) - } else { - // 尝试解析常见分隔符的字符串表示 - switch req.Delimiter { - case "comma", ",": - delimiter = ',' - case "semicolon", ";": - delimiter = ';' - case "tab", "\t": - delimiter = '\t' - case "pipe", "|": - delimiter = '|' - default: - response := OpenCSVResponse{ - Success: false, - Message: "无效的分隔符，请使用单个字符或预定义的分隔符名称", - } - w.WriteHeader(http.StatusBadRequest) - json.NewEncoder(w).Encode(response) - return - } - } - } - - // 6. 调用DLL函数 - handle, err := dll.OpenCSVFile(req.FilePath, byte(delimiter), req.HasHeader) - if err != nil { - response := OpenCSVResponse{ - Success: false, - Message: "打开CSV文件失败", - Error: err.Error(), - } - w.WriteHeader(http.StatusInternalServerError) - json.NewEncoder(w).Encode(response) - return - } - - // 7. 返回成功响应 - response := OpenCSVResponse{ - Success: true, - Message: "CSV文件打开成功", - Handle: handle, - } - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(response) -} - -// 解码从DLL读取的数据 -func decodeRowData(buffer []byte, maxBytes int) [][]string { - var rows [][]string - var currentRow []string - - offset := 0 - for offset < maxBytes { - if offset+4 > maxBytes { - break - } - - // 读取单元格长度 - cellLen := int(uint32(buffer[offset]) | - uint32(buffer[offset+1])<<8 | - uint32(buffer[offset+2])<<16 | - uint32(buffer[offset+3])<<24) - offset += 4 - - if cellLen == 0 { - // 行结束 - if len(currentRow) > 0 { - rows = append(rows, currentRow) - currentRow = nil - } - continue - } - - if offset+cellLen > maxBytes { - break - } - - // 读取单元格数据 - cell := string(buffer[offset : offset+cellLen]) - offset += cellLen - currentRow = append(currentRow, cell) - } - - return rows -} - -//func main() { -// 使用dll文件 -//dll, err := InitCsvDLL() -//if err != nil { -// -//} -//file, err := dll.CreateOpenCSVFile("csv/taskLog.csv", ',', true) -//if err != nil { -// -//} -//newRow := []string{"9787115524539", "100.00", "1", "上传成功", ""} -//safe, err := dll.UpdateCSVRowSafe(file.Data.HandleID, 9, newRow) -//if err != nil { -// -//} -//marshal, _ := json.Marshal(safe) -//fmt.Println(string(marshal)) - -//file, err := GetManager().OpenCSVFile("csv/taskLog1.csv", ',', true) -//if err != nil { -// fmt.Println(err) -//} -//fmt.Println(file) - -//handle, err := GetManager().getHandle(2) -//if err != nil { -// fmt.Println(err) -//} -// 获取指定数量的行 -//row, err := handle.readRows(100) -//if err != nil { -// fmt.Println(err) -//} -//for _, i := range row { -// fmt.Println(i) -//} - -//// 获取总行数 -//row, err := handle.getTotalRows() -//if err != nil { -// fmt.Println(err) -//} -//fmt.Println(row) -//newRow := []string{ -// "9787107267505", "20.00", "10", "上传成功", "877133619369", -//} -// -//row, err := GetManager().modifyRow(file, 1, newRow) -//if err != nil { -// fmt.Println(err) -//} -//fmt.Println(row) - -//http.HandleFunc("/csv/openCSVFile", handleOpenCSVFile) -//port := "8080" -//server := &http.Server{ -// Addr: ":" + port, -// Handler: nil, -//} -// -//// 4. 优雅关闭设置 -//done := make(chan bool, 1) -//quit := make(chan os.Signal, 1) -//signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) -// -//// 5. 优雅关闭协程 -//go func() { -// <-quit -// fmt.Println("\n服务器正在关闭...") -// -// ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) -// defer cancel() -// -// if err := server.Shutdown(ctx); err != nil { -// fmt.Printf("强制关闭服务器: %v\n", err) -// } -// close(done) -//}() -// -//// 启动服务器 -//if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { -// fmt.Printf("服务器启动失败: %s\n", err) -//} -//// 7. 等待关闭完成 -//<-done -//fmt.Println("服务器已关闭") -//} diff --git a/csv/dll/csv.dll b/csv/dll/csv.dll index 3eeb663..5ebdcba 100644 Binary files a/csv/dll/csv.dll and b/csv/dll/csv.dll differ diff --git a/csv/newcsv.go b/csv/newcsv.go index 52dd780..72fc84f 100644 --- a/csv/newcsv.go +++ b/csv/newcsv.go @@ -1,5 +1,6 @@ package main +// ///* //#include //*/ @@ -96,6 +97,31 @@ package main // // 创建CSV全局管理器 // mgr := getManager() // +// // 先检查文件是否已经打开 +// var existingHandle int64 = -1 +// mgr.files.Range(func(key, value interface{}) bool { +// file := value.(*CSVFile) +// // 比较文件路径（使用绝对路径确保一致性） +// absFilename, _ := filepath.Abs(filename) +// absExisting, _ := filepath.Abs(file.filename) +// +// if absFilename == absExisting { +// fmt.Println("absFilename", absFilename) +// fmt.Println("absExisting", absExisting) +// fmt.Println("bool", absFilename == absExisting) +// +// existingHandle = key.(int64) +// return false // 停止遍历 +// } +// return true // 继续遍历 +// }) +// +// // 如果文件已经打开，返回现有句柄 +// if existingHandle != -1 { +// fmt.Printf("文件已打开，返回现有句柄: %d\n", existingHandle) +// return existingHandle +// } +// // // 生成句柄 // handle := atomic.AddInt64(&mgr.nextHandle, 1) // @@ -111,7 +137,6 @@ package main // // // 存储到管理器 // mgr.files.Store(handle, file) -// // return handle //} // @@ -548,7 +573,10 @@ package main // // 异步保存 // go file.saveAsync() // -// return 0 +// // 获取总行数 +// count := getRowCount(handle) +// fmt.Println("count:", count) +// return int(count) //} // //// 打开/创建CSV文件(句柄) @@ -705,7 +733,9 @@ package main // file := val.(*CSVFile) // file.mu.RLock() // defer file.mu.RUnlock() -// +// if file.header != nil { +// return int64(len(file.data) + 1) +// } // return int64(len(file.data)) //} // @@ -997,10 +1027,10 @@ package main // // 从管理器移除 // mgr.files.Delete(handle) // -// // 安全关闭通道（如果存在） -// if file.done != nil { -// close(file.done) -// } +// //// 安全关闭通道（如果存在） +// //if file.done != nil { +// // close(file.done) +// //} // // return 0 //} @@ -1144,7 +1174,12 @@ package main //func WriteRows(handle C.longlong, rowsData *C.char, header C.int) C.int { // goData := C.GoString(rowsData) // goHeader := int(header) -// data := parseSimpleTable(goData) +// //data := parseSimpleTable(goData) +// var data [][]string +// err := json.Unmarshal([]byte(goData), &data) +// if err != nil { +// setError(err.Error()) +// } // result := writeRows(int64(handle), data, goHeader) // return C.int(result) //} @@ -1224,3 +1259,36 @@ package main //// main 函数是必需的，即使为空 //func main() { //} +// +//// 编码多行数据 +//func encodeRowsData(rows [][]string) []byte { +// var result []byte +// for _, row := range rows { +// rowData := encodeRowData(row) +// result = append(result, rowData...) +// } +// return result +//} +// +//// 编码行数据为DLL期望的格式 +//func encodeRowData(row []string) []byte { +// var result []byte +// +// for _, cell := range row { +// // 写入4字节长度 +// length := len(cell) +// result = append(result, +// byte(length&0xFF), +// byte((length>>8)&0xFF), +// byte((length>>16)&0xFF), +// byte((length>>24)&0xFF)) +// +// // 写入单元格数据 +// result = append(result, []byte(cell)...) +// } +// +// // 写入行结束标记 (4个0字节) +// result = append(result, 0, 0, 0, 0) +// +// return result +//} diff --git a/dll/csv.dll b/dll/csv.dll deleted file mode 100644 index 4deb73b..0000000 Binary files a/dll/csv.dll and /dev/null differ diff --git a/dll/csv.h b/dll/csv.h deleted file mode 100644 index ec41202..0000000 --- a/dll/csv.h +++ /dev/null @@ -1,134 +0,0 @@ -/* Code generated by cmd/cgo; DO NOT EDIT. */ - -/* package command-line-arguments */ - - -#line 1 "cgo-builtin-export-prolog" - -#include - -#ifndef GO_CGO_EXPORT_PROLOGUE_H -#define GO_CGO_EXPORT_PROLOGUE_H - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef struct { const char *p; ptrdiff_t n; } _GoString_; -extern size_t _GoStringLen(_GoString_ s); -extern const char *_GoStringPtr(_GoString_ s); -#endif - -#endif - -/* Start of preamble from import "C" comments. */ - - -#line 3 "csv.go" - -#include - -#line 1 "cgo-generated-wrapper" - - -/* End of preamble from import "C" comments. */ - - -/* Start of boilerplate cgo prologue. */ -#line 1 "cgo-gcc-export-header-prolog" - -#ifndef GO_CGO_PROLOGUE_H -#define GO_CGO_PROLOGUE_H - -typedef signed char GoInt8; -typedef unsigned char GoUint8; -typedef short GoInt16; -typedef unsigned short GoUint16; -typedef int GoInt32; -typedef unsigned int GoUint32; -typedef long long GoInt64; -typedef unsigned long long GoUint64; -typedef GoInt64 GoInt; -typedef GoUint64 GoUint; -typedef size_t GoUintptr; -typedef float GoFloat32; -typedef double GoFloat64; -#ifdef _MSC_VER -#if !defined(__cplusplus) || _MSVC_LANG <= 201402L -#include -typedef _Fcomplex GoComplex64; -typedef _Dcomplex GoComplex128; -#else -#include -typedef std::complex GoComplex64; -typedef std::complex GoComplex128; -#endif -#else -typedef float _Complex GoComplex64; -typedef double _Complex GoComplex128; -#endif - -/* - static assertion to make sure the file is being used on architecture - at least with matching size of GoInt. -*/ -typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef _GoString_ GoString; -#endif -typedef void *GoMap; -typedef void *GoChan; -typedef struct { void *t; void *v; } GoInterface; -typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; - -#endif - -/* End of boilerplate cgo prologue. */ - -#ifdef __cplusplus -extern "C" { -#endif - -extern __declspec(dllexport) long long int InitCSVManager(void); - -// OpenCSVFile 打开/创建CSV文件 -// -extern __declspec(dllexport) long long int OpenCSVFile(char* filename, char delimiter, int hasHeader); - -// ReadRows 读取多行数据 -// -extern __declspec(dllexport) long long int ReadRows(long long int handle, long long int startRow, long long int count, char* buffer, int bufferSize); - -// WriteRows 写入/覆盖行数据 -// -extern __declspec(dllexport) int WriteRows(long long int handle, char* rowsData, int dataSize, long long int rowCount); - -// AppendRows 追加行数据 -// -extern __declspec(dllexport) int AppendRows(long long int handle, char* rowsData, int dataSize, long long int rowCount); - -// GetRowCount 获取总行数 -// -extern __declspec(dllexport) long long int GetRowCount(long long int handle); - -// FindRows 搜索行 -// -extern __declspec(dllexport) long long int FindRows(long long int handle, char* searchText, long long int columnIndex, char* resultBuffer, int bufferSize, long long int maxResults); - -// MergeCSVFiles 合并多个CSV文件（线程安全） -// -extern __declspec(dllexport) long long int MergeCSVFiles(long long int* handlesPtr, int handlesCount, char* outputFilename, char delimiter, int hasHeader); - -// CloseCSVFile 关闭文件 -// -extern __declspec(dllexport) int CloseCSVFile(long long int handle); - -// GetError 获取错误信息 -// -extern __declspec(dllexport) int GetError(char* buffer, int bufferSize); - -// 导出函数：释放C字符串内存 -// -extern __declspec(dllexport) void FreeCString(char* str); - -#ifdef __cplusplus -} -#endif diff --git a/dll/excel.dll b/dll/excel.dll deleted file mode 100644 index 3b7a71d..0000000 Binary files a/dll/excel.dll and /dev/null differ diff --git a/dll/excel.h b/dll/excel.h deleted file mode 100644 index a5ccca5..0000000 --- a/dll/excel.h +++ /dev/null @@ -1,141 +0,0 @@ -/* Code generated by cmd/cgo; DO NOT EDIT. */ - -/* package command-line-arguments */ - - -#line 1 "cgo-builtin-export-prolog" - -#include - -#ifndef GO_CGO_EXPORT_PROLOGUE_H -#define GO_CGO_EXPORT_PROLOGUE_H - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef struct { const char *p; ptrdiff_t n; } _GoString_; -extern size_t _GoStringLen(_GoString_ s); -extern const char *_GoStringPtr(_GoString_ s); -#endif - -#endif - -/* Start of preamble from import "C" comments. */ - - -#line 3 "main.go" - -#include - -#line 1 "cgo-generated-wrapper" - - -/* End of preamble from import "C" comments. */ - - -/* Start of boilerplate cgo prologue. */ -#line 1 "cgo-gcc-export-header-prolog" - -#ifndef GO_CGO_PROLOGUE_H -#define GO_CGO_PROLOGUE_H - -typedef signed char GoInt8; -typedef unsigned char GoUint8; -typedef short GoInt16; -typedef unsigned short GoUint16; -typedef int GoInt32; -typedef unsigned int GoUint32; -typedef long long GoInt64; -typedef unsigned long long GoUint64; -typedef GoInt64 GoInt; -typedef GoUint64 GoUint; -typedef size_t GoUintptr; -typedef float GoFloat32; -typedef double GoFloat64; -#ifdef _MSC_VER -#if !defined(__cplusplus) || _MSVC_LANG <= 201402L -#include -typedef _Fcomplex GoComplex64; -typedef _Dcomplex GoComplex128; -#else -#include -typedef std::complex GoComplex64; -typedef std::complex GoComplex128; -#endif -#else -typedef float _Complex GoComplex64; -typedef double _Complex GoComplex128; -#endif - -/* - static assertion to make sure the file is being used on architecture - at least with matching size of GoInt. -*/ -typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef _GoString_ GoString; -#endif -typedef void *GoMap; -typedef void *GoChan; -typedef struct { void *t; void *v; } GoInterface; -typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; - -#endif - -/* End of boilerplate cgo prologue. */ - -#ifdef __cplusplus -extern "C" { -#endif - - -// 创建新的Excel管理器并返回指针 -// -extern __declspec(dllexport) uintptr_t NewExcelManagerInstance(void); - -// 释放Excel管理器 -// -extern __declspec(dllexport) void FreeExcelManager(uintptr_t handle); - -// 读取Excel数据 -// -extern __declspec(dllexport) int ReadExcelData(uintptr_t handle, char* filename, char* sheet, char** result); - -// 批量写入数据到Excel文件 -// -extern __declspec(dllexport) int WriteBatchData(uintptr_t handle, char* filename, char* sheet, char* cells, char* values, int count); - -// 追加数据到Excel文件末尾 -// -extern __declspec(dllexport) int AppendDataToExcel(uintptr_t handle, char* filename, char* sheet, char* values, int count); - -// 搜索包含关键字的单元格 -// -extern __declspec(dllexport) int SearchByKeyword(uintptr_t handle, char* filename, char* sheet, char* keyword, char** result); - -// 搜索整行包含关键字的行 -// -extern __declspec(dllexport) int SearchRowsByKeyword(uintptr_t handle, char* filename, char* sheet, char* keyword, char** result); - -// 创建新文件并写入数据 -// -extern __declspec(dllexport) int CreateAndWriteExcel(uintptr_t handle, char* filename, char* sheet, char* rowsData); - -// 增强版合并Excel文件（支持指定文件列表） -// -extern __declspec(dllexport) int MergeExcelFilesEx(uintptr_t handle, char* sourceDir, char* specificFiles, char* outputFile, char* sheetName, int mergeByColumn, int includeHeaders, int skipEmptyRows, char* filePattern, char* sourceSheet, int addSourceColumn, int addIndexColumn); - -// 并行合并Excel文件（增强版） -// -extern __declspec(dllexport) int MergeExcelFilesParallelEx(uintptr_t handle, char* sourceDir, char** specificFiles, int fileCount, char* outputFile, char* sheetName, int includeHeaders, int skipEmptyRows, char* filePattern, char* sourceSheet, int addSourceColumn, int addIndexColumn, int workers); - -// 合并同一文件中的多个sheet -// -extern __declspec(dllexport) int MergeSheetsInFile(uintptr_t handle, char* filename, char* outputFile, char* targetSheetName); - -// 释放C字符串 -// -extern __declspec(dllexport) void FreeCString(char* str); - -#ifdef __cplusplus -} -#endif diff --git a/dll/kongfz.h b/dll/kongfz.h deleted file mode 100644 index 2d5deb1..0000000 --- a/dll/kongfz.h +++ /dev/null @@ -1,146 +0,0 @@ -/* Code generated by cmd/cgo; DO NOT EDIT. */ - -/* package command-line-arguments */ - - -#line 1 "cgo-builtin-export-prolog" - -#include - -#ifndef GO_CGO_EXPORT_PROLOGUE_H -#define GO_CGO_EXPORT_PROLOGUE_H - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef struct { const char *p; ptrdiff_t n; } _GoString_; -extern size_t _GoStringLen(_GoString_ s); -extern const char *_GoStringPtr(_GoString_ s); -#endif - -#endif - -/* Start of preamble from import "C" comments. */ - - -#line 3 "main.go" - -#include - -// proxyConfig.dll 函数声明 -extern char* ProxyTypeManager(char* proxyType, char* username, char* password, char* machineCode); -extern void FreeCString(char* str); - -#line 1 "cgo-generated-wrapper" - - -/* End of preamble from import "C" comments. */ - - -/* Start of boilerplate cgo prologue. */ -#line 1 "cgo-gcc-export-header-prolog" - -#ifndef GO_CGO_PROLOGUE_H -#define GO_CGO_PROLOGUE_H - -typedef signed char GoInt8; -typedef unsigned char GoUint8; -typedef short GoInt16; -typedef unsigned short GoUint16; -typedef int GoInt32; -typedef unsigned int GoUint32; -typedef long long GoInt64; -typedef unsigned long long GoUint64; -typedef GoInt64 GoInt; -typedef GoUint64 GoUint; -typedef size_t GoUintptr; -typedef float GoFloat32; -typedef double GoFloat64; -#ifdef _MSC_VER -#if !defined(__cplusplus) || _MSVC_LANG <= 201402L -#include -typedef _Fcomplex GoComplex64; -typedef _Dcomplex GoComplex128; -#else -#include -typedef std::complex GoComplex64; -typedef std::complex GoComplex128; -#endif -#else -typedef float _Complex GoComplex64; -typedef double _Complex GoComplex128; -#endif - -/* - static assertion to make sure the file is being used on architecture - at least with matching size of GoInt. -*/ -typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef _GoString_ GoString; -#endif -typedef void *GoMap; -typedef void *GoChan; -typedef struct { void *t; void *v; } GoInterface; -typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; - -#endif - -/* End of boilerplate cgo prologue. */ - -#ifdef __cplusplus -extern "C" { -#endif - - -// 登录(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutLogin(char* username, char* password); - -// 获取用户信息(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetUserMsg(char* token); - -// 获取商品模版(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetGoodsTplMsg(char* token, char* itemId, char* proxy); - -// 获取商品列表-已登的店铺(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetGoodsListMsgFromSelfShop(char* token, char* proxy, char* itemSn, char* priceMin, char* priceMax, char* startCreateTime, char* endCreateTime, char* requestType, int isItemSnEqual, int page, int size); - -// 删除商品-已登的店铺(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutDelGoodsFromSelfShop(char* token, char* proxy, char* itemId); - -// 新增商品(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutAddGoods(char* token, char* proxy, char* formData); - -// 获取图片URL(官图和拍图)带有店铺过滤 -// -extern __declspec(dllexport) char* OutGetImageFilterShopId(char* token, char* isbn, int shopId, char* proxy, int isLiveImage, int isReturnMsg); - -// 获取商品图片(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetImageByIsbn(char* token, char* isbn, char* proxy, int isLiveImage, int isReturnMsg); - -// 获取商品列表通过店铺ID(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetGoodsListMsgByShopId(int shopId, char* proxy, int retPrice, int isImage, char* sortType, char* sort, float priceMin, float priceMax, int pageNum, int returnNum); - -// 获取商品信息通过商品详情链接(带有0ut的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetGoodsMsgByDetailUrl(char* detailUrl, char* proxy); - -// 获取销量榜商品列表(带有Out的都非官放标准接口) -// -extern __declspec(dllexport) char* OutGetTopGoodsListMsg(int catId, char* proxy); -extern __declspec(dllexport) char* Initialize(char* configJSON); - -// 导出函数：释放C字符串内存 -// -extern __declspec(dllexport) void FreeCString(char* str); - -#ifdef __cplusplus -} -#endif diff --git a/dll/proxy.h b/dll/proxy.h deleted file mode 100644 index dc81637..0000000 --- a/dll/proxy.h +++ /dev/null @@ -1,124 +0,0 @@ -/* Code generated by cmd/cgo; DO NOT EDIT. */ - -/* package command-line-arguments */ - - -#line 1 "cgo-builtin-export-prolog" - -#include - -#ifndef GO_CGO_EXPORT_PROLOGUE_H -#define GO_CGO_EXPORT_PROLOGUE_H - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef struct { const char *p; ptrdiff_t n; } _GoString_; -extern size_t _GoStringLen(_GoString_ s); -extern const char *_GoStringPtr(_GoString_ s); -#endif - -#endif - -/* Start of preamble from import "C" comments. */ - - -#line 3 "proxy.go" - #include - -#line 1 "cgo-generated-wrapper" - - -/* End of preamble from import "C" comments. */ - - -/* Start of boilerplate cgo prologue. */ -#line 1 "cgo-gcc-export-header-prolog" - -#ifndef GO_CGO_PROLOGUE_H -#define GO_CGO_PROLOGUE_H - -typedef signed char GoInt8; -typedef unsigned char GoUint8; -typedef short GoInt16; -typedef unsigned short GoUint16; -typedef int GoInt32; -typedef unsigned int GoUint32; -typedef long long GoInt64; -typedef unsigned long long GoUint64; -typedef GoInt64 GoInt; -typedef GoUint64 GoUint; -typedef size_t GoUintptr; -typedef float GoFloat32; -typedef double GoFloat64; -#ifdef _MSC_VER -#if !defined(__cplusplus) || _MSVC_LANG <= 201402L -#include -typedef _Fcomplex GoComplex64; -typedef _Dcomplex GoComplex128; -#else -#include -typedef std::complex GoComplex64; -typedef std::complex GoComplex128; -#endif -#else -typedef float _Complex GoComplex64; -typedef double _Complex GoComplex128; -#endif - -/* - static assertion to make sure the file is being used on architecture - at least with matching size of GoInt. -*/ -typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef _GoString_ GoString; -#endif -typedef void *GoMap; -typedef void *GoChan; -typedef struct { void *t; void *v; } GoInterface; -typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; - -#endif - -/* End of boilerplate cgo prologue. */ - -#ifdef __cplusplus -extern "C" { -#endif - - -// 导出函数：获取代理健康状态（用于调试） -// -extern __declspec(dllexport) char* GetProxyHealth(void); - -// 导出函数：代理类型管理器 -// -extern __declspec(dllexport) char* ProxyTypeManager(char* proxyType, char* username, char* password, char* machineCode); - -// 导出函数：查询机器码 -// -extern __declspec(dllexport) char* GetMachineCode(char* tailCardSecret); - -// 导出函数：充值卡密 -// -extern __declspec(dllexport) char* RechargeCard(char* tailCardSecret, char* machineCode); - -// 导出函数：获取代理服务器列表 -// -extern __declspec(dllexport) char* GetProxies(char* machineCode); - -// 导出函数：检查卡密是否过期 -// -extern __declspec(dllexport) char* CheckTailCardSecretExpired(char* tailCardSecret); - -// 导出函数：初始化代理管理器 -// -extern __declspec(dllexport) char* InitProxyManager(char* serversJson, char* username, char* password, char* tailCardSecret, char* proxyType); - -// 导出函数：释放C字符串内存 -// -extern __declspec(dllexport) void FreeCString(char* str); - -#ifdef __cplusplus -} -#endif diff --git a/erp/erp.go b/erp/erp.go new file mode 100644 index 0000000..1cddccd --- /dev/null +++ b/erp/erp.go @@ -0,0 +1,105 @@ +package main + +/* +#include +*/ +import "C" +import ( + "encoding/json" + "fmt" + "github.com/parnurzeal/gorequest" + "unsafe" +) + +/* + * 接口转发 + * param requestMethod[string] 请求方式 现在只支持GET\POST\PUT + * param erpUrl[string] erpURl地址 + * param requestJson[string] 请求json字符串 + * return 订单同步结构体字符串,错误信息 + */ +func interfaceForward(requestMethod, erpUrl string, requestJson string) (string, error) { + var reqJson map[string]interface{} + err := json.Unmarshal([]byte(requestJson), &reqJson) + if err != nil { + return "", fmt.Errorf("requestJson JSON解析失败: %v", err) + } + request := gorequest.New() + switch requestMethod { + case "POST": + resp, body, errs := request.Post(erpUrl). + Type("multipart"). + Send(reqJson). + End() + if errs != nil { + return "", fmt.Errorf("请求失败: %v", errs) + } + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP状态码异常: %d, 响应: %s", resp.StatusCode, body) + } + return body, nil + case "GET": + var erpNewUrl string + erpUrl = erpUrl + "?" + count := 0 + total := len(reqJson) + + for k, v := range reqJson { + fmt.Println(k, v) + erpNewUrl = fmt.Sprintf("%s=%s", k, v) + erpUrl = erpUrl + erpNewUrl + + count++ + if count < total { + erpUrl = erpUrl + "&" + } + } + fmt.Println(erpNewUrl) + resp, body, errs := request.Get(erpUrl). + End() + if errs != nil { + return "", fmt.Errorf("请求失败: %v", errs) + } + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP状态码异常: %d, 响应: %s", resp.StatusCode, body) + } + return body, nil + case "PUT": + resp, body, errs := request.Put(erpUrl). + Send(requestJson). + End() + if errs != nil { + return "", fmt.Errorf("请求失败: %v", errs) + } + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP状态码异常: %d, 响应: %s", resp.StatusCode, body) + } + return body, nil + } + return "", fmt.Errorf("请求失败，请求方式: %s", requestMethod) +} + +// InterfaceForward 接口转发 +// +//export InterfaceForward +func InterfaceForward(requestMethod, erpUrl *C.char, requestJson *C.char) *C.char { + goRequestMethod := C.GoString(requestMethod) + goErpUrl := C.GoString(erpUrl) + goEequestJson := C.GoString(requestJson) + synchronization, err := interfaceForward(goRequestMethod, goErpUrl, goEequestJson) + if err != nil { + return C.CString(err.Error()) + } + return C.CString(synchronization) +} + +// FreeCString 释放C字符串内存 +// +//export FreeCString +func FreeCString(str *C.char) { + C.free(unsafe.Pointer(str)) +} + +// 空main函数，编译DLL时需要 +func main() { +} diff --git a/es/main.go b/es/main.go deleted file mode 100644 index a57906f..0000000 --- a/es/main.go +++ /dev/null @@ -1,2088 +0,0 @@ -package main - -//import ( -// "context" -// "crypto/md5" -// "crypto/tls" -// "database/sql" -// "encoding/hex" -// "encoding/json" -// "fmt" -// "github.com/elastic/go-elasticsearch/v8" -// "github.com/elastic/go-elasticsearch/v8/esapi" -// "golang.org/x/image/bmp" -// "golang.org/x/image/draw" -// "golang.org/x/image/tiff" -// "golang.org/x/image/webp" -// "image" -// "image/color" -// "image/jpeg" -// "image/png" -// "io" -// "io/ioutil" -// "log" -// "mime/multipart" -// "net/http" -// "os" -// "path/filepath" -// "sort" -// "strings" -// "sync" -// "sync/atomic" -// "time" -// -// _ "github.com/go-sql-driver/mysql" -// "github.com/nfnt/resize" -//) -// -//// ES 配置 -//const ( -// esAddress = "http://103.236.91.138:9200" -// esUsername = "elastic" -// esPassword = "5mRDIUg52VC0fp14nw-F" -// esIndex = "books-from-mysql" -// ClientID = "203c5a7ba8bd4b8488d5e26f93052642" // 拼多多开放平台配置 -// ClientSecret = "892ffaa86e12b7a3d8d2942b669d9aa520ad8179" -// PDDApiURL = "https://gw-upload.pinduoduo.com/api/upload" -//) -// -//// 配置参数 -//const ( -// maxWorkers = 15 // 最大并发worker数量 -// maxRetries = 3 // 最大重试次数 -// retryDelay = 2 * time.Second // 重试延迟 -// progressInterval = 5 * time.Second // 进度报告间隔 -//) -// -//// ES 客户端封装 -//type ESClient struct { -// client *elasticsearch.Client -//} -// -//// 数据库记录结构体 -//type CrawlerRecord struct { -// BookISBN sql.NullString -// BookPicture sql.NullString -//} -// -//// 处理结果结构体 -//type ProcessResult struct { -// Record CrawlerRecord -// Success bool -// LocalPaths []string -// PDDURLs []string -// Error error -// WorkerID int -// ProcessedAt time.Time -//} -// -//// 全局统计 -//type Statistics struct { -// Total int32 -// Success int32 -// Failed int32 -// Skipped int32 -// CurrentIndex int32 -// StartTime time.Time -//} -// -//// NewESClient 初始化 ES 客户端 -//// 说明：保持一致的连接方式（禁用证书校验、设置超时和连接池参数） -//func NewESClient(addresses []string, username, password string) (*ESClient, error) { -// cfg := elasticsearch.Config{ -// Addresses: addresses, -// Username: username, -// Password: password, -// Transport: &http.Transport{ -// TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, -// MaxIdleConnsPerHost: 100, -// ResponseHeaderTimeout: 60 * time.Second, -// }, -// } -// cli, err := elasticsearch.NewClient(cfg) -// if err != nil { -// return nil, err -// } -// return &ESClient{client: cli}, nil -//} -// -//// CheckHealth 检查 ES 集群健康 -//// 行为：等待状态至少为 yellow，输出基本信息 -//func (es *ESClient) CheckHealth() error { -// res, err := es.client.Cluster.Health( -// es.client.Cluster.Health.WithWaitForStatus("yellow"), -// es.client.Cluster.Health.WithTimeout(30*time.Second), -// ) -// if err != nil { -// return err -// } -// defer res.Body.Close() -// if res.IsError() { -// return fmt.Errorf("Elasticsearch 健康检查失败: %s", res.String()) -// } -// var m map[string]interface{} -// if err := json.NewDecoder(res.Body).Decode(&m); err == nil { -// log.Printf("ES status=%v nodes=%v cluster=%v", m["status"], m["number_of_nodes"], m["cluster_name"]) -// } -// return nil -//} -// -//// PDDImageProcessor 实现图片处理器 -//// pdd上传图片官方接口 -//// 上传图片到拼多多 -//func uploadToPDD(token, imagePath string) (string, error) { -// // 检查token是否有效 -// if len(token) == 0 { -// return "", fmt.Errorf("获取到的token为空") -// } -// result, err := ProcessAndUploadImage(imagePath, token) -// if err != nil { -// return "", fmt.Errorf("拼多多图片上传失败: %v", err) -// } -// -// // 解析JSON响应获取URL -// var response struct { -// RequestID string `json:"request_id"` -// URL string `json:"url"` -// } -// -// err = json.Unmarshal([]byte(result), &response) -// if err != nil { -// return "", fmt.Errorf("解析上传响应失败: %v", err) -// } -// -// if response.URL == "" { -// return "", fmt.Errorf("上传响应中未找到URL") -// } -// -// return response.URL, nil -//} -//func ProcessAndUploadImage(imagePath, token string) (string, error) { -// // 打开图片文件 -// file, err := os.Open(imagePath) -// if err != nil { -// return "", fmt.Errorf("failed to open image file: %v", err) -// } -// defer file.Close() -// -// // 准备参数 - 不包含文件路径 -// params := map[string]string{ -// "access_token": token, -// "data_type": "JSON", -// "type": "pdd.goods.img.upload", -// "client_id": ClientID, -// "timestamp": fmt.Sprintf("%d", time.Now().Unix()), -// } -// -// // 生成签名（不包含文件路径） -// params["sign"] = generateSign(params) -// -// // 创建multipart表单 -// body := &strings.Builder{} -// writer := multipart.NewWriter(body) -// -// // 写入文本参数 -// for key, value := range params { -// if err := writer.WriteField(key, value); err != nil { -// return "", fmt.Errorf("failed to write field %s: %v", key, err) -// } -// } -// -// // 写入文件流 - 使用正确的字段名 "file" -// part, err := writer.CreateFormFile("file", filepath.Base(imagePath)) -// if err != nil { -// return "", fmt.Errorf("failed to create form file: %v", err) -// } -// -// if _, err := io.Copy(part, file); err != nil { -// return "", fmt.Errorf("failed to copy file data: %v", err) -// } -// -// // 关闭writer -// if err := writer.Close(); err != nil { -// return "", fmt.Errorf("failed to close writer: %v", err) -// } -// -// // 创建请求 -// req, err := http.NewRequest("POST", PDDApiURL, strings.NewReader(body.String())) -// if err != nil { -// return "", fmt.Errorf("failed to create request: %v", err) -// } -// -// // 设置请求头 -// req.Header.Set("Content-Type", writer.FormDataContentType()) -// req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") -// -// // 发送请求 -// client := &http.Client{Timeout: 30 * time.Second} -// resp, err := client.Do(req) -// if err != nil { -// return "", fmt.Errorf("failed to send request: %v", err) -// } -// defer resp.Body.Close() -// -// // 读取响应 -// respBody, err := io.ReadAll(resp.Body) -// if err != nil { -// return "", fmt.Errorf("failed to read response: %v", err) -// } -// -// log.Printf("拼多多API响应状态: %d", resp.StatusCode) -// log.Printf("拼多多API响应内容: %s", string(respBody)) -// -// if resp.StatusCode != http.StatusOK { -// return "", fmt.Errorf("API returned error status: %d, body: %s", resp.StatusCode, string(respBody)) -// } -// -// // 解析响应 -// var result map[string]interface{} -// if err := json.Unmarshal(respBody, &result); err != nil { -// return "", fmt.Errorf("failed to parse response: %v", err) -// } -// -// // 检查API返回的错误 -// if errorResponse, exists := result["error_response"]; exists { -// errorMsg, _ := json.Marshal(errorResponse) -// return "", fmt.Errorf("API returned error: %s", string(errorMsg)) -// } -// -// // 查找成功的响应 -// for key, value := range result { -// if key != "error_response" { -// successResponse, _ := json.Marshal(value) -// return string(successResponse), nil -// } -// } -// -// return string(respBody), nil -//} -// -//// generateSign 生成拼多多API签名 -//func generateSign(params map[string]string) string { -// // 按参数名排序 -// var keys []string -// for k := range params { -// keys = append(keys, k) -// } -// sort.Strings(keys) -// // 拼接参数字符串 -// var signStr string -// for _, k := range keys { -// signStr += k + params[k] -// } -// signStr = ClientSecret + signStr + ClientSecret -// // 计算MD5并转为大写 -// hasher := md5.New() -// hasher.Write([]byte(signStr)) -// result := strings.ToUpper(hex.EncodeToString(hasher.Sum(nil))) -// return result -//} -// -//// GetPddToken 获取PDD token（简化版） -//func GetPddToken() (string, error) { -// url := "https://api.buzhiyushu.cn/huidiao/pdd/getPddChildrenBooksToken" -// -// resp, err := http.Get(url) -// if err != nil { -// return "", err -// } -// defer resp.Body.Close() -// -// body, err := ioutil.ReadAll(resp.Body) -// if err != nil { -// return "", err -// } -// -// // 使用map解析JSON -// var result map[string]interface{} -// json.Unmarshal(body, &result) -// -// // 检查业务状态 -// if code, ok := result["code"].(float64); !ok || code != 200 { -// return "", fmt.Errorf("API错误: %v", result["msg"]) -// } -// -// // 提取token -// token := result["data"].(string) -// return token, nil -//} -// -//// 并发处理记录的主要函数 -//func processRecordsConcurrently(records []CrawlerRecord, imageDir string, es *ESClient, maxWorkers int, token string) []ProcessResult { -// var stats Statistics -// stats.Total = int32(len(records)) -// stats.StartTime = time.Now() -// -// // 创建通道 -// recordChan := make(chan CrawlerRecord, len(records)) -// resultChan := make(chan ProcessResult, len(records)) -// -// // 启动worker -// var wg sync.WaitGroup -// for i := 0; i < maxWorkers; i++ { -// wg.Add(1) -// go worker(token, i, &wg, recordChan, resultChan, imageDir, es, &stats) -// } -// -// // 发送任务到通道 -// go func() { -// for _, record := range records { -// recordChan <- record -// } -// close(recordChan) -// }() -// -// // 启动进度报告 -// go progressReporter(&stats) -// -// // 收集结果 -// var results []ProcessResult -// go func() { -// for result := range resultChan { -// results = append(results, result) -// } -// }() -// -// // 等待所有worker完成 -// wg.Wait() -// close(resultChan) -// -// return results -//} -// -//// worker 处理函数 -//func worker(token string, id int, wg *sync.WaitGroup, recordChan <-chan CrawlerRecord, resultChan chan<- ProcessResult, imageDir string, es *ESClient, stats *Statistics) { -// defer wg.Done() -// -// for record := range recordChan { -// currentIndex := atomic.AddInt32(&stats.CurrentIndex, 1) -// -// result := ProcessResult{ -// Record: record, -// WorkerID: id, -// ProcessedAt: time.Now(), -// } -// -// // 检查记录有效性 -// if !isRecordValid(record) { -// atomic.AddInt32(&stats.Skipped, 1) -// result.Success = false -// result.Error = fmt.Errorf("无效记录: ISBN或图片URL为空") -// resultChan <- result -// continue -// } -// -// // 处理记录（带重试机制） -// var localPaths, pddURLs []string -// var err error -// -// for attempt := 1; attempt <= maxRetries; attempt++ { -// localPaths, pddURLs, err = processSingleRecord(token, record, imageDir, es) -// if err == nil { -// break -// } -// -// // 如果是ES记录未找到的错误，不需要重试 -// if strings.Contains(err.Error(), "ES记录未找到") { -// break -// } -// -// if attempt < maxRetries { -// log.Printf("Worker %d: 第 %d 次尝试处理 ISBN %s 失败, %d 秒后重试: %v", -// id, attempt, record.BookISBN.String, retryDelay/time.Second, err) -// time.Sleep(retryDelay) -// } -// } -// -// if err != nil { -// atomic.AddInt32(&stats.Failed, 1) -// result.Success = false -// result.Error = err -// // 即使失败，也记录已处理的本地路径（如果有） -// result.LocalPaths = localPaths -// result.PDDURLs = pddURLs -// -// // 根据错误类型记录不同的日志 -// if strings.Contains(err.Error(), "ES记录未找到") { -// log.Printf("Worker %d: ES记录未找到 [%d/%d] ISBN: %s", -// id, currentIndex, stats.Total, record.BookISBN.String) -// } else { -// log.Printf("Worker %d: 处理失败 [%d/%d] ISBN: %s, 错误: %v", -// id, currentIndex, stats.Total, record.BookISBN.String, err) -// } -// } else { -// atomic.AddInt32(&stats.Success, 1) -// result.Success = true -// result.LocalPaths = localPaths -// result.PDDURLs = pddURLs -// -// log.Printf("Worker %d: 成功处理 [%d/%d] ISBN: %s, 生成 %d 个文件, 上传 %d 个URL", -// id, currentIndex, stats.Total, record.BookISBN.String, len(localPaths), len(pddURLs)) -// } -// -// resultChan <- result -// } -//} -// -//// 检查记录有效性 -//func isRecordValid(record CrawlerRecord) bool { -// if !record.BookISBN.Valid || record.BookISBN.String == "" { -// return false -// } -// if !record.BookPicture.Valid || record.BookPicture.String == "" { -// return false -// } -// return true -//} -// -//// 进度报告器 -//func progressReporter(stats *Statistics) { -// ticker := time.NewTicker(progressInterval) -// defer ticker.Stop() -// -// for range ticker.C { -// processed := atomic.LoadInt32(&stats.CurrentIndex) -// success := atomic.LoadInt32(&stats.Success) -// failed := atomic.LoadInt32(&stats.Failed) -// skipped := atomic.LoadInt32(&stats.Skipped) -// -// elapsed := time.Since(stats.StartTime) -// rate := float64(processed) / elapsed.Seconds() -// -// // 计算预估剩余时间 -// var eta time.Duration -// if processed > 0 && rate > 0 { -// remaining := float64(stats.Total - processed) -// eta = time.Duration(remaining/rate) * time.Second -// } -// -// fmt.Printf("[进度] 已处理: %d/%d (成功: %d, 失败: %d, 跳过: %d) | 速率: %.2f 条/秒 | 运行: %v | ETA: %v\n", -// processed, stats.Total, success, failed, skipped, rate, elapsed.Round(time.Second), eta.Round(time.Second)) -// -// if processed >= stats.Total { -// break -// } -// } -//} -// -//// 打印最终统计 -//func printFinalStatistics(results []ProcessResult) { -// var success, failed, skipped int -// var totalFilesGenerated int -// var totalURLsUploaded int -// -// // 失败原因分类 -// failureReasons := make(map[string]int) -// -// for _, result := range results { -// if result.Success { -// success++ -// totalFilesGenerated += len(result.LocalPaths) -// totalURLsUploaded += len(result.PDDURLs) -// } else if result.Error != nil && strings.Contains(result.Error.Error(), "无效记录") { -// skipped++ -// failureReasons["无效记录(ISBN或URL为空)"]++ -// } else { -// failed++ -// // 即使是失败的情况，也可能生成了部分文件 -// totalFilesGenerated += len(result.LocalPaths) -// totalURLsUploaded += len(result.PDDURLs) -// -// // 分类失败原因 -// errMsg := result.Error.Error() -// switch { -// case strings.Contains(errMsg, "ES记录未找到"): -// failureReasons["ES记录未找到"]++ -// case strings.Contains(errMsg, "查询ES中ID失败"): -// failureReasons["ES查询失败"]++ -// case strings.Contains(errMsg, "下载图片失败"): -// failureReasons["图片下载失败"]++ -// case strings.Contains(errMsg, "处理图片失败"): -// failureReasons["图片处理失败"]++ -// case strings.Contains(errMsg, "上传PNG图片失败"): -// failureReasons["PNG上传失败"]++ -// case strings.Contains(errMsg, "上传JPG图片失败"): -// failureReasons["JPG上传失败"]++ -// case strings.Contains(errMsg, "更新ES数据失败"): -// failureReasons["ES更新失败"]++ -// default: -// failureReasons["其他错误"]++ -// } -// } -// } -// -// fmt.Printf("\n=== 处理完成 ===\n") -// fmt.Printf("总记录数: %d\n", len(results)) -// fmt.Printf("成功: %d\n", success) -// fmt.Printf("失败: %d\n", failed) -// fmt.Printf("跳过: %d\n", skipped) -// fmt.Printf("成功率: %.2f%%\n", float64(success)/float64(len(results))*100) -// fmt.Printf("生成文件总数: %d (平均每条记录 %.1f 个文件)\n", totalFilesGenerated, float64(totalFilesGenerated)/float64(len(results))) -// fmt.Printf("上传URL总数: %d (平均每条记录 %.1f 个URL)\n", totalURLsUploaded, float64(totalURLsUploaded)/float64(len(results))) -// -// // 显示失败原因统计 -// if len(failureReasons) > 0 { -// fmt.Printf("\n=== 失败原因统计 ===\n") -// for reason, count := range failureReasons { -// fmt.Printf(" %s: %d\n", reason, count) -// } -// } -// -// // 显示处理详情示例 -// fmt.Printf("\n=== 处理详情示例 ===\n") -// successCount := 0 -// failedCount := 0 -// for _, result := range results { -// if result.Success && successCount < 3 { -// fmt.Printf("✅ 成功: ISBN %s -> 文件: %d 个, URL: %d 个\n", -// result.Record.BookISBN.String, -// len(result.LocalPaths), -// len(result.PDDURLs)) -// successCount++ -// } else if !result.Success && failedCount < 3 && !strings.Contains(result.Error.Error(), "无效记录") { -// fmt.Printf("❌ 失败: ISBN %s -> 错误: %v\n", -// result.Record.BookISBN.String, -// result.Error) -// failedCount++ -// } -// if successCount >= 3 && failedCount >= 3 { -// break -// } -// } -//} -// -//// 处理单条记录 -//func processSingleRecord(token string, record CrawlerRecord, imageDir string, es *ESClient) ([]string, []string, error) { -// // 更新ES -// ids, err := es.FindIDsByISBN(esIndex, record.BookISBN.String) -// if err != nil { -// return nil, nil, fmt.Errorf("查询ES中ID失败: %v", err) -// } -// var pngImageUrl string -// var jpgImageUrl string -// var localPaths []string -// var pddURLs []string -// if ids != "" { -// // 下载并处理图片 -// pngPath, jpgPath, err := processAndSaveImage(record, imageDir) -// if err != nil { -// err := saveISBNToFile(record.BookISBN.String, record.BookPicture.String) -// if err != nil { -// log.Printf("警告: 无法保存未找到的ISBN到文件: %v", err) -// } else { -// log.Printf("未找到ISBN %s 对应的ES记录，已保存到文件", record.BookISBN.String) -// } -// return nil, nil, fmt.Errorf("处理图片失败: %v", err) -// } -// localPaths = []string{pngPath, jpgPath} -// // 上传到PDD -// pngImageUrl, err = uploadToPDD(token, pngPath) -// if err != nil { -// err := saveISBNToFile(record.BookISBN.String, record.BookPicture.String) -// if err != nil { -// log.Printf("警告: 无法保存未找到的ISBN到文件: %v", err) -// } else { -// log.Printf("未找到ISBN %s 对应的ES记录，已保存到文件", record.BookISBN.String) -// } -// return nil, nil, fmt.Errorf("上传PNG图片失败: %v", err) -// } -// // 上传到PDD -// jpgImageUrl, err = uploadToPDD(token, jpgPath) -// if err != nil { -// err := saveISBNToFile(record.BookISBN.String, record.BookPicture.String) -// if err != nil { -// log.Printf("警告: 无法保存未找到的ISBN到文件: %v", err) -// } else { -// log.Printf("未找到ISBN %s 对应的ES记录，已保存到文件", record.BookISBN.String) -// } -// return nil, nil, fmt.Errorf("上传JPG图片失败: %v", err) -// } -// pddURLs = []string{pngImageUrl, jpgImageUrl} -// err = es.UpdateBookPicsByID(esIndex, ids, "", pngImageUrl, jpgImageUrl) -// if err != nil { -// err := saveISBNToFile(record.BookISBN.String, record.BookPicture.String) -// if err != nil { -// log.Printf("警告: 无法保存未找到的ISBN到文件: %v", err) -// } else { -// log.Printf("未找到ISBN %s 对应的ES记录，已保存到文件", record.BookISBN.String) -// } -// return nil, nil, fmt.Errorf("更新ES数据失败: %v", err) -// } -// -// for _, path := range localPaths { -// // ES更新成功后删除本地图片 -// if removeErr := os.Remove(path); removeErr == nil { -// log.Printf("ES更新成功，已删除本地图片: %s", path) -// } else { -// log.Printf("警告: 无法删除本地图片 %s: %v", path, removeErr) -// } -// } -// } else { -// // ids为空，将ISBN存储到txt文件 -// err := saveISBNToFile(record.BookISBN.String, record.BookPicture.String) -// if err != nil { -// log.Printf("警告: 无法保存未找到的ISBN到文件: %v", err) -// } else { -// log.Printf("未找到ISBN %s 对应的ES记录，已保存到文件", record.BookISBN.String) -// } -// return nil, nil, fmt.Errorf("未找到ISBN %s 对应的ES记录", record.BookISBN.String) -// } -// return localPaths, pddURLs, nil -//} -// -//// 保存未找到的ISBN和图片URL到txt文件（CSV格式，带去重） -//func saveISBNToFile(isbn string, imageUrl string) error { -// filename := "cmd/update_es_gt/xgy_not_found_isbns.txt" -// -// // 读取现有内容检查是否已存在 -// existingRecords := make(map[string]bool) -// if content, err := os.ReadFile(filename); err == nil { -// lines := strings.Split(string(content), "\n") -// for _, line := range lines { -// if line != "" && !strings.HasPrefix(line, "#") { -// parts := strings.Split(line, ",") -// if len(parts) > 0 { -// existingRecords[parts[0]] = true // 以ISBN作为去重依据 -// } -// } -// } -// } -// -// // 如果已存在，则不重复添加 -// if existingRecords[isbn] { -// return nil -// } -// // 以追加模式打开文件 -// file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) -// if err != nil { -// return fmt.Errorf("打开文件失败: %v", err) -// } -// defer file.Close() -// // 如果是空文件，先写入CSV表头 -// stat, err := file.Stat() -// if err == nil && stat.Size() == 0 { -// header := "# ISBN,ImageURL\n" -// if _, err := file.WriteString(header); err != nil { -// return fmt.Errorf("写入表头失败: %v", err) -// } -// } -// -// // 写入ISBN和图片URL，用逗号分隔，并添加换行符 -// line := fmt.Sprintf("%s,%s\n", isbn, imageUrl) -// _, err = file.WriteString(line) -// if err != nil { -// return fmt.Errorf("写入文件失败: %v", err) -// } -// -// return nil -//} -// -//// 下载并处理图片 -//func processAndSaveImage(record CrawlerRecord, saveDir string) (string, string, error) { -// // 下载图片 -// img, originalFormat, err := downloadImage(record.BookPicture.String) -// if err != nil { -// return "", "", fmt.Errorf("下载图片失败: %v", err) -// } -// -// fmt.Printf("下载成功，原始格式: %s\n", originalFormat) -// -// // 调整图片高度为600，等比例缩放 -// //resizedImg := resizeImageToHeight(img, 600) -// // 使用高质量缩放调整图片高度为600，等比例缩放 -// resizedImg := resizeToHeightHighQuality(img, 600) -// fmt.Printf("缩放后尺寸: %dx%d\n", resizedImg.Bounds().Dx(), resizedImg.Bounds().Dy()) -// -// // 创建800x800的透明背景 -// finalImg := createCenteredImage(resizedImg, 800, 800, true) -// -// // 创建800x800的白色背景（用于JPG） -// whiteImg := createCenteredImage(resizedImg, 800, 800, false) -// -// // 生成文件名 -// filename := fmt.Sprintf("%s", record.BookISBN.String) -// // 清理文件名中的非法字符 -// filename = sanitizeFilename(filename) -// -// // PNG文件路径 -// pngPath := filepath.Join(saveDir, filename+".png") -// // JPG文件路径 -// jpgPath := filepath.Join(saveDir, filename+".jpg") -// -// // 保存为PNG图片 -// err = savePNG(finalImg, pngPath) -// if err != nil { -// return "", "", fmt.Errorf("保存图片失败: %v", err) -// } -// -// // 保存为JPG图片（白色背景） -// err = saveJPG(whiteImg, jpgPath, 95) // 95%质量 -// if err != nil { -// return "", "", fmt.Errorf("保存JPG图片失败: %v", err) -// } -// -// fmt.Printf("转换成功: %s -> %s, 保存路径: %s\n", originalFormat, "PNG", pngPath) -// fmt.Printf("转换成功: %s -> %s, 保存路径: %s\n", originalFormat, "JPG", jpgPath) -// return pngPath, jpgPath, nil -//} -// -//// 下载图片 -//func downloadImage(url string) (image.Image, string, error) { -// // 创建HTTP客户端，设置超时等参数 -// client := &http.Client{ -// Timeout: 30 * time.Second, -// } -// -// resp, err := client.Get(url) -// if err != nil { -// return nil, "", err -// } -// defer resp.Body.Close() -// -// if resp.StatusCode != http.StatusOK { -// return nil, "", fmt.Errorf("HTTP请求失败，状态码: %d", resp.StatusCode) -// } -// -// // 读取响应体前几个字节来判断图片格式 -// peekBytes := make([]byte, 512) -// n, err := resp.Body.Read(peekBytes) -// if err != nil && err != io.EOF { -// return nil, "", err -// } -// -// // 创建一个新的Reader，包含已读取的数据和剩余数据 -// reader := io.MultiReader(strings.NewReader(string(peekBytes[:n])), resp.Body) -// -// // 根据文件头识别图片格式 -// contentType := http.DetectContentType(peekBytes[:n]) -// fmt.Printf("检测到的Content-Type: %s\n", contentType) -// -// var img image.Image -// var format string -// -// // 根据Content-Type或文件扩展名选择解码器 -// switch { -// case strings.Contains(contentType, "jpeg") || strings.HasSuffix(strings.ToLower(url), ".jpg") || strings.HasSuffix(strings.ToLower(url), ".jpeg"): -// img, err = jpeg.Decode(reader) -// format = "JPEG" -// case strings.Contains(contentType, "png") || strings.HasSuffix(strings.ToLower(url), ".png"): -// img, err = png.Decode(reader) -// format = "PNG" -// case strings.Contains(contentType, "webp") || strings.HasSuffix(strings.ToLower(url), ".webp"): -// img, err = webp.Decode(reader) -// format = "WEBP" -// case strings.Contains(contentType, "bmp") || strings.HasSuffix(strings.ToLower(url), ".bmp"): -// img, err = bmp.Decode(reader) -// format = "BMP" -// case strings.Contains(contentType, "tiff") || strings.HasSuffix(strings.ToLower(url), ".tiff") || strings.HasSuffix(strings.ToLower(url), ".tif"): -// img, err = tiff.Decode(reader) -// format = "TIFF" -// default: -// // 尝试通用解码 -// img, format, err = image.Decode(reader) -// if err != nil { -// return nil, "", fmt.Errorf("不支持的图片格式: %s, 错误: %v", contentType, err) -// } -// } -// -// if err != nil { -// return nil, "", fmt.Errorf("解码图片失败: %v", err) -// } -// -// return img, format, nil -//} -// -//// 高质量等比例缩放到指定高度 -//func resizeToHeightHighQuality(src image.Image, targetHeight int) image.Image { -// bounds := src.Bounds() -// srcWidth := bounds.Dx() -// srcHeight := bounds.Dy() -// -// // 如果原图高度已经小于等于目标高度，且宽度合适，可以直接返回 -// //if srcHeight <= targetHeight { -// // return src -// //} -// -// // 计算等比例缩放后的宽度 -// targetWidth := uint(float64(srcWidth) * float64(targetHeight) / float64(srcHeight)) -// -// // 使用 Lanczos3 插值算法进行高质量缩放 -// return resize.Resize(targetWidth, uint(targetHeight), src, resize.Lanczos3) -//} -// -//// 创建居中图片（将原图放在指定大小的透明背景中央） -//func createCenteredImage(src image.Image, width, height int, transparent bool) *image.RGBA { -// // 创建透明背景 -// dst := image.NewRGBA(image.Rect(0, 0, width, height)) -// -// // 设置背景颜色 -// var bgColor color.Color -// if transparent { -// bgColor = color.RGBA{0, 0, 0, 0} // 透明 -// } else { -// bgColor = color.RGBA{255, 255, 255, 255} // 白色 -// } -// -// // 填充透明背景 -// //transparent := color.RGBA{0, 0, 0, 0} -// draw.Draw(dst, dst.Bounds(), &image.Uniform{bgColor}, image.Point{}, draw.Src) -// -// // 计算居中位置 -// srcBounds := src.Bounds() -// srcWidth := srcBounds.Dx() -// srcHeight := srcBounds.Dy() -// -// x := (width - srcWidth) / 2 -// y := (height - srcHeight) / 2 -// -// // 将原图绘制到中央 -// draw.Draw(dst, image.Rect(x, y, x+srcWidth, y+srcHeight), src, image.Point{}, draw.Over) -// -// return dst -//} -// -//// 保存为PNG图片 -//func savePNG(img image.Image, filename string) error { -// file, err := os.Create(filename) -// if err != nil { -// return err -// } -// defer file.Close() -// -// return png.Encode(file, img) -//} -// -//// 保存为JPG图片 -//func saveJPG(img image.Image, filename string, quality int) error { -// file, err := os.Create(filename) -// if err != nil { -// return err -// } -// defer file.Close() -// -// // 设置JPEG编码选项 -// options := &jpeg.Options{ -// Quality: quality, // 1-100，越高质量越好 -// } -// -// return jpeg.Encode(file, img, options) -//} -// -//// 清理文件名中的非法字符 -//func sanitizeFilename(filename string) string { -// // 替换Windows文件名中不允许的字符 -// invalidChars := []string{"\\", "/", ":", "*", "?", "\"", "<", ">", "|"} -// for _, char := range invalidChars { -// filename = strings.ReplaceAll(filename, char, "_") -// } -// // 移除或替换其他可能的问题字符 -// filename = strings.TrimSpace(filename) -// if filename == "" { -// filename = "unknown" -// } -// return filename -//} -// -//// 从数据库获取记录 -//func getRecords(db *sql.DB) ([]CrawlerRecord, error) { -// // 查询所有记录，包括 NULL 值 -// query := "SELECT book_isbn, book_picture FROM dk_crawler_record_info" -// rows, err := db.Query(query) -// if err != nil { -// return nil, err -// } -// defer rows.Close() -// -// var records []CrawlerRecord -// for rows.Next() { -// var record CrawlerRecord -// // 使用 sql.NullString 来接收可能为 NULL 的字段 -// err := rows.Scan(&record.BookISBN, &record.BookPicture) -// if err != nil { -// fmt.Printf("扫描记录失败: %v\n", err) -// continue -// } -// records = append(records, record) -// } -// -// // 检查遍历过程中是否有错误 -// if err = rows.Err(); err != nil { -// return nil, err -// } -// -// return records, nil -//} -// -//// FindIDsByISBN 根据 ISBN 查询文档 ID 列表 -//func (es *ESClient) FindIDsByISBN(index, isbn string) (string, error) { -// q := map[string]interface{}{ -// "query": map[string]interface{}{ -// "term": map[string]interface{}{"isbn": isbn}, -// }, -// "_source": false, -// "size": 1000, -// } -// b, _ := json.Marshal(q) -// res, err := es.client.Search( -// es.client.Search.WithIndex(index), -// es.client.Search.WithBody(strings.NewReader(string(b))), -// es.client.Search.WithContext(context.Background()), -// ) -// if err != nil { -// return "", err -// } -// defer res.Body.Close() -// if res.IsError() { -// return "", fmt.Errorf("搜索失败: %s", res.String()) -// } -// var r map[string]interface{} -// if err := json.NewDecoder(res.Body).Decode(&r); err != nil { -// return "", err -// } -// hits, _ := r["hits"].(map[string]interface{}) -// arr, _ := hits["hits"].([]interface{}) -// var ids string -// for _, h := range arr { -// m, _ := h.(map[string]interface{}) -// id, _ := m["_id"].(string) -// if id != "" { -// //ids = append(ids, id) -// ids = id -// } -// } -// return ids, nil -//} -// -//func (es *ESClient) UpdateBookPicsByID(index, id, localImageS, pngImageUrl, jpgImageUrl string) error { -// bookPicJSON, err := json.Marshal(map[string]string{ -// "localPath": localImageS, -// "pddPath": jpgImageUrl, -// }) -// if err != nil { -// return fmt.Errorf("序列化 book_pic_w 失败: %w", err) -// } -// -// bookPicBJSON, err := json.Marshal(map[string]string{ -// "localPath": localImageS, -// "pddResponse": pngImageUrl, -// }) -// if err != nil { -// return fmt.Errorf("序列化 book_pic_b 失败: %w", err) -// } -// // 构建更新文档 -// payload := map[string]interface{}{ -// "doc": map[string]string{ -// "book_pic": string(bookPicJSON), -// "book_pic_b": string(bookPicBJSON), -// }, -// } -// // JSON 序列化整个更新请求 -// body, err := json.Marshal(payload) -// if err != nil { -// return fmt.Errorf("序列化更新请求失败: %w", err) -// } -// req := esapi.UpdateRequest{ -// Index: index, -// DocumentID: id, -// Body: strings.NewReader(string(body)), -// } -// res, err := req.Do(context.Background(), es.client) -// if err != nil { -// return err -// } -// defer res.Body.Close() -// if res.IsError() { -// data, _ := io.ReadAll(res.Body) -// return fmt.Errorf("ES 更新失败: %s", data) -// } -// return nil -//} -// -//// 从 sql.NullString 获取字符串值 -//func getStringValue(nullString sql.NullString) string { -// if nullString.Valid { -// return nullString.String -// } -// return "NULL" -//} -// -//func main() { -// //// 获取token -// //token, err := GetPddToken() -// //if err != nil { -// // fmt.Errorf("获取拼多多token失败: %v", err) -// //} -// //fmt.Println("token=", token) -// //// 数据源名称格式 -// //dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", -// // "root", -// // "123456", -// // "localhost", -// // 3306, -// // "book_image") -// //db, err := sql.Open("mysql", dsn) -// //if err != nil { -// // fmt.Printf("打开数据库连接失败: %v", err) -// //} -// //// 设置连接池参数 -// //db.SetMaxOpenConns(20) // 最大打开连接数 -// //db.SetMaxIdleConns(10) -// //err = db.Ping() -// //if err != nil { -// // fmt.Printf("数据库连接测试失败: %v", err) -// //} -// // -// //// 查询数据 -// //records, err := getRecords(db) -// //if err != nil { -// // fmt.Printf("查询失败: %v", err) -// //} -// //imageDir := "D:\\image" -// //err = os.MkdirAll(imageDir, 0755) -// //if err != nil { -// // fmt.Sprintf("创建目录失败: %v", err) -// //} -// //fmt.Printf("找到 %d 条记录需要处理\n", len(records)) -// // -// //es, err := NewESClient([]string{esAddress}, esUsername, esPassword) -// //if err != nil { -// // log.Fatalf("ES 连接失败: %v", err) -// //} -// //if err := es.CheckHealth(); err != nil { -// // log.Fatalf("ES 健康检查失败: %v", err) -// //} -// //// 启动并发处理 -// //results := processRecordsConcurrently(records, imageDir, es, maxWorkers, token) -// // -// //// 输出最终统计 -// //printFinalStatistics(results) -// -// //mainQuerySaleISBNs() -// //mainFindESOnlyISBNs() -// mainQuerySaleISBNsWithEmptyPic() -//} -// -//// 查询并导出有销售记录且book_pic字符串中pddPath为空的ISBN -//func queryAndExportSaleISBNs(es *ESClient, outputFile string) error { -// log.Printf("开始查询有销售记录且book_pic字符串中pddPath为空的ISBN...") -// -// // 使用其他字段排序，比如 isbn 字段或者时间字段 -// query := map[string]interface{}{ -// "query": map[string]interface{}{ -// "bool": map[string]interface{}{ -// "must": []map[string]interface{}{ -// { -// "bool": map[string]interface{}{ -// "should": []map[string]interface{}{ -// {"range": map[string]interface{}{"day_sale_7": map[string]interface{}{"gt": 0}}}, -// {"range": map[string]interface{}{"day_sale_15": map[string]interface{}{"gt": 0}}}, -// {"range": map[string]interface{}{"day_sale_30": map[string]interface{}{"gt": 0}}}, -// {"range": map[string]interface{}{"day_sale_60": map[string]interface{}{"gt": 0}}}, -// }, -// "minimum_should_match": 1, -// }, -// }, -// { -// "bool": map[string]interface{}{ -// "should": []map[string]interface{}{ -// // 匹配 pddPath:"" 的JSON字符串 -// {"regexp": map[string]interface{}{"book_pic": ".*\"pddPath\":\"\".*"}}, -// // 匹配 pddPath: "" (带空格的) -// {"regexp": map[string]interface{}{"book_pic": ".*\"pddPath\":\\s*\"\".*"}}, -// // 匹配整个book_pic字段为空 -// {"term": map[string]interface{}{"book_pic": ""}}, -// // 匹配book_pic字段不存在 -// { -// "bool": map[string]interface{}{ -// "must_not": map[string]interface{}{ -// "exists": map[string]interface{}{"field": "book_pic"}, -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// "_source": []string{"isbn"}, -// "sort": []map[string]interface{}{ -// {"isbn": "asc"}, // 使用 isbn 字段排序，或者使用其他可排序字段 -// }, -// "size": 10000, -// } -// -// // 打印查询条件用于验证 -// queryJSON, _ := json.MarshalIndent(query, "", " ") -// log.Printf("查询条件:\n%s", string(queryJSON)) -// -// var allISBNs []string -// var searchAfter interface{} -// totalCount := 0 -// page := 1 -// -// for { -// // 复制基础查询 -// currentQuery := make(map[string]interface{}) -// for k, v := range query { -// currentQuery[k] = v -// } -// -// // 添加游标 -// if searchAfter != nil { -// currentQuery["search_after"] = searchAfter -// } -// -// body, err := json.Marshal(currentQuery) -// if err != nil { -// return fmt.Errorf("序列化查询失败: %w", err) -// } -// -// log.Printf("执行第 %d 页查询...", page) -// -// // 执行搜索 -// res, err := es.client.Search( -// es.client.Search.WithIndex(esIndex), -// es.client.Search.WithBody(strings.NewReader(string(body))), -// es.client.Search.WithContext(context.Background()), -// ) -// if err != nil { -// return fmt.Errorf("ES搜索失败: %w", err) -// } -// defer res.Body.Close() -// -// if res.IsError() { -// bodyBytes, _ := io.ReadAll(res.Body) -// return fmt.Errorf("ES搜索返回错误: %s, 响应: %s", res.String(), string(bodyBytes)) -// } -// -// // 读取并解析响应体 -// bodyBytes, err := io.ReadAll(res.Body) -// if err != nil { -// return fmt.Errorf("读取响应体失败: %w", err) -// } -// -// var result map[string]interface{} -// if err := json.Unmarshal(bodyBytes, &result); err != nil { -// return fmt.Errorf("解析ES响应失败: %w", err) -// } -// -// // 检查是否有错误 -// if errMsg, exists := result["error"]; exists { -// return fmt.Errorf("ES返回错误: %v", errMsg) -// } -// -// hits, ok := result["hits"].(map[string]interface{}) -// if !ok { -// return fmt.Errorf("无法解析hits字段") -// } -// -// // 获取总命中数 -// if totalHits, exists := hits["total"].(map[string]interface{}); exists { -// if totalValue, exists := totalHits["value"]; exists { -// log.Printf("ES返回总命中数: %.0f", totalValue) -// } -// } -// -// hitList, ok := hits["hits"].([]interface{}) -// if !ok || len(hitList) == 0 { -// log.Printf("第 %d 页没有数据，查询完成", page) -// break // 没有更多数据 -// } -// -// // 处理当前批次的数据 -// batchCount := 0 -// for _, hit := range hitList { -// hitMap, ok := hit.(map[string]interface{}) -// if !ok { -// log.Printf("警告: 无法解析hit数据") -// continue -// } -// -// source, ok := hitMap["_source"].(map[string]interface{}) -// if !ok { -// log.Printf("警告: 无法解析_source字段") -// continue -// } -// -// isbn, ok := source["isbn"].(string) -// if ok && isbn != "" { -// allISBNs = append(allISBNs, isbn) -// batchCount++ -// } else { -// log.Printf("警告: 跳过空的ISBN字段") -// } -// -// // 更新游标（使用最后一个文档的排序值） -// sortValues, ok := hitMap["sort"].([]interface{}) -// if ok && len(sortValues) > 0 { -// searchAfter = sortValues -// } -// } -// -// totalCount += batchCount -// log.Printf("第 %d 页: 获取 %d 条ISBN记录，总计: %d", page, batchCount, totalCount) -// page++ -// -// // 如果返回的数量小于请求的数量，说明已经是最后一页 -// if len(hitList) < 10000 { -// log.Printf("最后一页数据量 %d < 10000，查询完成", len(hitList)) -// break -// } -// -// // 添加短暂延迟，避免对ES造成过大压力 -// time.Sleep(100 * time.Millisecond) -// } -// -// if len(allISBNs) == 0 { -// return fmt.Errorf("没有找到符合条件的ISBN记录") -// } -// -// // 去重 -// isbnSet := make(map[string]bool) -// uniqueISBNs := make([]string, 0) -// for _, isbn := range allISBNs { -// if !isbnSet[isbn] { -// isbnSet[isbn] = true -// uniqueISBNs = append(uniqueISBNs, isbn) -// } -// } -// -// log.Printf("去重前: %d 条, 去重后: %d 条", len(allISBNs), len(uniqueISBNs)) -// -// // 确保输出目录存在 -// outputDir := filepath.Dir(outputFile) -// if err := os.MkdirAll(outputDir, 0755); err != nil { -// return fmt.Errorf("创建输出目录失败: %w", err) -// } -// -// // 写入文件 -// file, err := os.Create(outputFile) -// if err != nil { -// return fmt.Errorf("创建文件失败: %w", err) -// } -// defer file.Close() -// -// // 写入文件头信息 -// header := fmt.Sprintf(`# 有销售记录且book_pic字符串中pddPath为空的ISBN列表 -//# 查询条件: (day_sale_7 > 0 OR day_sale_15 > 0 OR day_sale_30 > 0 OR day_sale_60 > 0) AND (book_pic包含"pddPath":"" 或 book_pic为空 或 book_pic字段不存在) -//# 索引: %s -//# 统计时间: %s -//# 总记录数: %d -// -//`, esIndex, time.Now().Format("2006-01-02 15:04:05"), len(uniqueISBNs)) -// -// if _, err := file.WriteString(header); err != nil { -// return fmt.Errorf("写入文件头失败: %w", err) -// } -// -// // 按字母顺序排序后写入 -// sort.Strings(uniqueISBNs) -// successCount := 0 -// for _, isbn := range uniqueISBNs { -// if _, err := file.WriteString(isbn + "\n"); err != nil { -// log.Printf("警告: 写入ISBN失败 %s: %v", isbn, err) -// continue -// } -// successCount++ -// } -// -// log.Printf("成功导出 %d/%d 个有销售记录且book_pic字符串中pddPath为空的ISBN到文件: %s", successCount, len(uniqueISBNs), outputFile) -// return nil -//} -// -//// 查询并导出销售ISBN的主函数 -//func mainQuerySaleISBNs() { -// // 初始化ES客户端 -// es, err := NewESClient([]string{esAddress}, esUsername, esPassword) -// if err != nil { -// log.Fatalf("ES连接失败: %v", err) -// } -// -// // 检查ES健康状态 -// if err := es.CheckHealth(); err != nil { -// log.Fatalf("ES健康检查失败: %v", err) -// } -// -// // 输出文件路径 -// outputFile := "cmd/update_es_gt/all_isbns.txt" -// -// // 查询并导出ISBN -// startTime := time.Now() -// if err := exportAllISBNs(es, outputFile); err != nil { -// log.Fatalf("导出销售ISBN失败: %v", err) -// } -// -// elapsed := time.Since(startTime) -// log.Printf("任务完成！耗时: %v，ISBN已导出到: %s", elapsed.Round(time.Millisecond), outputFile) -//} -// -//// 导出所有ISBN到txt文件 -//func exportAllISBNs(es *ESClient, outputFile string) error { -// log.Printf("开始导出所有ISBN...") -// -// // 查询所有包含isbn字段的文档 -// query := map[string]interface{}{ -// "query": map[string]interface{}{ -// "exists": map[string]interface{}{ -// "field": "isbn", -// }, -// }, -// "_source": []string{"isbn"}, -// "sort": []map[string]interface{}{ -// {"isbn": "asc"}, // 按ISBN排序 -// }, -// "size": 10000, -// } -// -// var allISBNs []string -// var searchAfter interface{} -// totalCount := 0 -// page := 1 -// -// for { -// // 复制基础查询 -// currentQuery := make(map[string]interface{}) -// for k, v := range query { -// currentQuery[k] = v -// } -// -// // 添加游标 -// if searchAfter != nil { -// currentQuery["search_after"] = searchAfter -// } -// -// body, err := json.Marshal(currentQuery) -// if err != nil { -// return fmt.Errorf("序列化查询失败: %w", err) -// } -// -// log.Printf("执行第 %d 页查询...", page) -// -// // 执行搜索 -// res, err := es.client.Search( -// es.client.Search.WithIndex(esIndex), -// es.client.Search.WithBody(strings.NewReader(string(body))), -// es.client.Search.WithContext(context.Background()), -// ) -// if err != nil { -// return fmt.Errorf("ES搜索失败: %w", err) -// } -// defer res.Body.Close() -// -// if res.IsError() { -// bodyBytes, _ := io.ReadAll(res.Body) -// return fmt.Errorf("ES搜索返回错误: %s, 响应: %s", res.String(), string(bodyBytes)) -// } -// -// // 读取并解析响应体 -// bodyBytes, err := io.ReadAll(res.Body) -// if err != nil { -// return fmt.Errorf("读取响应体失败: %w", err) -// } -// -// var result map[string]interface{} -// if err := json.Unmarshal(bodyBytes, &result); err != nil { -// return fmt.Errorf("解析ES响应失败: %w", err) -// } -// -// // 检查是否有错误 -// if errMsg, exists := result["error"]; exists { -// return fmt.Errorf("ES返回错误: %v", errMsg) -// } -// -// hits, ok := result["hits"].(map[string]interface{}) -// if !ok { -// return fmt.Errorf("无法解析hits字段") -// } -// -// // 获取总命中数 -// if totalHits, exists := hits["total"].(map[string]interface{}); exists { -// if totalValue, exists := totalHits["value"]; exists { -// if page == 1 { -// log.Printf("ES索引中共有 %.0f 条包含ISBN的记录", totalValue) -// } -// } -// } -// -// hitList, ok := hits["hits"].([]interface{}) -// if !ok || len(hitList) == 0 { -// log.Printf("第 %d 页没有数据，查询完成", page) -// break // 没有更多数据 -// } -// -// // 处理当前批次的数据 -// batchCount := 0 -// for _, hit := range hitList { -// hitMap, ok := hit.(map[string]interface{}) -// if !ok { -// log.Printf("警告: 无法解析hit数据") -// continue -// } -// -// source, ok := hitMap["_source"].(map[string]interface{}) -// if !ok { -// log.Printf("警告: 无法解析_source字段") -// continue -// } -// -// isbn, ok := source["isbn"].(string) -// if ok && isbn != "" { -// allISBNs = append(allISBNs, isbn) -// batchCount++ -// } else { -// log.Printf("警告: 跳过空的ISBN字段") -// } -// -// // 更新游标（使用最后一个文档的排序值） -// sortValues, ok := hitMap["sort"].([]interface{}) -// if ok && len(sortValues) > 0 { -// searchAfter = sortValues -// } -// } -// -// totalCount += batchCount -// log.Printf("第 %d 页: 获取 %d 条ISBN记录，总计: %d", page, batchCount, totalCount) -// page++ -// -// // 如果返回的数量小于请求的数量，说明已经是最后一页 -// if len(hitList) < 10000 { -// log.Printf("最后一页数据量 %d < 10000，查询完成", len(hitList)) -// break -// } -// -// // 添加短暂延迟，避免对ES造成过大压力 -// time.Sleep(100 * time.Millisecond) -// } -// -// if len(allISBNs) == 0 { -// return fmt.Errorf("没有找到包含ISBN字段的记录") -// } -// -// // 去重 -// isbnSet := make(map[string]bool) -// uniqueISBNs := make([]string, 0) -// for _, isbn := range allISBNs { -// if !isbnSet[isbn] { -// isbnSet[isbn] = true -// uniqueISBNs = append(uniqueISBNs, isbn) -// } -// } -// -// log.Printf("去重前: %d 条, 去重后: %d 条", len(allISBNs), len(uniqueISBNs)) -// -// // 确保输出目录存在 -// outputDir := filepath.Dir(outputFile) -// if err := os.MkdirAll(outputDir, 0755); err != nil { -// return fmt.Errorf("创建输出目录失败: %w", err) -// } -// -// // 写入文件 -// file, err := os.Create(outputFile) -// if err != nil { -// return fmt.Errorf("创建文件失败: %w", err) -// } -// defer file.Close() -// -// // 写入文件头信息 -// header := fmt.Sprintf(`# 所有ISBN列表 -//# 索引: %s -//# 导出时间: %s -//# 总记录数: %d -// -//`, esIndex, time.Now().Format("2006-01-02 15:04:05"), len(uniqueISBNs)) -// -// if _, err := file.WriteString(header); err != nil { -// return fmt.Errorf("写入文件头失败: %w", err) -// } -// -// // 按字母顺序排序后写入 -// sort.Strings(uniqueISBNs) -// successCount := 0 -// for _, isbn := range uniqueISBNs { -// if _, err := file.WriteString(isbn + "\n"); err != nil { -// log.Printf("警告: 写入ISBN失败 %s: %v", isbn, err) -// continue -// } -// successCount++ -// } -// -// log.Printf("成功导出 %d/%d 个ISBN到文件: %s", successCount, len(uniqueISBNs), outputFile) -// return nil -//} -// -//// 从ES获取所有ISBN -//func getAllISBNsFromES(es *ESClient) ([]string, error) { -// log.Printf("开始从ES索引 %s 获取所有ISBN...", esIndex) -// -// var allISBNs []string -// var searchAfter interface{} -// totalCount := 0 -// page := 1 -// -// for { -// query := map[string]interface{}{ -// "query": map[string]interface{}{ -// "exists": map[string]interface{}{ -// "field": "isbn", -// }, -// }, -// "_source": []string{"isbn"}, -// "sort": []map[string]interface{}{ -// {"isbn": "asc"}, -// }, -// "size": 10000, -// } -// -// // 添加游标 -// if searchAfter != nil { -// query["search_after"] = searchAfter -// } -// -// body, err := json.Marshal(query) -// if err != nil { -// return nil, fmt.Errorf("序列化查询失败: %w", err) -// } -// -// log.Printf("执行第 %d 页ES查询...", page) -// -// // 执行搜索 -// res, err := es.client.Search( -// es.client.Search.WithIndex(esIndex), -// es.client.Search.WithBody(strings.NewReader(string(body))), -// es.client.Search.WithContext(context.Background()), -// ) -// if err != nil { -// return nil, fmt.Errorf("ES搜索失败: %w", err) -// } -// defer res.Body.Close() -// -// if res.IsError() { -// bodyBytes, _ := io.ReadAll(res.Body) -// return nil, fmt.Errorf("ES搜索返回错误: %s, 响应: %s", res.String(), string(bodyBytes)) -// } -// -// // 解析响应 -// var result map[string]interface{} -// if err := json.NewDecoder(res.Body).Decode(&result); err != nil { -// return nil, fmt.Errorf("解析ES响应失败: %w", err) -// } -// -// hits, ok := result["hits"].(map[string]interface{}) -// if !ok { -// return nil, fmt.Errorf("无法解析hits字段") -// } -// -// hitList, ok := hits["hits"].([]interface{}) -// if !ok || len(hitList) == 0 { -// log.Printf("第 %d 页没有数据，查询完成", page) -// break -// } -// -// // 处理当前批次的数据 -// batchCount := 0 -// for _, hit := range hitList { -// hitMap, ok := hit.(map[string]interface{}) -// if !ok { -// log.Printf("警告: 无法解析hit数据") -// continue -// } -// -// source, ok := hitMap["_source"].(map[string]interface{}) -// if !ok { -// log.Printf("警告: 无法解析_source字段") -// continue -// } -// -// isbn, ok := source["isbn"].(string) -// if ok && isbn != "" { -// allISBNs = append(allISBNs, isbn) -// batchCount++ -// } -// -// // 更新游标 -// sortValues, ok := hitMap["sort"].([]interface{}) -// if ok && len(sortValues) > 0 { -// searchAfter = sortValues -// } -// } -// -// totalCount += batchCount -// log.Printf("第 %d 页: 获取 %d 条ISBN记录，总计: %d", page, batchCount, totalCount) -// page++ -// -// // 如果返回的数量小于请求的数量，说明已经是最后一页 -// if len(hitList) < 10000 { -// log.Printf("最后一页数据量 %d < 10000，查询完成", len(hitList)) -// break -// } -// -// time.Sleep(100 * time.Millisecond) -// } -// -// if len(allISBNs) == 0 { -// return nil, fmt.Errorf("ES中没有找到包含ISBN字段的记录") -// } -// -// log.Printf("从ES中获取到 %d 个ISBN", len(allISBNs)) -// return allISBNs, nil -//} -// -//// 批量检查ISBN在数据库中是否存在 -//func checkISBNsInDB(db *sql.DB, isbns []string) (map[string]bool, error) { -// log.Printf("开始检查 %d 个ISBN在数据库中的存在情况...", len(isbns)) -// -// existsMap := make(map[string]bool) -// -// // 分批处理，避免SQL语句过长 -// batchSize := 1000 -// totalBatches := (len(isbns) + batchSize - 1) / batchSize -// -// for batch := 0; batch < totalBatches; batch++ { -// start := batch * batchSize -// end := start + batchSize -// if end > len(isbns) { -// end = len(isbns) -// } -// -// batchISBNs := isbns[start:end] -// log.Printf("处理数据库批次 %d/%d: ISBN范围 %d-%d", batch+1, totalBatches, start+1, end) -// -// // 构建IN查询的占位符 -// placeholders := make([]string, len(batchISBNs)) -// args := make([]interface{}, len(batchISBNs)) -// for i, isbn := range batchISBNs { -// placeholders[i] = "?" -// args[i] = isbn -// } -// -// query := fmt.Sprintf( -// "SELECT isbn FROM xgy_base_item WHERE isbn IN (%s)", -// strings.Join(placeholders, ","), -// ) -// -// rows, err := db.Query(query, args...) -// if err != nil { -// return nil, fmt.Errorf("数据库查询失败: %w", err) -// } -// -// // 读取存在的ISBN -// for rows.Next() { -// var isbn string -// if err := rows.Scan(&isbn); err != nil { -// rows.Close() -// return nil, fmt.Errorf("扫描ISBN失败: %w", err) -// } -// existsMap[isbn] = true -// } -// rows.Close() -// -// if err = rows.Err(); err != nil { -// return nil, fmt.Errorf("遍历数据库行时出错: %w", err) -// } -// -// // 添加延迟避免对数据库造成压力 -// if batch < totalBatches-1 { -// time.Sleep(50 * time.Millisecond) -// } -// } -// -// log.Printf("数据库中存在 %d 个匹配的ISBN", len(existsMap)) -// return existsMap, nil -//} -// -//// 找出数据库中不存在的ISBN（ES中有但数据库中没有） -//func findESOnlyISBNs(esISBNs []string, dbExistsMap map[string]bool) []string { -// var esOnlyISBNs []string -// -// for _, isbn := range esISBNs { -// if !dbExistsMap[isbn] { -// esOnlyISBNs = append(esOnlyISBNs, isbn) -// } -// } -// -// log.Printf("ES中有 %d 个ISBN在数据库中不存在", len(esOnlyISBNs)) -// return esOnlyISBNs -//} -// -//// 导出ES独有ISBN到txt文件 -//func exportESOnlyISBNs(esOnlyISBNs []string, outputFile string) error { -// if len(esOnlyISBNs) == 0 { -// log.Printf("没有ES独有的ISBN需要导出") -// return nil -// } -// -// // 确保输出目录存在 -// outputDir := filepath.Dir(outputFile) -// if err := os.MkdirAll(outputDir, 0755); err != nil { -// return fmt.Errorf("创建输出目录失败: %w", err) -// } -// -// // 写入文件 -// file, err := os.Create(outputFile) -// if err != nil { -// return fmt.Errorf("创建文件失败: %w", err) -// } -// defer file.Close() -// -// // 写入文件头信息 -// header := fmt.Sprintf(`# ES中有但数据库中没有的ISBN列表 -//# 数据库表: xgy_base_item -//# ES索引: %s -//# 导出时间: %s -//# 记录数: %d -//# 说明: 这些ISBN在ES索引中存在但在数据库表中不存在 -// -//`, esIndex, time.Now().Format("2006-01-02 15:04:05"), len(esOnlyISBNs)) -// -// if _, err := file.WriteString(header); err != nil { -// return fmt.Errorf("写入文件头失败: %w", err) -// } -// -// // 按字母顺序排序后写入 -// sort.Strings(esOnlyISBNs) -// successCount := 0 -// for _, isbn := range esOnlyISBNs { -// if _, err := file.WriteString(isbn + "\n"); err != nil { -// log.Printf("警告: 写入ISBN失败 %s: %v", isbn, err) -// continue -// } -// successCount++ -// } -// -// log.Printf("成功导出 %d/%d 个ES独有ISBN到文件: %s", successCount, len(esOnlyISBNs), outputFile) -// return nil -//} -// -//// 主函数：查询ES中有但数据库中没有的ISBN -//func mainFindESOnlyISBNs() { -// // 初始化数据库连接 -// dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", -// "root", -// "123456", -// "localhost", -// 3306, -// "book_image") // 请根据实际情况修改数据库名 -// -// db, err := sql.Open("mysql", dsn) -// if err != nil { -// log.Fatalf("打开数据库连接失败: %v", err) -// } -// defer db.Close() -// -// // 设置连接池参数 -// db.SetMaxOpenConns(20) -// db.SetMaxIdleConns(10) -// -// // 测试数据库连接 -// if err := db.Ping(); err != nil { -// log.Fatalf("数据库连接测试失败: %v", err) -// } -// -// // 初始化ES客户端 -// es, err := NewESClient([]string{esAddress}, esUsername, esPassword) -// if err != nil { -// log.Fatalf("ES连接失败: %v", err) -// } -// -// // 检查ES健康状态 -// if err := es.CheckHealth(); err != nil { -// log.Fatalf("ES健康检查失败: %v", err) -// } -// -// startTime := time.Now() -// log.Printf("开始处理ES与数据库的ISBN匹配...") -// -// // 步骤1: 从ES获取所有ISBN -// esISBNs, err := getAllISBNsFromES(es) -// if err != nil { -// log.Fatalf("获取ES ISBN失败: %v", err) -// } -// -// // 步骤2: 检查ISBN在数据库中的存在情况 -// dbExistsMap, err := checkISBNsInDB(db, esISBNs) -// if err != nil { -// log.Fatalf("检查数据库中ISBN存在情况失败: %v", err) -// } -// -// // 步骤3: 找出ES独有ISBN（ES中有但数据库中没有） -// esOnlyISBNs := findESOnlyISBNs(esISBNs, dbExistsMap) -// -// // 步骤4: 导出ES独有ISBN -// outputFile := "cmd/update_es_gt/missing_isbns.txt" -// if err := exportESOnlyISBNs(esOnlyISBNs, outputFile); err != nil { -// log.Fatalf("导出ES独有ISBN失败: %v", err) -// } -// -// elapsed := time.Since(startTime) -// -// // 输出统计信息 -// fmt.Printf("\n=== 处理完成 ===\n") -// fmt.Printf("ES中ISBN总数: %d\n", len(esISBNs)) -// fmt.Printf("数据库中匹配的ISBN数: %d\n", len(dbExistsMap)) -// fmt.Printf("ES独有ISBN数（数据库中没有的）: %d\n", len(esOnlyISBNs)) -// fmt.Printf("独有比例: %.2f%%\n", float64(len(esOnlyISBNs))/float64(len(esISBNs))*100) -// fmt.Printf("耗时: %v\n", elapsed.Round(time.Millisecond)) -// fmt.Printf("输出文件: %s\n", outputFile) -// -// // 显示部分ES独有ISBN示例 -// if len(esOnlyISBNs) > 0 { -// fmt.Printf("\nES独有ISBN示例 (前10个):\n") -// for i := 0; i < 10 && i < len(esOnlyISBNs); i++ { -// fmt.Printf(" %s\n", esOnlyISBNs[i]) -// } -// if len(esOnlyISBNs) > 10 { -// fmt.Printf(" ... 还有 %d 个\n", len(esOnlyISBNs)-10) -// } -// } -//} -// -//// 查询并导出有销售记录且book_pic为空的ISBN -//func queryAndExportSaleISBNsWithEmptyPic(es *ESClient, outputFile string) error { -// log.Printf("开始查询有销售记录且book_pic为空的ISBN...") -// -// // 查询条件: (day_sale_7 > 0 OR day_sale_15 > 0 OR day_sale_30 > 0) AND book_pic为空 -// query := map[string]interface{}{ -// "query": map[string]interface{}{ -// "bool": map[string]interface{}{ -// "must": []map[string]interface{}{ -// { -// "bool": map[string]interface{}{ -// "should": []map[string]interface{}{ -// {"range": map[string]interface{}{"day_sale_7": map[string]interface{}{"gt": 0}}}, -// {"range": map[string]interface{}{"day_sale_15": map[string]interface{}{"gt": 0}}}, -// {"range": map[string]interface{}{"day_sale_30": map[string]interface{}{"gt": 0}}}, -// }, -// "minimum_should_match": 1, -// }, -// }, -// { -// "bool": map[string]interface{}{ -// "should": []map[string]interface{}{ -// // 匹配book_pic字段为空 -// {"term": map[string]interface{}{"book_pic": ""}}, -// // 匹配book_pic字段不存在 -// { -// "bool": map[string]interface{}{ -// "must_not": map[string]interface{}{ -// "exists": map[string]interface{}{"field": "book_pic"}, -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// }, -// "_source": []string{"isbn"}, -// "sort": []map[string]interface{}{ -// {"isbn": "asc"}, // 按ISBN排序 -// }, -// "size": 10000, -// } -// -// // 打印查询条件用于验证 -// queryJSON, _ := json.MarshalIndent(query, "", " ") -// log.Printf("查询条件:\n%s", string(queryJSON)) -// -// var allISBNs []string -// var searchAfter interface{} -// totalCount := 0 -// page := 1 -// -// for { -// // 复制基础查询 -// currentQuery := make(map[string]interface{}) -// for k, v := range query { -// currentQuery[k] = v -// } -// -// // 添加游标 -// if searchAfter != nil { -// currentQuery["search_after"] = searchAfter -// } -// -// body, err := json.Marshal(currentQuery) -// if err != nil { -// return fmt.Errorf("序列化查询失败: %w", err) -// } -// -// log.Printf("执行第 %d 页查询...", page) -// -// // 执行搜索 -// res, err := es.client.Search( -// es.client.Search.WithIndex(esIndex), -// es.client.Search.WithBody(strings.NewReader(string(body))), -// es.client.Search.WithContext(context.Background()), -// ) -// if err != nil { -// return fmt.Errorf("ES搜索失败: %w", err) -// } -// defer res.Body.Close() -// -// if res.IsError() { -// bodyBytes, _ := io.ReadAll(res.Body) -// return fmt.Errorf("ES搜索返回错误: %s, 响应: %s", res.String(), string(bodyBytes)) -// } -// -// // 读取并解析响应体 -// bodyBytes, err := io.ReadAll(res.Body) -// if err != nil { -// return fmt.Errorf("读取响应体失败: %w", err) -// } -// -// var result map[string]interface{} -// if err := json.Unmarshal(bodyBytes, &result); err != nil { -// return fmt.Errorf("解析ES响应失败: %w", err) -// } -// -// // 检查是否有错误 -// if errMsg, exists := result["error"]; exists { -// return fmt.Errorf("ES返回错误: %v", errMsg) -// } -// -// hits, ok := result["hits"].(map[string]interface{}) -// if !ok { -// return fmt.Errorf("无法解析hits字段") -// } -// -// // 获取总命中数 -// if totalHits, exists := hits["total"].(map[string]interface{}); exists { -// if totalValue, exists := totalHits["value"]; exists { -// log.Printf("ES返回总命中数: %.0f", totalValue) -// } -// } -// -// hitList, ok := hits["hits"].([]interface{}) -// if !ok || len(hitList) == 0 { -// log.Printf("第 %d 页没有数据，查询完成", page) -// break // 没有更多数据 -// } -// -// // 处理当前批次的数据 -// batchCount := 0 -// for _, hit := range hitList { -// hitMap, ok := hit.(map[string]interface{}) -// if !ok { -// log.Printf("警告: 无法解析hit数据") -// continue -// } -// -// source, ok := hitMap["_source"].(map[string]interface{}) -// if !ok { -// log.Printf("警告: 无法解析_source字段") -// continue -// } -// -// isbn, ok := source["isbn"].(string) -// if ok && isbn != "" { -// allISBNs = append(allISBNs, isbn) -// batchCount++ -// } else { -// log.Printf("警告: 跳过空的ISBN字段") -// } -// -// // 更新游标（使用最后一个文档的排序值） -// sortValues, ok := hitMap["sort"].([]interface{}) -// if ok && len(sortValues) > 0 { -// searchAfter = sortValues -// } -// } -// -// totalCount += batchCount -// log.Printf("第 %d 页: 获取 %d 条ISBN记录，总计: %d", page, batchCount, totalCount) -// page++ -// -// // 如果返回的数量小于请求的数量，说明已经是最后一页 -// if len(hitList) < 10000 { -// log.Printf("最后一页数据量 %d < 10000，查询完成", len(hitList)) -// break -// } -// -// // 添加短暂延迟，避免对ES造成过大压力 -// time.Sleep(100 * time.Millisecond) -// } -// -// if len(allISBNs) == 0 { -// return fmt.Errorf("没有找到符合条件的ISBN记录") -// } -// -// // 去重 -// isbnSet := make(map[string]bool) -// uniqueISBNs := make([]string, 0) -// for _, isbn := range allISBNs { -// if !isbnSet[isbn] { -// isbnSet[isbn] = true -// uniqueISBNs = append(uniqueISBNs, isbn) -// } -// } -// -// log.Printf("去重前: %d 条, 去重后: %d 条", len(allISBNs), len(uniqueISBNs)) -// -// // 确保输出目录存在 -// outputDir := filepath.Dir(outputFile) -// if err := os.MkdirAll(outputDir, 0755); err != nil { -// return fmt.Errorf("创建输出目录失败: %w", err) -// } -// -// // 写入文件 -// file, err := os.Create(outputFile) -// if err != nil { -// return fmt.Errorf("创建文件失败: %w", err) -// } -// defer file.Close() -// -// // 写入文件头信息 -// header := fmt.Sprintf(`# 有销售记录且book_pic为空的ISBN列表 -//# 查询条件: (day_sale_7 > 0 OR day_sale_15 > 0 OR day_sale_30 > 0) AND (book_pic为空 或 book_pic字段不存在) -//# 索引: %s -//# 查询时间: %s -//# 总记录数: %d -// -//`, esIndex, time.Now().Format("2006-01-02 15:04:05"), len(uniqueISBNs)) -// -// if _, err := file.WriteString(header); err != nil { -// return fmt.Errorf("写入文件头失败: %w", err) -// } -// -// // 按字母顺序排序后写入 -// sort.Strings(uniqueISBNs) -// successCount := 0 -// for _, isbn := range uniqueISBNs { -// if _, err := file.WriteString(isbn + "\n"); err != nil { -// log.Printf("警告: 写入ISBN失败 %s: %v", isbn, err) -// continue -// } -// successCount++ -// } -// -// log.Printf("成功导出 %d/%d 个符合条件的ISBN到文件: %s", successCount, len(uniqueISBNs), outputFile) -// return nil -//} -// -//// 查询并导出有销售记录且book_pic为空的ISBN主函数 -//func mainQuerySaleISBNsWithEmptyPic() { -// // 初始化ES客户端 -// es, err := NewESClient([]string{esAddress}, esUsername, esPassword) -// if err != nil { -// log.Fatalf("ES连接失败: %v", err) -// } -// -// // 检查ES健康状态 -// if err := es.CheckHealth(); err != nil { -// log.Fatalf("ES健康检查失败: %v", err) -// } -// -// // 输出文件路径 -// outputFile := "es/sale_isbns_empty_pic.txt" -// -// // 查询并导出ISBN -// startTime := time.Now() -// if err := queryAndExportSaleISBNsWithEmptyPic(es, outputFile); err != nil { -// log.Fatalf("导出有销售记录且book_pic为空的ISBN失败: %v", err) -// } -// -// elapsed := time.Since(startTime) -// log.Printf("任务完成！耗时: %v，ISBN已导出到: %s", elapsed.Round(time.Millisecond), outputFile) -//} diff --git a/go.mod b/go.mod index cf5ffc4..1c4ae62 100644 --- a/go.mod +++ b/go.mod @@ -4,14 +4,17 @@ go 1.25 require ( github.com/PuerkitoBio/goquery v1.10.3 - github.com/chromedp/chromedp v0.14.2 + github.com/boombuler/barcode v1.1.0 github.com/disintegration/imaging v1.6.2 github.com/elastic/go-elasticsearch/v8 v8.19.0 + github.com/fogleman/gg v1.3.0 github.com/gin-gonic/gin v1.11.0 github.com/go-sql-driver/mysql v1.9.3 + github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 github.com/makiuchi-d/gozxing v0.1.1 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/parnurzeal/gorequest v0.3.0 + github.com/redis/go-redis/v9 v9.17.2 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/xuri/excelize/v2 v2.10.0 golang.org/x/image v0.25.0 @@ -23,22 +26,18 @@ require ( github.com/andybalholm/cascadia v1.3.3 // indirect github.com/bytedance/sonic v1.14.0 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect - github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 // indirect - github.com/chromedp/sysutil v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/elastic/elastic-transport-go/v8 v8.7.0 // indirect github.com/elazarl/goproxy v1.7.2 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/gin-contrib/sse v1.1.0 // indirect - github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.27.0 // indirect - github.com/gobwas/httphead v0.1.0 // indirect - github.com/gobwas/pool v0.2.1 // indirect - github.com/gobwas/ws v1.4.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/goccy/go-yaml v1.18.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index 843f618..934fc07 100644 --- a/go.sum +++ b/go.sum @@ -4,21 +4,25 @@ github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiU github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/boombuler/barcode v1.1.0 h1:ChaYjBR63fr4LFyGn8E8nt7dBSt3MiU3zMOZqFvVkHo= +github.com/boombuler/barcode v1.1.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 h1:UQ4AU+BGti3Sy/aLU8KVseYKNALcX9UXY6DfpwQ6J8E= -github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= -github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= -github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= -github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= -github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/elastic/elastic-transport-go/v8 v8.7.0 h1:OgTneVuXP2uip4BA658Xi6Hfw+PeIOod2rY3GVMGoVE= @@ -27,14 +31,14 @@ github.com/elastic/go-elasticsearch/v8 v8.19.0 h1:VmfBLNRORY7RZL+9hTxBD97ehl9H8N github.com/elastic/go-elasticsearch/v8 v8.19.0/go.mod h1:F3j9e+BubmKvzvLjNui/1++nJuJxbkhHefbaT0kFKGY= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= -github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= -github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -50,16 +54,12 @@ github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHO github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= -github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= -github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= -github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= -github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -72,8 +72,6 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= -github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/makiuchi-d/gozxing v0.1.1 h1:xxqijhoedi+/lZlhINteGbywIrewVdVv2wl9r5O9S1I= @@ -88,8 +86,6 @@ github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= -github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= -github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/parnurzeal/gorequest v0.3.0 h1:SoFyqCDC9COr1xuS6VA8fC8RU7XyrJZN2ona1kEX7FI= github.com/parnurzeal/gorequest v0.3.0/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= @@ -102,6 +98,8 @@ github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI= +github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370= github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= diff --git a/goroutine-pool/goroutine_pool.go b/goroutine-pool/goroutine_pool.go new file mode 100644 index 0000000..7b706ef --- /dev/null +++ b/goroutine-pool/goroutine_pool.go @@ -0,0 +1,3302 @@ +// Package goroutine_pool 提供高性能、功能完整的无锁协程池实现 +package goroutine_pool + +import ( + "context" + "encoding/json" + "fmt" + "os" + "runtime" + "runtime/debug" + "strings" + "sync" + "sync/atomic" + "time" + "unsafe" +) + +// ==================== 常量定义 ==================== + +const ( + SuccessCode = 0 + ErrorPoolNotInitialized = -1 + ErrorWorkerNotFound = -2 + ErrorWorkerNotIdle = -3 + ErrorWorkerNotRunning = -4 + ErrorWorkerNotPaused = -5 + ErrorTaskSubmitTimeout = -6 + ErrorQueueFull = -7 + ErrorInvalidConfig = -8 + ErrorMemoryExhausted = -9 + ErrorMaxWorkersExceeded = -10 + ErrorInvalidLanguage = -11 + ErrorWorkerAlreadyRunning = -12 + ErrorFunctionNotRegistered = -13 + ErrorTaskNotFound = -14 + ErrorTaskAlreadyCompleted = -15 + ErrorTaskCancelled = -16 + ErrorTaskTimeout = -17 + ErrorSystemError = -18 + ErrorInvalidParameter = -19 + ErrorResourceUnavailable = -20 +) + +const ( + WorkerStatusIdle = 0 + WorkerStatusRunning = 1 + WorkerStatusPaused = 2 + WorkerStatusStopped = 3 + WorkerStatusDraining = 4 +) + +const ( + TaskStatusPending = 0 + TaskStatusRunning = 1 + TaskStatusCompleted = 2 + TaskStatusFailed = 3 + TaskStatusCancelled = 4 + TaskStatusTimeout = 5 +) + +const ( + LogLevelDebug = 0 + LogLevelInfo = 1 + LogLevelWarn = 2 + LogLevelError = 3 + LogLevelFatal = 4 +) + +const ( + DefaultQueueSize = 1000 + DefaultTaskTimeout = 30000 + DefaultIdleTimeout = 60000 + DefaultMaxMemoryMB = 1024 + DefaultMinWorkers = 2 + DefaultMaxWorkers = 100 + DefaultShutdownTimeout = 30000 + DefaultHealthCheckDelay = 5000 + DefaultMaxRetries = 3 + DefaultResultBufferSize = 65536 +) + +const ( + WorkerTypeAsync = 0 + WorkerTypeSync = 1 +) + +const ( + ControlMsgStop = 1 + ControlMsgPause = 2 + ControlMsgResume = 3 + ControlMsgDrain = 4 + ControlMsgRestart = 5 + ControlMsgUpdate = 6 + ControlMsgHealth = 7 + ControlMsgGraceful = 8 + ControlMsgDelete = 9 +) + +const ( + PriorityLow = 0 + PriorityNormal = 5 + PriorityHigh = 10 + PriorityUrgent = 15 +) + +const ( + LanguageCN = "cn_zh" + LanguageEN = "en_us" +) + +// ==================== 原子数据结构 ==================== + +// AtomicTask 原子任务 +type AtomicTask struct { + id int64 + goroutineID int32 + status int32 + priority int32 + retryCount int32 + maxRetries int32 + submitTime int64 + queueTime int64 + dequeueTime int64 + startTime int64 + endTime int64 + waitDuration int64 + execDuration int64 + totalDuration int64 + + funcName atomic.Value // string + param atomic.Value // string + result atomic.Value // string + errorMsg atomic.Value // string + + callback unsafe.Pointer // *TaskCallback + userData unsafe.Pointer // interface{} + + ctx context.Context + cancel context.CancelFunc +} + +// NewAtomicTask 创建原子任务 +func NewAtomicTask(goroutineID int, funcName, param string, priority, maxRetries, timeoutMs int) *AtomicTask { + if timeoutMs <= 0 { + timeoutMs = DefaultTaskTimeout + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutMs)*time.Millisecond) + now := time.Now().UnixNano() / 1e6 + + task := &AtomicTask{ + id: atomic.AddInt64(&taskIDCounter, 1), + goroutineID: int32(goroutineID), + status: int32(TaskStatusPending), + priority: int32(priority), + retryCount: 0, + maxRetries: int32(maxRetries), + submitTime: now, + queueTime: now, + ctx: ctx, + cancel: cancel, + } + + task.funcName.Store(funcName) + task.param.Store(param) + + return task +} + +// ==================== 增强的无锁环形队列 ==================== + +// EnhancedRingQueue 增强的环形队列（支持暂停和排空） +type EnhancedRingQueue struct { + items []unsafe.Pointer + capacity int32 + head int32 + tail int32 + size int32 + paused int32 // 暂停状态 + draining int32 // 排空状态 + cond *sync.Cond +} + +// NewEnhancedRingQueue 创建增强环形队列 +func NewEnhancedRingQueue(capacity int32) *EnhancedRingQueue { + if capacity <= 0 { + capacity = 1000 + } + + capacity = roundUpToPowerOfTwo(capacity) + + queue := &EnhancedRingQueue{ + items: make([]unsafe.Pointer, capacity), + capacity: capacity, + cond: sync.NewCond(&sync.Mutex{}), + } + + return queue +} + +// Push 入队（支持暂停检查） +func (q *EnhancedRingQueue) Push(task *AtomicTask) bool { + // 检查是否暂停（排空模式不接受新任务） + if atomic.LoadInt32(&q.paused) == 1 { + return false // 暂停时立即返回失败 + } + + // 排空模式只处理已有任务 + if atomic.LoadInt32(&q.draining) == 1 { + return false + } + + for { + tail := atomic.LoadInt32(&q.tail) + head := atomic.LoadInt32(&q.head) + next := (tail + 1) & (q.capacity - 1) + + if next == head { + return false // 队列满 + } + + if atomic.CompareAndSwapInt32(&q.tail, tail, next) { + atomic.StorePointer(&q.items[tail], unsafe.Pointer(task)) + atomic.AddInt32(&q.size, 1) + q.cond.Broadcast() + return true + } + } +} + +// Pop 出队（支持排空模式） +func (q *EnhancedRingQueue) Pop() *AtomicTask { + for { + // 排空模式：只处理已有任务，不接受新任务 + if atomic.LoadInt32(&q.draining) == 1 && atomic.LoadInt32(&q.size) == 0 { + return nil + } + + head := atomic.LoadInt32(&q.head) + tail := atomic.LoadInt32(&q.tail) + + if head == tail { + // 等待新任务 + q.cond.L.Lock() + for atomic.LoadInt32(&q.size) == 0 { + if atomic.LoadInt32(&q.draining) == 1 { + q.cond.L.Unlock() + return nil + } + q.cond.Wait() + } + q.cond.L.Unlock() + continue + } + + idx := head & (q.capacity - 1) + task := (*AtomicTask)(atomic.LoadPointer(&q.items[idx])) + + if task == nil { + atomic.CompareAndSwapInt32(&q.head, head, (head+1)&(q.capacity-1)) + atomic.AddInt32(&q.size, -1) + continue + } + + if atomic.CompareAndSwapInt32(&q.head, head, (head+1)&(q.capacity-1)) { + atomic.StorePointer(&q.items[idx], nil) + atomic.AddInt32(&q.size, -1) + return task + } + } +} + +// Size 获取队列大小 +func (q *EnhancedRingQueue) Size() int32 { + return atomic.LoadInt32(&q.size) +} + +// Capacity 获取队列容量 +func (q *EnhancedRingQueue) Capacity() int32 { + return q.capacity +} + +// Pause 暂停队列 +func (q *EnhancedRingQueue) Pause() { + atomic.StoreInt32(&q.paused, 1) +} + +// Resume 恢复队列 +func (q *EnhancedRingQueue) Resume() { + atomic.StoreInt32(&q.paused, 0) + q.cond.Broadcast() +} + +// Drain 排空队列 +func (q *EnhancedRingQueue) Drain() { + atomic.StoreInt32(&q.draining, 1) + q.cond.Broadcast() +} + +// ResetDrain 重置排空状态 +func (q *EnhancedRingQueue) ResetDrain() { + atomic.StoreInt32(&q.draining, 0) +} + +// ==================== 完整的工作者实现 ==================== + +// CompleteWorker 完整的工作者实现 +type CompleteWorker struct { + // 原子状态 + id int32 + status int32 + lastActive int64 + createdAt int64 + + // 队列和控制 + taskQueue *EnhancedRingQueue + controlChan chan ControlMessage + healthChan chan bool + + // 统计 + taskCount int64 + errorCount int64 + successCount int64 + activeTasks int32 + queuedTasks int32 + restartCount int32 + + // 配置和上下文 + config *WorkerConfig + ctx context.Context + cancel context.CancelFunc + + // 同步原语 + once sync.Once + mu sync.RWMutex + stopped int32 // 新增：明确的停止标志 + + // 任务管理 + configMu sync.RWMutex + tasksMu sync.RWMutex + activeTasksMap map[int64]*AtomicTask +} + +// WorkerConfig 工作者配置 +type WorkerConfig struct { + ID int32 + Name string + Type int32 + QueueSize int32 + TimeoutMs int32 + MaxRetries int32 + MemoryLimitKB int64 + AutoRestart bool + EnableRecovery bool + RecoveryDelayMs int32 + PriorityEnabled bool +} + +// NewCompleteWorker 创建完整的工作者 +func NewCompleteWorker(id int32, name string, config *WorkerConfig) *CompleteWorker { + if config == nil { + config = &WorkerConfig{ + QueueSize: DefaultQueueSize, + TimeoutMs: DefaultTaskTimeout, + MaxRetries: DefaultMaxRetries, + AutoRestart: true, + EnableRecovery: true, + RecoveryDelayMs: 1000, + PriorityEnabled: false, + } + } + + ctx, cancel := context.WithCancel(context.Background()) + + return &CompleteWorker{ + id: id, + status: int32(WorkerStatusIdle), + taskQueue: NewEnhancedRingQueue(config.QueueSize), + controlChan: make(chan ControlMessage, 32), + healthChan: make(chan bool, 1), + config: config, + ctx: ctx, + cancel: cancel, + createdAt: time.Now().UnixNano() / 1e6, + activeTasksMap: make(map[int64]*AtomicTask), + } +} + +// Run 运行工作者 +func (w *CompleteWorker) Run() { + // 设置运行状态 + atomic.StoreInt32(&w.status, int32(WorkerStatusRunning)) + atomic.StoreInt64(&w.lastActive, time.Now().UnixNano()/1e6) + w.logInfo("Worker started") + + // 使用 once 确保只关闭一次 + var once sync.Once + taskChan := make(chan *AtomicTask, 1) + + // 安全关闭函数 + safeCloseTaskChan := func() { + once.Do(func() { + close(taskChan) + }) + } + + // 启动任务获取 goroutine + go func() { + defer safeCloseTaskChan() + + for { + select { + case <-w.ctx.Done(): + return + default: + } + + status := atomic.LoadInt32(&w.status) + if status == int32(WorkerStatusStopped) { + return + } + + // 检查是否暂停 + if status == int32(WorkerStatusPaused) { + time.Sleep(100 * time.Millisecond) + continue + } + + // 获取任务 + task := w.taskQueue.Pop() + if task != nil { + select { + case taskChan <- task: + // 任务已发送 + case <-w.ctx.Done(): + return + } + } else { + // 检查是否正在排空且队列为空 + if status == int32(WorkerStatusDraining) && w.taskQueue.Size() == 0 { + return + } + time.Sleep(10 * time.Millisecond) + } + } + }() + + // 启动健康检查 goroutine + healthCheckDone := make(chan bool, 1) + go func() { + defer func() { + healthCheckDone <- true + }() + w.healthCheckLoop() + }() + + defer func() { + safeCloseTaskChan() + + // 等待健康检查循环退出 + select { + case <-healthCheckDone: + // 健康检查已退出 + case <-time.After(200 * time.Millisecond): + // 超时，继续执行 + } + + if r := recover(); r != nil { + w.logError("Worker panic: %v\nStack: %s", r, string(debug.Stack())) + } + + // 清理资源 + w.cleanup() + w.logInfo("Worker stopped") + }() + + idleTimer := time.NewTimer(time.Duration(w.getConfig().TimeoutMs) * time.Millisecond) + defer idleTimer.Stop() + + for { + select { + case <-w.ctx.Done(): + return + + case msg, ok := <-w.controlChan: + if !ok { + // 控制通道已关闭 + return + } + w.handleControlMessage(msg) + atomic.StoreInt64(&w.lastActive, time.Now().UnixNano()/1e6) + + // 检查是否需要停止 + if msg.Type == ControlMsgStop || msg.Type == ControlMsgDelete { + return + } + + case task, ok := <-taskChan: + if !ok { + // 任务通道已关闭 + return + } + if task == nil { + continue + } + + // 重置空闲计时器 + if !idleTimer.Stop() { + select { + case <-idleTimer.C: + default: + } + } + idleTimer.Reset(time.Duration(w.getConfig().TimeoutMs) * time.Millisecond) + + // 处理任务 + w.executeTask(task) + atomic.StoreInt64(&w.lastActive, time.Now().UnixNano()/1e6) + + case <-idleTimer.C: + if atomic.LoadInt32(&w.status) == int32(WorkerStatusIdle) { + w.logInfo("Worker idle timeout") + return + } + idleTimer.Reset(time.Duration(w.getConfig().TimeoutMs) * time.Millisecond) + } + } +} + +// healthCheckLoop 健康检查循环 +func (w *CompleteWorker) healthCheckLoop() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-w.ctx.Done(): + return + case <-ticker.C: + // 安全发送健康信号 + w.safeSendHealthSignal() + } + } +} + +// safeSendHealthSignal 安全发送健康信号 +func (w *CompleteWorker) safeSendHealthSignal() { + defer func() { + if r := recover(); r != nil { + // 忽略通道已关闭的 panic + w.logWarn("Health signal send panic (channel closed): %v", r) + } + }() + + // 检查通道是否可用 + if w.healthChan == nil { + return + } + + // 尝试发送健康信号（非阻塞） + select { + case w.healthChan <- true: + // 成功发送 + default: + // 通道满，忽略 + } +} + +// getTask 获取任务（支持暂停和排空） +func (w *CompleteWorker) getTask() <-chan *AtomicTask { + taskChan := make(chan *AtomicTask, 1) + + go func() { + for { + status := atomic.LoadInt32(&w.status) + + // 检查状态 + switch status { + case WorkerStatusPaused: + time.Sleep(100 * time.Millisecond) + continue + case WorkerStatusStopped: + close(taskChan) + return + case WorkerStatusDraining: + // 排空模式：只处理现有任务 + if w.taskQueue.Size() == 0 { + close(taskChan) + return + } + } + + // 获取任务 + if task := w.taskQueue.Pop(); task != nil { + taskChan <- task + return + } + + // 短暂休眠避免忙等待 + time.Sleep(10 * time.Millisecond) + } + }() + + return taskChan +} + +// executeTask 执行任务 +func (w *CompleteWorker) executeTask(task *AtomicTask) { + // 记录开始时间 + atomic.StoreInt64(&w.lastActive, time.Now().Unix()) + //atomic.StoreInt64(&task.startTime, time.Now().UnixNano()/1e6) + atomic.StoreInt32(&task.status, int32(TaskStatusRunning)) + + // 添加到活动任务 + w.addActiveTask(task) + + // 更新统计 + atomic.AddInt32(&w.activeTasks, 1) + atomic.AddInt64(&w.taskCount, 1) + + defer func() { + // 移除活动任务 + w.removeActiveTask(task.id) + + // 更新统计 + atomic.AddInt32(&w.activeTasks, -1) + atomic.StoreInt64(&w.lastActive, time.Now().UnixNano()/1e6) + + // 异常恢复 + if r := recover(); r != nil { + w.handlePanic(task, r) + } + }() + + // 执行任务 + w.processTask(task) +} + +// processTask 处理任务 +func (w *CompleteWorker) processTask(task *AtomicTask) { + // 获取函数并执行 + fnName := task.funcName.Load().(string) + param := task.param.Load().(string) + + // 调用全局函数获取器 + fn := globalManager.getFunction(fnName) + if fn == nil { + task.errorMsg.Store(fmt.Sprintf("Function not found: %s", fnName)) + atomic.StoreInt32(&task.status, int32(TaskStatusFailed)) + atomic.AddInt64(&w.errorCount, 1) + // 更新池的失败统计 + w.updatePoolFailedStats() + return + } + + // 执行函数 + startTime := time.Now() + result, err := fn(param) + execDuration := time.Since(startTime).Milliseconds() + + // 记录执行时间 + atomic.StoreInt64(&task.execDuration, execDuration) + atomic.StoreInt64(&task.endTime, time.Now().UnixNano()/1e6) + atomic.StoreInt64(&task.totalDuration, task.endTime-task.submitTime) + + if err != nil { + task.errorMsg.Store(err.Error()) + atomic.StoreInt32(&task.status, int32(TaskStatusFailed)) + atomic.AddInt64(&w.errorCount, 1) + // 更新池的失败统计 + w.updatePoolFailedStats() + } else { + task.result.Store(result) + atomic.StoreInt32(&task.status, int32(TaskStatusCompleted)) + atomic.AddInt64(&w.successCount, 1) + // 更新池的成功统计 + w.updatePoolSuccessStats() + } +} + +// 更新池的成功统计 +func (w *CompleteWorker) updatePoolSuccessStats() { + if globalManager != nil && globalManager.pool != nil { + atomic.AddInt64(&globalManager.pool.successTasks, 1) + } +} + +// 更新池的失败统计 +func (w *CompleteWorker) updatePoolFailedStats() { + if globalManager != nil && globalManager.pool != nil { + atomic.AddInt64(&globalManager.pool.failedTasks, 1) + } +} + +// handlePanic 处理panic +func (w *CompleteWorker) handlePanic(task *AtomicTask, r interface{}) { + task.errorMsg.Store(fmt.Sprintf("Panic: %v", r)) + atomic.StoreInt32(&task.status, int32(TaskStatusFailed)) + atomic.AddInt64(&w.errorCount, 1) + // 更新池的失败统计 + w.updatePoolFailedStats() + + // 记录堆栈 + w.logError("Task panic: %v\nStack: %s", r, string(debug.Stack())) + + // 自动恢复 + if w.getConfig().EnableRecovery && atomic.LoadInt32(&w.status) != int32(WorkerStatusStopped) { + w.restart() + } +} + +// restart 重启工作者 +func (w *CompleteWorker) restart() { + atomic.AddInt32(&w.restartCount, 1) + w.logInfo("Worker restarting (attempt %d)", w.restartCount) + + // 发送重启控制消息 + w.controlChan <- ControlMessage{Type: ControlMsgRestart} + + // 延迟重启 + time.Sleep(time.Duration(w.getConfig().RecoveryDelayMs) * time.Millisecond) +} + +// handleControlMessage 处理控制消息 +func (w *CompleteWorker) handleControlMessage(msg ControlMessage) { + // 确保通道存在 + ensureChannels := func() { + if msg.Response == nil { + msg.Response = make(chan interface{}, 1) + } + if msg.Error == nil { + msg.Error = make(chan error, 1) + } + if msg.Done == nil { + msg.Done = make(chan bool, 1) + } + } + + ensureChannels() + + // 处理消息 + var response interface{} + var err error + + switch msg.Type { + case ControlMsgPause: + w.pause() + response = true + case ControlMsgResume: + w.resume() + response = true + case ControlMsgStop: + // 停止工作者 + w.stop() + response = true + case ControlMsgDrain: + w.drain() + response = true + case ControlMsgRestart: + w.restartWorker() + response = true + case ControlMsgUpdate: + w.updateConfig(msg.Data) + response = true + case ControlMsgHealth: + response = w.getHealthStatus() + case ControlMsgGraceful: + w.gracefulStop() + response = true + case ControlMsgDelete: + w.delete() + response = true + default: + err = fmt.Errorf("unknown control message type: %d", msg.Type) + } + + // 安全发送响应 + w.sendResponse(msg, response, err) +} + +// sendResponse 安全发送响应 +func (w *CompleteWorker) sendResponse(msg ControlMessage, response interface{}, err error) { + // 恢复可能发生的panic + defer func() { + if r := recover(); r != nil { + w.logWarn("Ignore panic when sending response: %v", r) + } + }() + + // 检查是否已停止 + if atomic.LoadInt32(&w.stopped) == 1 { + return + } + + // 发送错误（如果有） + if err != nil && msg.Error != nil { + select { + case msg.Error <- err: + // 成功发送错误 + case <-time.After(100 * time.Millisecond): + // 超时不阻塞，记录日志但不panic + w.logWarn("Error channel blocked or closed, message: %v", err) + } + } + + // 发送响应 + if response != nil && msg.Response != nil { + select { + case msg.Response <- response: + // 成功发送响应 + case <-time.After(100 * time.Millisecond): + // 超时不阻塞，记录日志但不panic + w.logWarn("Response channel blocked or closed, response: %v", response) + } + } + + // 通知完成 + if msg.Done != nil { + select { + case msg.Done <- true: + // 成功通知 + case <-time.After(100 * time.Millisecond): + // 安全关闭通道 + w.logWarn("Done channel blocked, closing safely") + func() { + defer func() { recover() }() + close(msg.Done) + }() + } + } +} + +// GetStatus 获取工作者状态 +func (w *CompleteWorker) GetStatus() int32 { + return atomic.LoadInt32(&w.status) +} + +// IsRunning 检查工作者是否正在运行 +func (w *CompleteWorker) IsRunning() bool { + status := atomic.LoadInt32(&w.status) + return status == int32(WorkerStatusRunning) || + status == int32(WorkerStatusIdle) +} + +// CanAcceptControl 检查工作者是否可以接收控制消息 +func (w *CompleteWorker) CanAcceptControl() bool { + status := atomic.LoadInt32(&w.status) + return status != int32(WorkerStatusStopped) && + atomic.LoadInt32(&w.stopped) == 0 +} + +// 添加健康状态获取方法 +func (w *CompleteWorker) getHealthStatus() map[string]interface{} { + return map[string]interface{}{ + "id": w.id, + "status": atomic.LoadInt32(&w.status), + "taskCount": atomic.LoadInt64(&w.taskCount), + "errorCount": atomic.LoadInt64(&w.errorCount), + "successCount": atomic.LoadInt64(&w.successCount), + "activeTasks": atomic.LoadInt32(&w.activeTasks), + "queuedTasks": w.taskQueue.Size(), + "lastActive": atomic.LoadInt64(&w.lastActive), + "uptime": time.Now().UnixNano()/1e6 - w.createdAt, + "restartCount": atomic.LoadInt32(&w.restartCount), + } +} + +// pause 暂停工作者 +func (w *CompleteWorker) pause() { + // 使用原子操作确保状态正确 + oldStatus := atomic.LoadInt32(&w.status) + + // 允许从运行或空闲状态暂停 + if oldStatus != int32(WorkerStatusRunning) && + oldStatus != int32(WorkerStatusIdle) && + oldStatus != int32(WorkerStatusDraining) { + return + } + + // 首先暂停队列（防止新任务进入） + w.taskQueue.Pause() + + // 原子更新状态 + atomic.StoreInt32(&w.status, int32(WorkerStatusPaused)) + + // 记录日志 + w.logInfo("Worker paused, old status: %d", oldStatus) + + // 更新活动时间 + atomic.StoreInt64(&w.lastActive, time.Now().UnixNano()/1e6) +} + +// resume 恢复工作者（优化版本） +func (w *CompleteWorker) resume() { + oldStatus := atomic.LoadInt32(&w.status) + + // 只允许从暂停状态恢复 + if oldStatus != int32(WorkerStatusPaused) { + // 如果已经在运行状态，也视为成功 + if oldStatus == int32(WorkerStatusRunning) { + w.logInfo("Worker already running, no need to resume") + return + } + w.logWarn("Cannot resume worker from status %d, expected paused (%d)", + oldStatus, WorkerStatusPaused) + return + } + + // 重置排空状态（如果有） + if atomic.LoadInt32(&w.status) == int32(WorkerStatusDraining) { + atomic.StoreInt32(&w.status, int32(WorkerStatusPaused)) + w.taskQueue.ResetDrain() + w.logInfo("Reset worker from draining to paused before resume") + } + + // 恢复队列 + w.taskQueue.Resume() + + // 更新状态为运行中 + if atomic.CompareAndSwapInt32(&w.status, int32(WorkerStatusPaused), int32(WorkerStatusRunning)) { + atomic.StoreInt64(&w.lastActive, time.Now().UnixNano()/1e6) + w.logInfo("Worker resumed from paused state") + } else { + w.logWarn("Worker status changed during resume, current status: %d", + atomic.LoadInt32(&w.status)) + } + + // 发送健康信号 + select { + case w.healthChan <- true: + default: + // 通道满，忽略 + } +} + +// stop 停止工作者 +func (w *CompleteWorker) stop() { + // 使用原子操作确保只停止一次 + if !atomic.CompareAndSwapInt32(&w.stopped, 0, 1) { + w.logWarn("Worker already stopped, ignoring duplicate stop request") + return + } + + // 更新状态 + atomic.StoreInt32(&w.status, int32(WorkerStatusStopped)) + w.logInfo("Worker %d stopping...", w.id) + + // 先取消上下文，这会停止 Run 主循环 + if w.cancel != nil { + w.cancel() + } + + // 排空队列（如果队列不为nil） + if w.taskQueue != nil { + w.taskQueue.Drain() + } + + // 安全清理资源（延迟执行，确保 Run 循环完全退出） + go func() { + // 等待 Run 循环完全退出 + time.Sleep(100 * time.Millisecond) + w.cleanup() + }() +} + +// IsStopped 检查工作者是否已停止 +func (w *CompleteWorker) IsStopped() bool { + return atomic.LoadInt32(&w.stopped) == 1 +} + +// drain 排空工作者 +func (w *CompleteWorker) drain() { + atomic.StoreInt32(&w.status, int32(WorkerStatusDraining)) + w.taskQueue.Drain() + w.logInfo("Worker draining") +} + +// restartWorker 重启工作者 +func (w *CompleteWorker) restartWorker() { + currentStatus := atomic.LoadInt32(&w.status) + + // 如果已经在停止状态，直接返回 + if currentStatus == int32(WorkerStatusStopped) { + return + } + + w.logInfo("Worker restarting (attempt %d)", atomic.LoadInt32(&w.restartCount)+1) + + // 标记为排空状态 + atomic.StoreInt32(&w.status, int32(WorkerStatusDraining)) + w.taskQueue.Drain() + + // 等待活动任务完成 + for i := 0; i < 10; i++ { + if atomic.LoadInt32(&w.activeTasks) == 0 { + break + } + time.Sleep(50 * time.Millisecond) + } + + // 停止当前工作者 + w.stop() + + // 如果是自动重启，创建新的工作者 + if w.getConfig().AutoRestart { + // 创建新的工作者 + newWorker := NewCompleteWorker(w.id, w.config.Name, w.config) + newWorker.restartCount = atomic.AddInt32(&w.restartCount, 1) + + // 替换全局管理器中的工作者 + if globalManager != nil { + globalManager.mu.Lock() + if existing, ok := globalManager.pool.workers.Load(w.id); ok { + // 确保旧工作者完全停止 + if existing.(*CompleteWorker) != w { + existing.(*CompleteWorker).stop() + } + } + globalManager.replaceWorker(w.id, newWorker) + globalManager.mu.Unlock() + + // 启动新的工作者 + go newWorker.Run() + w.logInfo("Worker %d restarted successfully", w.id) + } + } +} + +// updateConfig 更新配置 +func (w *CompleteWorker) updateConfig(data interface{}) { + if config, ok := data.(*WorkerConfig); ok { + w.configMu.Lock() + w.config = config + w.configMu.Unlock() + w.logInfo("Worker config updated") + } +} + +// reportHealth 报告健康状态 +func (w *CompleteWorker) reportHealth(msg ControlMessage) { + health := map[string]interface{}{ + "id": w.id, + "status": atomic.LoadInt32(&w.status), + "taskCount": atomic.LoadInt64(&w.taskCount), + "errorCount": atomic.LoadInt64(&w.errorCount), + "activeTasks": atomic.LoadInt32(&w.activeTasks), + "queuedTasks": w.taskQueue.Size(), + "lastActive": atomic.LoadInt64(&w.lastActive), + "uptime": time.Now().UnixNano()/1e6 - w.createdAt, + "restartCount": atomic.LoadInt32(&w.restartCount), + } + + if msg.Response != nil { + select { + case msg.Response <- health: + default: + } + } +} + +// delete 删除工作者 +func (w *CompleteWorker) delete() { + w.stop() + globalManager.removeWorker(w.id) + w.logInfo("Worker deleted") +} + +// SendControlSafe 安全地发送控制消息 +func (w *CompleteWorker) SendControlSafe(msg ControlMessage) (response interface{}, err error) { + // 检查是否已停止 + if w.IsStopped() { + return nil, fmt.Errorf("worker is stopped") + } + + // 检查状态是否允许接收控制消息 + status := atomic.LoadInt32(&w.status) + if status == int32(WorkerStatusStopped) { + return nil, fmt.Errorf("worker is stopped, status: %d", status) + } + + // 确保响应通道 + if msg.Response == nil { + msg.Response = make(chan interface{}, 1) + } + if msg.Error == nil { + msg.Error = make(chan error, 1) + } + if msg.Done == nil { + msg.Done = make(chan bool, 1) + } + + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("send control panic: %v", r) + } + }() + + // 尝试发送消息（带超时） + timeout := time.After(3 * time.Second) + + select { + case w.controlChan <- msg: + // 消息已发送，等待响应 + select { + case <-msg.Done: + // 尝试从响应通道获取数据 + select { + case resp := <-msg.Response: + return resp, nil + case err := <-msg.Error: + return nil, err + case <-time.After(500 * time.Millisecond): + // 如果没有响应，检查工作者是否已停止 + if w.IsStopped() { + return nil, fmt.Errorf("worker stopped during operation") + } + return nil, fmt.Errorf("response timeout") + } + case <-timeout: + // 检查工作者是否已停止 + if w.IsStopped() { + return nil, fmt.Errorf("worker stopped during timeout") + } + return nil, fmt.Errorf("message processing timeout") + } + case <-timeout: + // 检查工作者是否已停止 + if w.IsStopped() { + return nil, fmt.Errorf("worker stopped, cannot send control") + } + return nil, fmt.Errorf("control channel send timeout") + case <-w.ctx.Done(): + return nil, fmt.Errorf("worker context cancelled") + } +} + +// gracefulStop 优雅停止 +func (w *CompleteWorker) gracefulStop() { + w.drain() + + // 等待所有任务完成 + for { + if atomic.LoadInt32(&w.activeTasks) == 0 && w.taskQueue.Size() == 0 { + break + } + time.Sleep(100 * time.Millisecond) + } + + w.stop() + w.logInfo("Worker gracefully stopped") +} + +// checkHealth 检查健康状态 +func (w *CompleteWorker) checkHealth() { + // 检查是否卡住 + lastActive := atomic.LoadInt64(&w.lastActive) + now := time.Now().UnixNano() / 1e6 + + // 如果 lastActive 是0，说明刚创建，跳过检查 + if lastActive == 0 { + return + } + + idleTime := now - lastActive + + if idleTime > int64(w.getConfig().TimeoutMs*2) { + w.logWarn("Worker可能卡住，最后活动时间: %dms前", idleTime) + + // 自动恢复 + if w.getConfig().EnableRecovery { + w.restart() + } + } + + // 检查内存使用 + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + if w.getConfig().MemoryLimitKB > 0 { + memoryUsageKB := int64(memStats.HeapInuse / 1024) + if memoryUsageKB > w.getConfig().MemoryLimitKB { + w.logWarn("Worker内存使用过高: %dKB (限制: %dKB)", + memoryUsageKB, w.getConfig().MemoryLimitKB) + } + } +} + +// addActiveTask 添加活动任务 +func (w *CompleteWorker) addActiveTask(task *AtomicTask) { + w.tasksMu.Lock() + w.activeTasksMap[task.id] = task + w.tasksMu.Unlock() +} + +// removeActiveTask 移除活动任务 +func (w *CompleteWorker) removeActiveTask(taskID int64) { + w.tasksMu.Lock() + delete(w.activeTasksMap, taskID) + w.tasksMu.Unlock() +} + +// getActiveTasks 获取活动任务 +func (w *CompleteWorker) getActiveTasks() []*AtomicTask { + w.tasksMu.RLock() + defer w.tasksMu.RUnlock() + + tasks := make([]*AtomicTask, 0, len(w.activeTasksMap)) + for _, task := range w.activeTasksMap { + tasks = append(tasks, task) + } + return tasks +} + +// getConfig 获取配置（线程安全） +func (w *CompleteWorker) getConfig() *WorkerConfig { + w.configMu.RLock() + defer w.configMu.RUnlock() + return w.config +} + +// cleanup 清理资源 +func (w *CompleteWorker) cleanup() { + w.once.Do(func() { + // 首先标记为已停止 + atomic.StoreInt32(&w.status, int32(WorkerStatusStopped)) + atomic.StoreInt32(&w.stopped, 1) + + w.logInfo("Worker %d cleanup started", w.id) + + // 首先取消上下文，这会触发 healthCheckLoop 退出 + if w.cancel != nil { + w.cancel() + } + + // 等待一小段时间，确保循环退出 + time.Sleep(50 * time.Millisecond) + + // 安全关闭健康通道 + w.safeCloseHealthChan() + + // 然后关闭控制通道 + w.safeCloseControlChan() + + // 取消所有活动任务 + w.tasksMu.Lock() + for _, task := range w.activeTasksMap { + if task.cancel != nil { + task.cancel() + } + } + w.activeTasksMap = make(map[int64]*AtomicTask) + w.tasksMu.Unlock() + + // 排空队列 + w.taskQueue.Drain() + + w.logInfo("Worker %d cleanup completed", w.id) + }) +} + +// safeCloseHealthChan 安全关闭健康通道 +func (w *CompleteWorker) safeCloseHealthChan() { + defer func() { + if r := recover(); r != nil { + // 忽略通道已关闭的 panic + w.logWarn("Ignore panic when closing health channel: %v", r) + } + }() + + if w.healthChan != nil { + close(w.healthChan) + w.healthChan = nil + } +} + +// safeCloseControlChan 安全关闭控制通道 +func (w *CompleteWorker) safeCloseControlChan() { + defer func() { + if r := recover(); r != nil { + // 忽略通道已关闭的 panic + w.logWarn("Ignore panic when closing control channel: %v", r) + } + }() + + if w.controlChan != nil { + close(w.controlChan) + w.controlChan = nil + } +} + +// waitForStatus 状态验证 +func (w *CompleteWorker) waitForStatus(targetStatus int32, timeoutMs int) bool { + deadline := time.Now().Add(time.Duration(timeoutMs) * time.Millisecond) + + for time.Now().Before(deadline) { + currentStatus := atomic.LoadInt32(&w.status) + if currentStatus == targetStatus { + return true + } + time.Sleep(10 * time.Millisecond) + } + + return false +} + +// logInfo 记录信息日志 +func (w *CompleteWorker) logInfo(format string, args ...interface{}) { + logMessage(LogLevelInfo, "[Worker %d] %s", w.id, fmt.Sprintf(format, args...)) +} + +// logError 记录错误日志 +func (w *CompleteWorker) logError(format string, args ...interface{}) { + logMessage(LogLevelError, "[Worker %d] %s", w.id, fmt.Sprintf(format, args...)) +} + +// logWarn 记录警告日志 +func (w *CompleteWorker) logWarn(format string, args ...interface{}) { + logMessage(LogLevelWarn, "[Worker %d] %s", w.id, fmt.Sprintf(format, args...)) +} + +// ==================== 完整的协程池实现 ==================== + +// CompletePool 完整的协程池 +type CompletePool struct { + // 原子状态 + status int32 + startTime int64 + workerSeq int32 + + // 工作者管理 + workers sync.Map // map[int32]*CompleteWorker + workerCount int32 + workerStatus map[int32][]int32 // 按状态分组 + + // 配置 + config *PoolConfig + configMu sync.RWMutex + + // 原子统计 + totalTasks int64 + successTasks int64 + failedTasks int64 + cancelledTasks int64 + totalWorkers int32 + runningWorkers int32 + idleWorkers int32 + pausedWorkers int32 + stoppedWorkers int32 + drainingWorkers int32 + queuedTasks int32 + activeTasks int32 + + // 函数注册 + functions sync.Map // map[string]TaskFunc + + // 控制 + controlChan chan PoolControlMessage + shutdownCtx context.Context + shutdownCancel context.CancelFunc +} + +// PoolControlMessage 池控制消息 +type PoolControlMessage struct { + Type int + WorkerID int32 + Data interface{} + Response chan interface{} + Error chan error + Done chan bool +} + +// NewCompletePool 创建完整的协程池 +func NewCompletePool(config *PoolConfig) *CompletePool { + if config == nil { + config = &PoolConfig{ + MinWorkers: DefaultMinWorkers, + MaxWorkers: DefaultMaxWorkers, + QueueSize: DefaultQueueSize, + TaskTimeoutMs: DefaultTaskTimeout, + WorkerIdleTimeoutMs: DefaultIdleTimeout, + MemoryLimitMB: DefaultMaxMemoryMB, + ShutdownTimeoutMs: DefaultShutdownTimeout, + HealthCheckDelayMs: DefaultHealthCheckDelay, + MaxRetries: DefaultMaxRetries, + DefaultLanguage: LanguageEN, + EnableMetrics: true, + EnableAutoScaling: true, + PriorityQueue: false, + } + } + + ctx, cancel := context.WithCancel(context.Background()) + + pool := &CompletePool{ + status: int32(WorkerStatusRunning), + startTime: time.Now().UnixNano() / 1e6, + config: config, + controlChan: make(chan PoolControlMessage, 100), + shutdownCtx: ctx, + shutdownCancel: cancel, + workerStatus: make(map[int32][]int32), + } + + // 初始化工作者 + pool.initializeWorkers() + + // 启动控制循环 + go pool.controlLoop() + + // 启动统计收集 + if config.EnableMetrics { + go pool.collectMetrics() + } + + pool.logInfo("Pool initialized with %d workers", config.MinWorkers) + return pool +} + +// initializeWorkers 初始化工作者 +func (p *CompletePool) initializeWorkers() { + for i := 0; i < p.config.MinWorkers; i++ { + p.createWorker() + } + + atomic.StoreInt32(&p.totalWorkers, int32(p.config.MinWorkers)) + atomic.StoreInt32(&p.runningWorkers, int32(p.config.MinWorkers)) +} + +// createWorker 创建工作者 +func (p *CompletePool) createWorker() int32 { + // 检查最大工作者数 + if atomic.LoadInt32(&p.workerCount) >= int32(p.config.MaxWorkers) { + return -1 + } + + workerID := atomic.AddInt32(&p.workerSeq, 1) + config := &WorkerConfig{ + ID: workerID, + Name: fmt.Sprintf("worker_%d", workerID), + QueueSize: int32(p.config.QueueSize), + TimeoutMs: int32(p.config.TaskTimeoutMs), + MaxRetries: int32(p.config.MaxRetries), + AutoRestart: true, // 改为 true + EnableRecovery: true, // 改为 true + RecoveryDelayMs: 1000, + PriorityEnabled: p.config.PriorityQueue, + } + + worker := NewCompleteWorker(workerID, config.Name, config) + + // 保存工作者 + p.workers.Store(workerID, worker) + atomic.AddInt32(&p.workerCount, 1) + + // 更新状态分组 + p.addWorkerToStatus(workerID, int32(WorkerStatusRunning)) + + // 启动工作者 + go worker.Run() + + // 等待工作者完全启动（减少等待时间） + time.Sleep(10 * time.Millisecond) + + p.logInfo("Worker %d created", workerID) + return workerID +} + +// controlLoop 控制循环 +func (p *CompletePool) controlLoop() { + for { + select { + case <-p.shutdownCtx.Done(): + return + case msg := <-p.controlChan: + p.handlePoolControl(msg) + } + } +} + +// handlePoolControl 处理池控制消息 +func (p *CompletePool) handlePoolControl(msg PoolControlMessage) { + // 创建响应和错误通道（如果未提供） + var responseChan chan interface{} + var errorChan chan error + var doneChan chan bool + + if msg.Response == nil { + responseChan = make(chan interface{}, 1) + msg.Response = responseChan + } + + if msg.Error == nil { + errorChan = make(chan error, 1) + msg.Error = errorChan + } + + if msg.Done == nil { + doneChan = make(chan bool, 1) + msg.Done = doneChan + } + + // 在goroutine中处理，避免阻塞控制循环 + go func() { + defer func() { + if r := recover(); r != nil { + if msg.Error != nil { + select { + case msg.Error <- fmt.Errorf("panic in control handler: %v", r): + default: + } + } + } + if msg.Done != nil { + close(msg.Done) + } + }() + + switch msg.Type { + case 1: // 暂停工作者 + p.pauseWorker(msg.WorkerID, msg) + case 2: // 恢复工作者 + p.resumeWorker(msg.WorkerID, msg) + case 3: // 停止工作者 + p.stopWorker(msg.WorkerID, msg) + case 4: // 排空工作者 + p.drainWorker(msg.WorkerID, msg) + case 5: // 重启工作者 + p.restartWorker(msg.WorkerID, msg) + case 6: // 更新工作者配置 + p.updateWorkerConfig(msg.WorkerID, msg.Data, msg) + case 7: // 删除工作者 + p.deleteWorker(msg.WorkerID, msg) + case 8: // 获取工作者状态 + p.getWorkerStatus(msg.WorkerID, msg) + case 9: // 创建工作者 + p.createNewWorker(msg) + case 10: // 暂停所有工作者 + p.pauseAllWorkers(msg) + case 11: // 恢复所有工作者 + p.resumeAllWorkers(msg) + case 12: // 停止所有工作者 + p.stopAllWorkers(msg) + case 13: // 优雅关闭 + p.gracefulShutdown(msg) + case 14: // 更新池配置 + p.updatePoolConfig(msg.Data, msg) + default: + if msg.Error != nil { + msg.Error <- fmt.Errorf("unknown control type: %d", msg.Type) + } + } + }() +} + +// pauseWorker 暂停工作者 +func (p *CompletePool) pauseWorker(workerID int32, msg PoolControlMessage) { + if worker, ok := p.workers.Load(workerID); ok { + // 创建响应通道 + responseChan := make(chan interface{}, 1) + errorChan := make(chan error, 1) + doneChan := make(chan bool, 1) + + workerMsg := ControlMessage{ + Type: ControlMsgPause, + Response: responseChan, + Error: errorChan, + Done: doneChan, + } + + // 发送控制消息 + select { + case worker.(*CompleteWorker).controlChan <- workerMsg: + // 等待响应 + select { + case <-doneChan: + // 更新状态分组 + p.removeWorkerFromStatus(workerID, int32(WorkerStatusRunning)) + p.addWorkerToStatus(workerID, int32(WorkerStatusPaused)) + + atomic.AddInt32(&p.runningWorkers, -1) + atomic.AddInt32(&p.pausedWorkers, 1) + + // 发送成功响应 + if msg.Response != nil { + select { + case msg.Response <- true: + default: + } + } + case <-time.After(2 * time.Second): + if msg.Error != nil { + msg.Error <- fmt.Errorf("pause timeout") + } + } + case <-time.After(1 * time.Second): + if msg.Error != nil { + msg.Error <- fmt.Errorf("failed to send control message") + } + } + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("worker not found") + } +} + +// resumeWorker 恢复工作者 +func (p *CompletePool) resumeWorker(workerID int32, msg PoolControlMessage) { + if worker, ok := p.workers.Load(workerID); ok { + // 简化逻辑，直接发送控制消息 + responseChan := make(chan interface{}, 1) + errorChan := make(chan error, 1) + doneChan := make(chan bool, 1) + + // 使用goroutine发送控制消息，避免阻塞 + go func() { + defer func() { + if r := recover(); r != nil { + if errorChan != nil { + errorChan <- fmt.Errorf("panic in resume worker: %v", r) + } + } + }() + + // 使用工作者提供的安全发送方法 + resp, err := worker.(*CompleteWorker).SendControlSafe(ControlMessage{ + Type: ControlMsgResume, + Response: responseChan, + Error: errorChan, + Done: doneChan, + }) + + if err != nil { + if errorChan != nil { + errorChan <- err + } + } else if responseChan != nil { + responseChan <- resp + } + }() + + // 等待响应 + timeout := time.After(5 * time.Second) + + select { + case resp := <-responseChan: + // 更新状态分组和统计 + p.removeWorkerFromStatus(workerID, int32(WorkerStatusPaused)) + p.addWorkerToStatus(workerID, int32(WorkerStatusRunning)) + + // 更新统计 + if atomic.AddInt32(&p.pausedWorkers, -1) < 0 { + atomic.StoreInt32(&p.pausedWorkers, 0) + } + atomic.AddInt32(&p.runningWorkers, 1) + + // 发送响应给调用者 + if msg.Response != nil { + select { + case msg.Response <- resp: + default: + } + } + + case err := <-errorChan: + if msg.Error != nil { + msg.Error <- err + } + + case <-doneChan: + // 操作完成，发送默认响应 + if msg.Response != nil { + msg.Response <- true + } + + case <-timeout: + if msg.Error != nil { + msg.Error <- fmt.Errorf("resume worker timeout") + } + } + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("worker not found") + } +} + +// stopWorker 停止工作者 +func (p *CompletePool) stopWorker(workerID int32, msg PoolControlMessage) { + if worker, ok := p.workers.Load(workerID); ok { + workerInst := worker.(*CompleteWorker) + + // 检查是否已停止 + if workerInst.IsStopped() { + if msg.Error != nil { + select { + case msg.Error <- fmt.Errorf("worker already stopped"): + default: + } + } + return + } + + // 创建响应通道 + responseChan := make(chan interface{}, 1) + errorChan := make(chan error, 1) + doneChan := make(chan bool, 1) + + workerMsg := ControlMessage{ + Type: ControlMsgStop, + Response: responseChan, + Error: errorChan, + Done: doneChan, + } + + // 使用goroutine发送停止消息，避免阻塞 + go func() { + defer func() { + if r := recover(); r != nil { + // 发送错误响应 + if msg.Error != nil { + select { + case msg.Error <- fmt.Errorf("worker stop panic: %v", r): + default: + } + } + } + }() + + // 使用安全的发送方法 + _, err := workerInst.SendControlSafe(workerMsg) + if err != nil { + if msg.Error != nil { + select { + case msg.Error <- err: + default: + } + } + return + } + + // 等待停止完成 + select { + case <-doneChan: + // 从池中移除工作者 + p.workers.Delete(workerID) + atomic.AddInt32(&p.workerCount, -1) + + // 更新状态分组和统计 + p.removeWorkerFromStatus(workerID, int32(WorkerStatusStopped)) + atomic.AddInt32(&p.stoppedWorkers, 1) + + // 发送成功响应 + if msg.Response != nil { + select { + case msg.Response <- map[string]interface{}{ + "success": true, + "workerID": workerID, + }: + default: + } + } + case <-time.After(5 * time.Second): + // 超时，强制移除 + p.workers.Delete(workerID) + if msg.Error != nil { + select { + case msg.Error <- fmt.Errorf("worker stop timeout, forced removal"): + default: + } + } + } + }() + } else if msg.Error != nil { + select { + case msg.Error <- fmt.Errorf("worker not found"): + default: + } + } +} + +// drainWorker 排空工作者 +func (p *CompletePool) drainWorker(workerID int32, msg PoolControlMessage) { + if worker, ok := p.workers.Load(workerID); ok { + worker.(*CompleteWorker).controlChan <- ControlMessage{ + Type: ControlMsgDrain, + Response: msg.Response, + Error: msg.Error, + } + + // 更新状态分组 + oldStatus := worker.(*CompleteWorker).status + p.removeWorkerFromStatus(workerID, oldStatus) + p.addWorkerToStatus(workerID, int32(WorkerStatusDraining)) + + // 更新统计 + switch oldStatus { + case WorkerStatusRunning: + atomic.AddInt32(&p.runningWorkers, -1) + case WorkerStatusPaused: + atomic.AddInt32(&p.pausedWorkers, -1) + } + + atomic.AddInt32(&p.drainingWorkers, 1) + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("worker not found") + } +} + +// restartWorker 重启工作者 +func (p *CompletePool) restartWorker(workerID int32, msg PoolControlMessage) { + if worker, ok := p.workers.Load(workerID); ok { + worker.(*CompleteWorker).controlChan <- ControlMessage{ + Type: ControlMsgRestart, + Response: msg.Response, + Error: msg.Error, + } + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("worker not found") + } +} + +// deleteWorker 删除工作者 +func (p *CompletePool) deleteWorker(workerID int32, msg PoolControlMessage) { + if worker, ok := p.workers.Load(workerID); ok { + worker.(*CompleteWorker).controlChan <- ControlMessage{ + Type: ControlMsgDelete, + Response: msg.Response, + Error: msg.Error, + } + + // 从池中移除 + p.workers.Delete(workerID) + atomic.AddInt32(&p.workerCount, -1) + + // 更新统计 + atomic.AddInt32(&p.totalWorkers, -1) + status := atomic.LoadInt32(&worker.(*CompleteWorker).status) + switch status { + case WorkerStatusRunning: + atomic.AddInt32(&p.runningWorkers, -1) + case WorkerStatusPaused: + atomic.AddInt32(&p.pausedWorkers, -1) + case WorkerStatusIdle: + atomic.AddInt32(&p.idleWorkers, -1) + case WorkerStatusStopped: + atomic.AddInt32(&p.stoppedWorkers, -1) + case WorkerStatusDraining: + atomic.AddInt32(&p.drainingWorkers, -1) + } + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("worker not found") + } +} + +// updateWorkerConfig 更新工作者配置 +func (p *CompletePool) updateWorkerConfig(workerID int32, data interface{}, msg PoolControlMessage) { + if worker, ok := p.workers.Load(workerID); ok { + if config, ok := data.(*WorkerConfig); ok { + worker.(*CompleteWorker).controlChan <- ControlMessage{ + Type: ControlMsgUpdate, + Data: config, + Response: msg.Response, + Error: msg.Error, + } + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("invalid config data") + } + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("worker not found") + } +} + +// getWorkerStatus 获取工作者状态 +func (p *CompletePool) getWorkerStatus(workerID int32, msg PoolControlMessage) { + if worker, ok := p.workers.Load(workerID); ok { + // 使用带缓冲的通道 + responseChan := make(chan interface{}, 1) + errorChan := make(chan error, 1) + doneChan := make(chan bool, 1) + + workerMsg := ControlMessage{ + Type: ControlMsgHealth, + Response: responseChan, + Error: errorChan, + Done: doneChan, + } + + // 发送控制消息 + select { + case worker.(*CompleteWorker).controlChan <- workerMsg: + // 等待响应 + select { + case <-doneChan: + select { + case resp := <-responseChan: + if msg.Response != nil { + msg.Response <- resp + } + case err := <-errorChan: + if msg.Error != nil { + msg.Error <- err + } + default: + if msg.Error != nil { + msg.Error <- fmt.Errorf("no response from worker") + } + } + case <-time.After(2 * time.Second): + if msg.Error != nil { + msg.Error <- fmt.Errorf("response timeout") + } + } + case <-time.After(1 * time.Second): + if msg.Error != nil { + msg.Error <- fmt.Errorf("failed to send control message") + } + } + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("worker not found") + } +} + +// createNewWorker 创建新工作者 +func (p *CompletePool) createNewWorker(msg PoolControlMessage) { + workerID := p.createWorker() + if workerID > 0 && msg.Response != nil { + msg.Response <- map[string]interface{}{ + "workerID": workerID, + "success": true, + } + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("failed to create worker") + } +} + +// pauseAllWorkers 暂停所有工作者 +func (p *CompletePool) pauseAllWorkers(msg PoolControlMessage) { + p.workers.Range(func(key, value interface{}) bool { + worker := value.(*CompleteWorker) + worker.controlChan <- ControlMessage{ + Type: ControlMsgPause, + } + return true + }) + + // 更新统计 + atomic.StoreInt32(&p.pausedWorkers, atomic.LoadInt32(&p.workerCount)) + atomic.StoreInt32(&p.runningWorkers, 0) + atomic.StoreInt32(&p.idleWorkers, 0) + atomic.StoreInt32(&p.drainingWorkers, 0) + + if msg.Response != nil { + msg.Response <- map[string]interface{}{ + "success": true, + "workers": atomic.LoadInt32(&p.workerCount), + } + } +} + +// resumeAllWorkers 恢复所有工作者 +func (p *CompletePool) resumeAllWorkers(msg PoolControlMessage) { + p.workers.Range(func(key, value interface{}) bool { + worker := value.(*CompleteWorker) + if atomic.LoadInt32(&worker.status) == int32(WorkerStatusPaused) { + worker.controlChan <- ControlMessage{ + Type: ControlMsgResume, + } + } + return true + }) + + // 更新统计 + atomic.StoreInt32(&p.runningWorkers, atomic.LoadInt32(&p.pausedWorkers)) + atomic.StoreInt32(&p.pausedWorkers, 0) + + if msg.Response != nil { + msg.Response <- map[string]interface{}{ + "success": true, + "workers": atomic.LoadInt32(&p.runningWorkers), + } + } +} + +// stopAllWorkers 停止所有工作者 +func (p *CompletePool) stopAllWorkers(msg PoolControlMessage) { + p.workers.Range(func(key, value interface{}) bool { + worker := value.(*CompleteWorker) + worker.controlChan <- ControlMessage{ + Type: ControlMsgStop, + } + return true + }) + + // 更新统计 + atomic.StoreInt32(&p.stoppedWorkers, atomic.LoadInt32(&p.workerCount)) + atomic.StoreInt32(&p.runningWorkers, 0) + atomic.StoreInt32(&p.pausedWorkers, 0) + atomic.StoreInt32(&p.idleWorkers, 0) + atomic.StoreInt32(&p.drainingWorkers, 0) + + if msg.Response != nil { + msg.Response <- map[string]interface{}{ + "success": true, + "workers": atomic.LoadInt32(&p.workerCount), + } + } +} + +// gracefulShutdown 优雅关闭 +func (p *CompletePool) gracefulShutdown(msg PoolControlMessage) { + // 先标记为排空状态，防止新任务提交 + atomic.StoreInt32(&p.status, int32(WorkerStatusDraining)) + + logMessage(LogLevelInfo, "Pool graceful shutdown initiated") + + // 收集所有工作者 + workers := make([]*CompleteWorker, 0) + p.workers.Range(func(key, value interface{}) bool { + worker := value.(*CompleteWorker) + workers = append(workers, worker) + return true + }) + + // 排空所有工作者 + for _, worker := range workers { + p.safeSendControlToWorker(worker.id, ControlMessage{Type: ControlMsgDrain}) + } + + // 等待所有任务完成 + timeout := time.Duration(p.config.ShutdownTimeoutMs) * time.Millisecond + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + allTasksCompleted := false + attempts := 0 + + for !allTasksCompleted && attempts < 100 { // 最多等待10秒 + select { + case <-ctx.Done(): + logMessage(LogLevelWarn, "Graceful shutdown timeout, forcing stop") + break + case <-ticker.C: + // 检查所有任务是否完成 + allDone := true + for _, worker := range workers { + if atomic.LoadInt32(&worker.activeTasks) > 0 || worker.taskQueue.Size() > 0 { + allDone = false + break + } + } + + if allDone { + allTasksCompleted = true + logMessage(LogLevelInfo, "All tasks completed, stopping workers") + break + } + + attempts++ + + // 每1秒记录一次进度 + if attempts%10 == 0 { + totalActive := int32(0) + totalQueued := int32(0) + for _, worker := range workers { + totalActive += atomic.LoadInt32(&worker.activeTasks) + totalQueued += worker.taskQueue.Size() + } + logMessage(LogLevelInfo, "Graceful shutdown in progress: %d active tasks, %d queued tasks", + totalActive, totalQueued) + } + } + } + + // 停止所有工作者 + for _, worker := range workers { + p.safeSendControlToWorker(worker.id, ControlMessage{Type: ControlMsgStop}) + } + + // 等待工作者停止 + time.Sleep(200 * time.Millisecond) + + // 关闭池 + p.shutdownCancel() + + // 更新状态 + atomic.StoreInt32(&p.status, int32(WorkerStatusStopped)) + + // 清空工作者列表 + p.workers = sync.Map{} + atomic.StoreInt32(&p.workerCount, 0) + + logMessage(LogLevelInfo, "Pool graceful shutdown completed") + + // 发送响应 + if msg.Response != nil { + select { + case msg.Response <- map[string]interface{}{ + "success": true, + "message": "pool gracefully shutdown", + }: + default: + } + } +} + +// updatePoolConfig 更新池配置 +func (p *CompletePool) updatePoolConfig(data interface{}, msg PoolControlMessage) { + if config, ok := data.(*PoolConfig); ok { + p.configMu.Lock() + p.config = config + p.configMu.Unlock() + + if msg.Response != nil { + msg.Response <- map[string]interface{}{ + "success": true, + "message": "pool config updated", + } + } + } else if msg.Error != nil { + msg.Error <- fmt.Errorf("invalid config data") + } +} + +// addWorkerToStatus 添加工作者到状态分组 +func (p *CompletePool) addWorkerToStatus(workerID, status int32) { + // 这里需要锁来更新map + // 由于访问不频繁，使用锁是可以接受的 + if _, ok := p.workerStatus[status]; !ok { + p.workerStatus[status] = make([]int32, 0) + } + p.workerStatus[status] = append(p.workerStatus[status], workerID) +} + +// removeWorkerFromStatus 从状态分组移除工作者 +func (p *CompletePool) removeWorkerFromStatus(workerID, status int32) { + if workers, ok := p.workerStatus[status]; ok { + for i, id := range workers { + if id == workerID { + p.workerStatus[status] = append(workers[:i], workers[i+1:]...) + break + } + } + } +} + +// collectMetrics 收集指标 +func (p *CompletePool) collectMetrics() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + p.updateStatistics() + case <-p.shutdownCtx.Done(): + return + } + } +} + +// updateStatistics 更新统计 +func (p *CompletePool) updateStatistics() { + // 收集工作者统计 + var totalQueued, totalActive int32 + p.workers.Range(func(key, value interface{}) bool { + worker := value.(*CompleteWorker) + totalQueued += worker.taskQueue.Size() + totalActive += atomic.LoadInt32(&worker.activeTasks) + return true + }) + + atomic.StoreInt32(&p.queuedTasks, totalQueued) + atomic.StoreInt32(&p.activeTasks, totalActive) +} + +// SubmitTask 提交任务 +func (p *CompletePool) SubmitTask(workerID int32, funcName, param string, priority int32) (int64, error) { + if atomic.LoadInt32(&p.status) != int32(WorkerStatusRunning) { + return -1, fmt.Errorf("pool not running") + } + + // 检查函数是否注册 + if _, ok := p.functions.Load(funcName); !ok { + return -1, fmt.Errorf("function not registered") + } + + // 选择工作者 + var worker *CompleteWorker + if workerID > 0 { + if w, ok := p.workers.Load(workerID); ok { + worker = w.(*CompleteWorker) + // 检查工作者状态 - 增强检查 + workerStatus := atomic.LoadInt32(&worker.status) + if workerStatus != int32(WorkerStatusRunning) && + workerStatus != int32(WorkerStatusIdle) && + workerStatus != int32(WorkerStatusDraining) { + return -1, fmt.Errorf("worker not available, status: %d", workerStatus) + } + // 额外检查：如果工作者暂停了，队列也应该拒绝任务 + if workerStatus == int32(WorkerStatusPaused) { + return -1, fmt.Errorf("worker paused") + } + } else { + return -1, fmt.Errorf("worker not found") + } + } else { + // 负载均衡选择 - 排除暂停的工作者 + worker = p.selectWorker() + if worker == nil { + return -1, fmt.Errorf("no available worker") + } + // 确保选中的工作者不是暂停状态 + if atomic.LoadInt32(&worker.status) == int32(WorkerStatusPaused) { + return -1, fmt.Errorf("selected worker is paused") + } + } + + // 创建任务 + task := NewAtomicTask(int(worker.id), funcName, param, + int(priority), int(p.config.MaxRetries), p.config.TaskTimeoutMs) + + // 提交到任务队列 + if !worker.taskQueue.Push(task) { + return -1, fmt.Errorf("queue full or worker paused") + } + + // 注册任务到全局注册表 + if globalManager != nil && globalManager.registry != nil { + globalManager.registry.Register(task.id, task) + } + + // 更新统计 + atomic.AddInt64(&p.totalTasks, 1) + + return task.id, nil +} + +// selectWorker 选择工作者（负载均衡） +func (p *CompletePool) selectWorker() *CompleteWorker { + var selected *CompleteWorker + var minLoad int32 = 1<<31 - 1 + + p.workers.Range(func(key, value interface{}) bool { + worker := value.(*CompleteWorker) + status := atomic.LoadInt32(&worker.status) + + // 只选择运行中的工作者 + if status != int32(WorkerStatusRunning) && status != int32(WorkerStatusIdle) { + return true + } + + load := worker.taskQueue.Size() + if load < minLoad { + minLoad = load + selected = worker + } + + return true + }) + + return selected +} + +// RegisterFunction 注册函数 +func (p *CompletePool) RegisterFunction(name string, fn TaskFunc) error { + p.functions.Store(name, fn) + return nil +} + +// GetFunction 获取函数 +func (p *CompletePool) GetFunction(name string) TaskFunc { + if fn, ok := p.functions.Load(name); ok { + return fn.(TaskFunc) + } + return nil +} + +// SendControl 发送控制消息 +func (p *CompletePool) SendControl(msg PoolControlMessage) (interface{}, error) { + // 检查池状态 + if atomic.LoadInt32(&p.status) != int32(WorkerStatusRunning) { + return nil, fmt.Errorf("pool not running") + } + + done := make(chan bool, 1) + response := make(chan interface{}, 1) + errChan := make(chan error, 1) + + msg.Done = done + msg.Response = response + msg.Error = errChan + + // 发送控制消息 + select { + case p.controlChan <- msg: + // 等待响应 + select { + case <-done: + select { + case resp := <-response: + return resp, nil + case err := <-errChan: + return nil, err + default: + return nil, fmt.Errorf("no response from worker") + } + case <-time.After(5 * time.Second): + return nil, fmt.Errorf("control message timeout") + } + case <-time.After(2 * time.Second): + return nil, fmt.Errorf("control channel full or pool stopped") + } +} + +func (p *CompletePool) safeSendControlToWorker(workerID int32, msg ControlMessage) bool { + if worker, ok := p.workers.Load(workerID); ok { + workerInst := worker.(*CompleteWorker) + status := atomic.LoadInt32(&workerInst.status) + + // 检查工作者状态 + if status == int32(WorkerStatusStopped) { + return false + } + + // 尝试发送消息 + select { + case workerInst.controlChan <- msg: + return true + case <-time.After(1 * time.Second): + return false + case <-p.shutdownCtx.Done(): + return false + } + } + return false +} + +// Shutdown 关闭池 +func (p *CompletePool) Shutdown() { + p.gracefulShutdown(PoolControlMessage{ + Type: ControlMsgGraceful, + }) +} + +// GetStats 获取统计信息 +func (p *CompletePool) GetStats() map[string]interface{} { + stats := make(map[string]interface{}) + stats["totalWorkers"] = atomic.LoadInt32(&p.totalWorkers) + stats["runningWorkers"] = atomic.LoadInt32(&p.runningWorkers) + stats["idleWorkers"] = atomic.LoadInt32(&p.idleWorkers) + stats["pausedWorkers"] = atomic.LoadInt32(&p.pausedWorkers) + stats["stoppedWorkers"] = atomic.LoadInt32(&p.stoppedWorkers) + stats["drainingWorkers"] = atomic.LoadInt32(&p.drainingWorkers) + stats["totalTasks"] = atomic.LoadInt64(&p.totalTasks) + stats["successTasks"] = atomic.LoadInt64(&p.successTasks) + stats["failedTasks"] = atomic.LoadInt64(&p.failedTasks) + stats["cancelledTasks"] = atomic.LoadInt64(&p.cancelledTasks) + stats["queuedTasks"] = atomic.LoadInt32(&p.queuedTasks) + stats["activeTasks"] = atomic.LoadInt32(&p.activeTasks) + + // 收集内存使用 + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + stats["memoryUsageMB"] = float64(memStats.Alloc) / 1024 / 1024 + + // 计算运行时间 + uptimeSeconds := float64(time.Now().UnixNano()/1e6-p.startTime) / 1000.0 + stats["uptimeSeconds"] = uptimeSeconds + + // 计算吞吐量 + if uptimeSeconds > 0 { + stats["throughput"] = float64(atomic.LoadInt64(&p.totalTasks)) / uptimeSeconds + } else { + stats["throughput"] = 0.0 + } + + return stats +} + +// logInfo 记录信息日志 +func (p *CompletePool) logInfo(format string, args ...interface{}) { + logMessage(LogLevelInfo, "[Pool] %s", fmt.Sprintf(format, args...)) +} + +// ==================== 任务注册表 ==================== + +// TaskRegistry 任务注册表 +type TaskRegistry struct { + tasks sync.Map + cleanupMu sync.RWMutex +} + +// NewTaskRegistry 创建任务注册表 +func NewTaskRegistry() *TaskRegistry { + return &TaskRegistry{} +} + +// Register 注册任务 +func (tr *TaskRegistry) Register(taskID int64, task *AtomicTask) { + tr.tasks.Store(taskID, task) +} + +// Get 获取任务 +func (tr *TaskRegistry) Get(taskID int64) (*AtomicTask, bool) { + if task, ok := tr.tasks.Load(taskID); ok { + return task.(*AtomicTask), true + } + return nil, false +} + +// Remove 移除任务 +func (tr *TaskRegistry) Remove(taskID int64) { + tr.tasks.Delete(taskID) +} + +// Cleanup 清理过期任务 +func (tr *TaskRegistry) Cleanup(maxAge time.Duration) int { + tr.cleanupMu.Lock() + defer tr.cleanupMu.Unlock() + + now := time.Now().UnixNano() / 1e6 + removed := 0 + + tr.tasks.Range(func(key, value interface{}) bool { + task := value.(*AtomicTask) + taskAge := now - atomic.LoadInt64(&task.endTime) + + // 如果任务已完成且超过最大年龄，或者任务创建时间超过24小时，则删除 + if (atomic.LoadInt32(&task.status) >= TaskStatusCompleted && taskAge > maxAge.Milliseconds()) || + (now-task.submitTime > 24*60*60*1000) { + tr.tasks.Delete(key) + removed++ + } + return true + }) + + return removed +} + +// ==================== 全局管理器 ==================== + +// GlobalManager 全局管理器 +type GlobalManager struct { + pool *CompletePool + registry *TaskRegistry + + mu sync.RWMutex + startTime int64 + cleanupTicker *time.Ticker + cleanupDone chan bool +} + +var ( + globalManager *GlobalManager + taskIDCounter int64 + initialized int32 + shuttingDown int32 +) + +// InitGlobalManager 初始化全局管理器 +func InitGlobalManager(config *PoolConfig) error { + if atomic.LoadInt32(&shuttingDown) == 1 { + return fmt.Errorf("system shutting down") + } + + manager := &GlobalManager{ + registry: NewTaskRegistry(), + startTime: time.Now().UnixNano() / 1e6, + cleanupDone: make(chan bool), + } + + pool := NewCompletePool(config) + manager.pool = pool + + // 启动任务清理器 + manager.startCleanup() + + // 原子交换 + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&globalManager)), + unsafe.Pointer(manager)) + + atomic.StoreInt32(&initialized, 1) + + logMessage(LogLevelInfo, "Global manager initialized") + return nil +} + +// startCleanup 启动任务清理器 +func (gm *GlobalManager) startCleanup() { + gm.cleanupTicker = time.NewTicker(30 * time.Minute) // 每30分钟清理一次 + + go func() { + for { + select { + case <-gm.cleanupTicker.C: + removed := gm.registry.Cleanup(1 * time.Hour) // 清理超过1小时的已完成任务 + if removed > 0 { + logMessage(LogLevelInfo, "Task cleanup: removed %d expired tasks", removed) + } + case <-gm.cleanupDone: + return + } + } + }() +} + +// stopCleanup 停止任务清理器 +func (gm *GlobalManager) stopCleanup() { + if gm.cleanupTicker != nil { + gm.cleanupTicker.Stop() + } + close(gm.cleanupDone) +} + +// getFunction 获取函数（线程安全） +func (gm *GlobalManager) getFunction(name string) TaskFunc { + if gm == nil || gm.pool == nil { + return nil + } + + gm.mu.RLock() + defer gm.mu.RUnlock() + + return gm.pool.GetFunction(name) +} + +// replaceWorker 替换工作者 +func (gm *GlobalManager) replaceWorker(workerID int32, worker *CompleteWorker) { + if gm == nil || gm.pool == nil { + return + } + + gm.mu.Lock() + defer gm.mu.Unlock() + + gm.pool.workers.Store(workerID, worker) +} + +// removeWorker 移除工作者 +func (gm *GlobalManager) removeWorker(workerID int32) { + if gm == nil || gm.pool == nil { + return + } + + gm.mu.Lock() + defer gm.mu.Unlock() + + gm.pool.workers.Delete(workerID) + atomic.AddInt32(&gm.pool.workerCount, -1) +} + +// sendControlToWorker 发送控制消息给工作者 +func (gm *GlobalManager) sendControlToWorkerWithRetry(workerID int32, msgType int, data interface{}, maxRetries int) (interface{}, error) { + var lastErr error + + for attempt := 0; attempt <= maxRetries; attempt++ { + if attempt > 0 { + // 指数退避等待 + waitTime := time.Duration(200*attempt) * time.Millisecond + logMessage(LogLevelInfo, "Retrying control message in %v (attempt %d/%d)", + waitTime, attempt, maxRetries+1) + time.Sleep(waitTime) + } + + // 检查工作者是否存在 + worker, ok := gm.pool.workers.Load(workerID) + if !ok { + return nil, fmt.Errorf("worker not found") + } + + // 创建响应通道 + responseChan := make(chan interface{}, 1) + errorChan := make(chan error, 1) + doneChan := make(chan bool, 1) + + // 发送控制消息 + select { + case worker.(*CompleteWorker).controlChan <- ControlMessage{ + Type: msgType, + Data: data, + Response: responseChan, + Error: errorChan, + Done: doneChan, + }: + // 等待响应 + select { + case <-doneChan: + select { + case resp := <-responseChan: + return resp, nil + case err := <-errorChan: + lastErr = err + // 如果是致命错误，不重试 + if strings.Contains(err.Error(), "worker not found") || + strings.Contains(err.Error(), "worker is stopped") { + return nil, err + } + continue + } + case <-time.After(2 * time.Second): + lastErr = fmt.Errorf("response timeout") + continue + } + case <-time.After(1 * time.Second): + lastErr = fmt.Errorf("control channel busy") + continue + } + } + + return nil, fmt.Errorf("control message failed after %d retries: %v", maxRetries, lastErr) +} + +// ==================== 公共API ==================== + +// TaskFunc 任务函数类型 +type TaskFunc func(param string) (string, error) + +// PoolConfig 池配置 +type PoolConfig struct { + MinWorkers int `json:"minWorkers"` + MaxWorkers int `json:"maxWorkers"` + QueueSize int `json:"queueSize"` + TaskTimeoutMs int `json:"taskTimeoutMs"` + WorkerIdleTimeoutMs int `json:"workerIdleTimeoutMs"` + MemoryLimitMB int `json:"memoryLimitMB"` + ShutdownTimeoutMs int `json:"shutdownTimeoutMs"` + HealthCheckDelayMs int `json:"healthCheckDelayMs"` + MaxRetries int `json:"maxRetries"` + DefaultLanguage string `json:"defaultLanguage"` + EnableMetrics bool `json:"enableMetrics"` + EnableAutoScaling bool `json:"enableAutoScaling"` + PriorityQueue bool `json:"priorityQueue"` + Name string `json:"name"` +} + +// ControlMessage 控制消息（公共API使用） +type ControlMessage struct { + Type int + Data interface{} + Done chan bool + Error chan error + Response chan interface{} +} + +// Initialize 初始化 +func Initialize(minWorkers, maxWorkers int, configJSON string) error { + config := &PoolConfig{ + MinWorkers: minWorkers, // 最小工作线程数 - 池启动时的基础线程数量 + MaxWorkers: maxWorkers, // 最大工作线程数 - 池允许创建的最大线程数 + QueueSize: DefaultQueueSize, // 任务队列大小 - 当所有工作线程忙时，可排队的任务数量 + TaskTimeoutMs: DefaultTaskTimeout, // 任务超时时间(毫秒) - 单个任务执行的最大时间 + WorkerIdleTimeoutMs: DefaultIdleTimeout, // 工作线程空闲超时(毫秒) - 空闲线程被回收的时间 + MemoryLimitMB: DefaultMaxMemoryMB, // 内存限制(MB) - 池允许使用的最大内存 + ShutdownTimeoutMs: DefaultShutdownTimeout, // 关闭超时(毫秒) - 优雅关闭的最大等待时间 + HealthCheckDelayMs: DefaultHealthCheckDelay, // 健康检查延迟(毫秒) - 健康检查的时间间隔 + MaxRetries: DefaultMaxRetries, // 最大重试次数 - 任务失败时的重试次数 + DefaultLanguage: LanguageEN, // 默认语言 - 用于日志/错误消息 + EnableMetrics: true, // 启用指标收集 - 是否收集性能指标 + EnableAutoScaling: true, // 启用自动伸缩 - 是否根据负载自动调整线程数 + PriorityQueue: false, // 优先级队列 - 是否按优先级处理任务 + Name: "default", // 池名称 - 用于标识和监控 + } + + if configJSON != "" { + if err := json.Unmarshal([]byte(configJSON), config); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + } + + return InitGlobalManager(config) +} + +// SubmitTask 提交任务 +func SubmitTask(workerID int, funcName, param string, priority int) (int64, error) { + if atomic.LoadInt32(&initialized) == 0 { + return -1, fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return -1, fmt.Errorf("pool not initialized") + } + + // 检查函数是否注册 + if manager.pool.GetFunction(funcName) == nil { + return -1, fmt.Errorf("function not registered") + } + + // 如果指定了workerID，检查工作者状态 + if workerID > 0 { + if worker, ok := manager.pool.workers.Load(int32(workerID)); ok { + workerStatus := atomic.LoadInt32(&worker.(*CompleteWorker).status) + // 如果工作者暂停，不允许提交任务 + if workerStatus == int32(WorkerStatusPaused) { + return -1, fmt.Errorf("worker %d is paused", workerID) + } + } + } + + return manager.pool.SubmitTask(int32(workerID), funcName, param, int32(priority)) +} + +// RegisterFunction 注册函数 +func RegisterFunction(name string, fn TaskFunc) error { + if atomic.LoadInt32(&initialized) == 0 { + return fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return fmt.Errorf("pool not initialized") + } + + return manager.pool.RegisterFunction(name, fn) +} + +// PauseWorker 暂停工作者 +func PauseWorker(workerID int) error { + return sendControlToWorker(workerID, 1, nil) +} + +// ResumeWorker 恢复工作者 +func ResumeWorker(workerID int) error { + return sendControlToWorker(workerID, 2, nil) +} + +// StopWorker 停止工作者 +func StopWorker(workerID int) error { + if atomic.LoadInt32(&initialized) == 0 { + return fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return fmt.Errorf("pool not initialized") + } + + // 检查工作者是否存在 + worker, ok := manager.pool.workers.Load(int32(workerID)) + if !ok { + // 工作者不存在，视为成功停止 + return nil + } + + workerInst := worker.(*CompleteWorker) + + // 先检查工作者状态 + if workerInst.IsStopped() { + // 工作者已经停止，视为成功 + return nil + } + + // 使用简单的直接调用，不使用重试循环 + responseChan := make(chan interface{}, 1) + errorChan := make(chan error, 1) + doneChan := make(chan bool, 1) + + // 创建一个简单的控制消息 + msg := ControlMessage{ + Type: ControlMsgStop, + Response: responseChan, + Error: errorChan, + Done: doneChan, + } + + // 直接发送控制消息 + go func() { + defer func() { + if r := recover(); r != nil { + select { + case errorChan <- fmt.Errorf("panic in stop worker: %v", r): + default: + } + } + }() + + // 使用工作者的安全发送方法 + resp, err := workerInst.SendControlSafe(msg) + if err != nil { + select { + case errorChan <- err: + default: + } + } else if responseChan != nil { + responseChan <- resp + } + }() + + // 等待响应，超时设为3秒 + timeout := time.After(3 * time.Second) + + select { + case <-doneChan: + // 成功停止 + return nil + case err := <-errorChan: + // 如果是"worker is stopped"或"worker not found"，视为成功 + if strings.Contains(strings.ToLower(err.Error()), "worker is stopped") || + strings.Contains(strings.ToLower(err.Error()), "worker not found") { + return nil + } + return err + case <-timeout: + // 超时，检查工作者是否已经停止 + if workerInst.IsStopped() { + return nil + } + return fmt.Errorf("stop worker timeout") + } +} + +// DrainWorker 排空工作者 +func DrainWorker(workerID int) error { + return sendControlToWorker(workerID, 4, nil) +} + +// RestartWorker 重启工作者 +func RestartWorker(workerID int) error { + return sendControlToWorker(workerID, 5, nil) +} + +// DeleteWorker 删除工作者 +func DeleteWorker(workerID int) error { + return sendControlToWorker(workerID, 7, nil) +} + +// GetWorkerStatus 获取工作者状态 +func GetWorkerStatus(workerID int) (map[string]interface{}, error) { + if atomic.LoadInt32(&initialized) == 0 { + return nil, fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return nil, fmt.Errorf("pool not initialized") + } + + // 直接访问工作者状态 + if worker, ok := manager.pool.workers.Load(int32(workerID)); ok { + return worker.(*CompleteWorker).getHealthStatus(), nil + } + + return nil, fmt.Errorf("worker not found") +} + +// CreateWorker 创建工作者 +func CreateWorker() (int, error) { + resp, err := sendControlToPool(9, nil) + if err != nil { + return -1, err + } + + if data, ok := resp.(map[string]interface{}); ok { + if workerID, ok := data["workerID"].(int32); ok { + return int(workerID), nil + } + } + + return -1, fmt.Errorf("invalid response") +} + +// PauseAllWorkers 暂停所有工作者 +func PauseAllWorkers() (int, error) { + resp, err := sendControlToPool(10, nil) + if err != nil { + return 0, err + } + + if data, ok := resp.(map[string]interface{}); ok { + if workers, ok := data["workers"].(int32); ok { + return int(workers), nil + } + } + + return 0, fmt.Errorf("invalid response") +} + +// ResumeAllWorkers 恢复所有工作者 +func ResumeAllWorkers() (int, error) { + if atomic.LoadInt32(&initialized) == 0 { + return 0, fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return 0, fmt.Errorf("pool not initialized") + } + + // 收集所有暂停的工作者 + pausedWorkers := make([]int32, 0) + manager.pool.workers.Range(func(key, value interface{}) bool { + worker := value.(*CompleteWorker) + if atomic.LoadInt32(&worker.status) == int32(WorkerStatusPaused) { + pausedWorkers = append(pausedWorkers, worker.id) + } + return true + }) + + // 恢复每个暂停的工作者 + resumedCount := 0 + for _, workerID := range pausedWorkers { + if worker, ok := manager.pool.workers.Load(workerID); ok { + // 直接调用工作者的恢复方法，而不是通过控制通道 + worker.(*CompleteWorker).resume() + resumedCount++ + + // 等待一小段时间确保状态更新 + time.Sleep(10 * time.Millisecond) + } + } + + // 更新池统计 + if resumedCount > 0 { + atomic.StoreInt32(&manager.pool.runningWorkers, + atomic.LoadInt32(&manager.pool.runningWorkers)+int32(resumedCount)) + atomic.StoreInt32(&manager.pool.pausedWorkers, + atomic.LoadInt32(&manager.pool.pausedWorkers)-int32(resumedCount)) + } + + return resumedCount, nil +} + +// StopAllWorkers 停止所有工作者 +func StopAllWorkers() (int, error) { + resp, err := sendControlToPool(12, nil) + if err != nil { + return 0, err + } + + if data, ok := resp.(map[string]interface{}); ok { + if workers, ok := data["workers"].(int32); ok { + return int(workers), nil + } + } + + return 0, fmt.Errorf("invalid response") +} + +// GracefulShutdown 优雅关闭 +func GracefulShutdown() error { + if atomic.LoadInt32(&initialized) == 0 { + return fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return fmt.Errorf("pool not initialized") + } + + // 发送优雅关闭控制消息（同步等待） + _, err := sendControlToPool(13, nil) + if err != nil { + return err + } + + // 等待池完全关闭（最多30秒） + timeout := time.After(30 * time.Second) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-timeout: + return fmt.Errorf("graceful shutdown timeout after 30 seconds") + case <-ticker.C: + // 检查池是否已完全关闭 + if !IsInitialized() { + logMessage(LogLevelInfo, "Graceful shutdown completed successfully") + return nil + } + } + } + + return nil +} + +// UpdatePoolConfig 更新池配置 +func UpdatePoolConfig(configJSON string) error { + var config PoolConfig + if err := json.Unmarshal([]byte(configJSON), &config); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + + _, err := sendControlToPool(14, &config) + return err +} + +// GetPoolStats 获取池统计 +func GetPoolStats() (map[string]interface{}, error) { + if atomic.LoadInt32(&initialized) == 0 { + return nil, fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return nil, fmt.Errorf("pool not initialized") + } + + return manager.pool.GetStats(), nil +} + +// GetTaskInfo 获取任务信息 +func GetTaskInfo(taskID int64) (map[string]interface{}, error) { + if atomic.LoadInt32(&initialized) == 0 { + return nil, fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.registry == nil { + return nil, fmt.Errorf("pool not initialized") + } + + task, ok := manager.registry.Get(taskID) + if !ok { + return nil, fmt.Errorf("task not found") + } + + result := make(map[string]interface{}) + result["id"] = task.id + result["goroutineID"] = task.goroutineID + result["status"] = atomic.LoadInt32(&task.status) + + // 安全获取原子值 + if funcName := task.funcName.Load(); funcName != nil { + result["funcName"] = funcName + } else { + result["funcName"] = "" + } + + if param := task.param.Load(); param != nil { + result["param"] = param + } else { + result["param"] = "" + } + + if resultVal := task.result.Load(); resultVal != nil { + result["result"] = resultVal + } else { + result["result"] = "" + } + + if errorMsg := task.errorMsg.Load(); errorMsg != nil { + result["error"] = errorMsg + } else { + result["error"] = "" + } + + result["priority"] = atomic.LoadInt32(&task.priority) + result["submitTime"] = task.submitTime + result["queueTime"] = task.queueTime + result["dequeueTime"] = task.dequeueTime + result["startTime"] = atomic.LoadInt64(&task.startTime) + result["endTime"] = atomic.LoadInt64(&task.endTime) + result["waitDuration"] = atomic.LoadInt64(&task.waitDuration) + result["execDuration"] = atomic.LoadInt64(&task.execDuration) + result["totalDuration"] = atomic.LoadInt64(&task.totalDuration) + + return result, nil +} + +// CancelTask 取消任务 +func CancelTask(taskID int64) error { + if atomic.LoadInt32(&initialized) == 0 { + return fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.registry == nil { + return fmt.Errorf("pool not initialized") + } + + task, ok := manager.registry.Get(taskID) + if !ok { + return fmt.Errorf("task not found") + } + + // 取消任务 + if task.cancel != nil { + task.cancel() + } + + // 更新状态 + if atomic.CompareAndSwapInt32(&task.status, + int32(TaskStatusPending), int32(TaskStatusCancelled)) { + task.errorMsg.Store("Task cancelled") + // 更新池的取消统计 + if globalManager != nil && globalManager.pool != nil { + atomic.AddInt64(&globalManager.pool.cancelledTasks, 1) + } + return nil + } + + if atomic.CompareAndSwapInt32(&task.status, + int32(TaskStatusRunning), int32(TaskStatusCancelled)) { + task.errorMsg.Store("Task cancelled") + // 更新池的取消统计 + if globalManager != nil && globalManager.pool != nil { + atomic.AddInt64(&globalManager.pool.cancelledTasks, 1) + } + return nil + } + + return fmt.Errorf("task cannot be cancelled") +} + +// Shutdown 关闭池 +func Shutdown() { + if atomic.LoadInt32(&initialized) == 0 { + return + } + + manager := globalManager + if manager != nil { + if manager.pool != nil { + manager.pool.Shutdown() + } + manager.stopCleanup() + } + + // 清空全局变量 + globalManager = nil + atomic.StoreInt32(&initialized, 0) + + logMessage(LogLevelInfo, "Pool shutdown completed") +} + +// IsInitialized 检查是否初始化 +func IsInitialized() bool { + if atomic.LoadInt32(&initialized) == 0 { + return false + } + + // 额外的检查：确保池仍然有效 + manager := globalManager + if manager == nil || manager.pool == nil { + atomic.StoreInt32(&initialized, 0) // 自动修复状态 + return false + } + + // 检查池的状态 + poolStatus := atomic.LoadInt32(&manager.pool.status) + if poolStatus == int32(WorkerStatusStopped) { + // 池已停止，但全局状态还未更新 + atomic.StoreInt32(&initialized, 0) + return false + } + + return true +} + +// 辅助函数 +func sendControlToWorker(workerID, msgType int, data interface{}) error { + _, err := sendControlToWorkerWithResponse(workerID, msgType, data) + return err +} + +// sendControlToWorkerWithResponse 发送控制消息到工作者并获取响应 +func sendControlToWorkerWithResponse(workerID, msgType int, data interface{}) (interface{}, error) { + if atomic.LoadInt32(&initialized) == 0 { + return nil, fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return nil, fmt.Errorf("pool not initialized") + } + + // 直接发送，不重试 + resp, err := manager.pool.SendControl(PoolControlMessage{ + Type: msgType, + WorkerID: int32(workerID), + Data: data, + }) + + if err != nil { + // 如果是超时错误，可以重试一次 + if strings.Contains(err.Error(), "timeout") { + time.Sleep(100 * time.Millisecond) + return manager.pool.SendControl(PoolControlMessage{ + Type: msgType, + WorkerID: int32(workerID), + Data: data, + }) + } + } + + return resp, err +} + +// WaitForWorkerState 等待工作者达到指定状态 +func WaitForWorkerState(workerID int, targetState int, timeoutMs int) bool { + deadline := time.Now().Add(time.Duration(timeoutMs) * time.Millisecond) + + for time.Now().Before(deadline) { + status, err := GetWorkerStatus(workerID) + if err != nil { + // 如果工作者不存在，检查是否目标状态是停止 + if strings.Contains(err.Error(), "not found") && targetState == WorkerStatusStopped { + return true + } + time.Sleep(50 * time.Millisecond) + continue + } + + if s, ok := status["status"].(float64); ok && int(s) == targetState { + return true + } + + time.Sleep(50 * time.Millisecond) + } + + return false +} + +func sendControlToPool(msgType int, data interface{}) (interface{}, error) { + if atomic.LoadInt32(&initialized) == 0 { + return nil, fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return nil, fmt.Errorf("pool not initialized") + } + + return manager.pool.SendControl(PoolControlMessage{ + Type: msgType, + Data: data, + }) +} + +// IsWorkerAvailable 检查工作者是否可接受任务 +func IsWorkerAvailable(workerID int) (bool, error) { + if atomic.LoadInt32(&initialized) == 0 { + return false, fmt.Errorf("pool not initialized") + } + + manager := globalManager + if manager == nil || manager.pool == nil { + return false, fmt.Errorf("pool not initialized") + } + + if worker, ok := manager.pool.workers.Load(int32(workerID)); ok { + status := atomic.LoadInt32(&worker.(*CompleteWorker).status) + return status == int32(WorkerStatusRunning) || status == int32(WorkerStatusIdle), nil + } + + return false, fmt.Errorf("worker not found") +} + +// ==================== 辅助函数 ==================== + +func logMessage(level int, format string, args ...interface{}) { + if atomic.LoadInt32(&shuttingDown) == 1 { + return + } + + levelStr := "INFO" + switch level { + case LogLevelDebug: + levelStr = "DEBUG" + case LogLevelWarn: + levelStr = "WARN" + case LogLevelError: + levelStr = "ERROR" + case LogLevelFatal: + levelStr = "FATAL" + } + + timestamp := time.Now().Format("2006-01-02 15:04:05.000") + message := fmt.Sprintf(format, args...) + + fmt.Printf("[%s] [%s] %s\n", timestamp, levelStr, message) +} + +func roundUpToPowerOfTwo(n int32) int32 { + n-- + n |= n >> 1 + n |= n >> 2 + n |= n >> 4 + n |= n >> 8 + n |= n >> 16 + n++ + return n +} + +func init() { + atomic.StoreInt32(&initialized, 0) + atomic.StoreInt32(&shuttingDown, 0) + debug.SetGCPercent(200) + debug.SetMemoryLimit(int64(DefaultMaxMemoryMB) * 1024 * 1024) + + // 设置panic处理器 + defer func() { + if r := recover(); r != nil { + logMessage(LogLevelFatal, "Global panic: %v\nStack: %s", + r, string(debug.Stack())) + os.Exit(1) + } + }() + + logMessage(LogLevelInfo, "Goroutine pool module initialized") +} diff --git a/goroutine-pool/goroutine_pool_test.go b/goroutine-pool/goroutine_pool_test.go new file mode 100644 index 0000000..32517df --- /dev/null +++ b/goroutine-pool/goroutine_pool_test.go @@ -0,0 +1,949 @@ +package goroutine_pool + +import ( + "fmt" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// ==================== 测试辅助函数 ==================== + +// 测试任务函数1: 字符串处理 +func testStringTask(param string) (string, error) { + time.Sleep(10 * time.Millisecond) // 模拟处理时间 + return "Processed: " + param, nil +} + +// 测试任务函数2: 数值计算 +func testNumberTask(param string) (string, error) { + n, err := strconv.Atoi(param) + if err != nil { + return "", fmt.Errorf("invalid number: %s", param) + } + time.Sleep(5 * time.Millisecond) // 模拟计算时间 + return strconv.Itoa(n * n), nil +} + +// 测试任务函数3: 错误测试 +func testErrorTask(param string) (string, error) { + time.Sleep(3 * time.Millisecond) + return "", fmt.Errorf("simulated error for: %s", param) +} + +// 测试任务函数4: 长时任务 +func testLongTask(param string) (string, error) { + time.Sleep(100 * time.Millisecond) + return "Long task completed: " + param, nil +} + +// 测试任务函数5: 内存密集型 +func testMemoryIntensiveTask(param string) (string, error) { + // 分配一些内存 + data := make([]byte, 1024*1024) // 1MB + for i := range data { + data[i] = byte(i % 256) + } + time.Sleep(20 * time.Millisecond) + return fmt.Sprintf("Allocated %d bytes", len(data)), nil +} + +// ==================== 完整功能测试 ==================== + +// TestFullFunctionality 测试完整功能 +func TestFullFunctionality(t *testing.T) { + fmt.Println("========== 开始完整功能测试 ==========") + + // 1. 初始化池 + fmt.Println("\n1. 初始化协程池...") + err := Initialize(3, 10, `{ + "minWorkers": 3, + "maxWorkers": 10, + "queueSize": 50, + "taskTimeoutMs": 30000, + "workerIdleTimeoutMs": 60000, + "memoryLimitMB": 512, + "shutdownTimeoutMs": 10000, + "enableMetrics": true, + "enableAutoScaling": true, + "priorityQueue": true + }`) + if err != nil { + t.Fatalf("初始化失败: %v", err) + } + fmt.Println("✓ 池初始化成功") + + // 等待池完全启动 + time.Sleep(100 * time.Millisecond) + + // 2. 注册任务函数 + fmt.Println("\n2. 注册任务函数...") + RegisterFunction("string_task", testStringTask) + RegisterFunction("number_task", testNumberTask) + RegisterFunction("error_task", testErrorTask) + RegisterFunction("long_task", testLongTask) + RegisterFunction("memory_task", testMemoryIntensiveTask) + fmt.Println("✓ 任务函数注册完成") + + // 3. 基础任务提交测试 + fmt.Println("\n3. 基础任务提交测试...") + testBasicTasks(t) + + //// 4. 并发测试 + //fmt.Println("\n4. 并发任务测试...") + //testConcurrentTasks(t) + // + //// 5. 工作者控制测试 + //fmt.Println("\n5. 工作者控制测试...") + //testWorkerControl(t) + + //// 6. 池控制测试 + //fmt.Println("\n6. 池控制测试...") + //testPoolControl(t) + + //// 7. 优先级测试 + //fmt.Println("\n7. 优先级任务测试...") + //testPriorityTasks(t) + + //// 8. 错误处理测试 + //fmt.Println("\n8. 错误处理测试...") + //testErrorHandling(t) + + ////9. 内存和性能测试 + //fmt.Println("\n9. 内存和性能测试...") + //testMemoryAndPerformance(t) + + // 10. 优雅关闭测试 + fmt.Println("\n10. 优雅关闭测试...") + testGracefulShutdown(t) + // + //// 11. 重新初始化测试 + //fmt.Println("\n11. 重新初始化测试...") + //testReinitialization(t) + + fmt.Println("\n========== 所有测试完成 ==========") + + // 确保最终清理 + if IsInitialized() { + Shutdown() + } +} + +// testBasicTasks 测试基础任务功能 +func testBasicTasks(t *testing.T) { + // 提交单个任务 + taskID, err := SubmitTask(0, "string_task", "Hello World", PriorityNormal) + if err != nil { + t.Errorf("提交任务失败: %v", err) + } else { + fmt.Printf(" ✓ 任务提交成功: ID=%d\n", taskID) + } + + // 等待任务完成 + time.Sleep(50 * time.Millisecond) + + // 获取任务信息 + info, err := GetTaskInfo(taskID) + if err != nil { + t.Errorf("获取任务信息失败: %v", err) + } else { + fmt.Printf(" 任务信息: ID=%v, 状态=%v\n", info["id"], info["status"]) + } + + // 批量提交任务 + var wg sync.WaitGroup + for i := 1; i <= 5; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + taskID, err := SubmitTask(0, "number_task", strconv.Itoa(index), PriorityNormal) + if err != nil { + fmt.Printf(" 批量任务%d提交失败: %v\n", index, err) + } else { + fmt.Printf(" ✓ 批量任务%d提交成功: ID=%d\n", index, taskID) + } + }(i) + } + wg.Wait() + + // 等待所有任务完成 + time.Sleep(200 * time.Millisecond) + + // 获取统计信息 + stats, err := GetPoolStats() + if err != nil { + t.Errorf("获取统计信息失败: %v", err) + } else { + fmt.Printf(" 池统计: 总任务=%v, 成功=%v, 失败=%v, 排队=%v\n", + stats["totalTasks"], stats["successTasks"], stats["failedTasks"], stats["queuedTasks"]) + } +} + +// testConcurrentTasks 测试并发任务 +func testConcurrentTasks(t *testing.T) { + const concurrentCount = 50 + var completed int32 + var errors int32 + + var wg sync.WaitGroup + startTime := time.Now() + + // 提交大量并发任务 + for i := 0; i < concurrentCount; i++ { + wg.Add(1) + go func(taskNum int) { + defer wg.Done() + + taskID, err := SubmitTask(0, "number_task", strconv.Itoa(taskNum), PriorityNormal) + if err != nil { + atomic.AddInt32(&errors, 1) + fmt.Printf(" 并发任务%d提交失败: %v\n", taskNum, err) + return + } + + // 等待任务完成 + for retry := 0; retry < 10; retry++ { + info, err := GetTaskInfo(taskID) + if err == nil { + status := info["status"].(int32) + if status == TaskStatusCompleted || status == TaskStatusFailed { + atomic.AddInt32(&completed, 1) + break + } + } + time.Sleep(50 * time.Millisecond) + } + }(i) + } + + wg.Wait() + duration := time.Since(startTime) + + fmt.Printf(" ✓ 并发测试完成: 总数=%d, 完成=%d, 错误=%d, 耗时=%v\n", + concurrentCount, completed, errors, duration) + fmt.Printf(" 平均吞吐量: %.2f 任务/秒\n", float64(completed)/duration.Seconds()) +} + +// testWorkerControl 测试工作者控制 +func testWorkerControl(t *testing.T) { + fmt.Println(" 创建新工作者...") + workerID, err := CreateWorker() + if err != nil { + t.Errorf("创建工作者失败: %v", err) + return + } + fmt.Printf(" ✓ 工作者创建成功: ID=%d\n", workerID) + + // 等待工作者完全启动 + time.Sleep(500 * time.Millisecond) + + // 先获取状态 + status, err := GetWorkerStatus(workerID) + if err != nil { + t.Errorf("获取工作者状态失败: %v", err) + return + } + fmt.Printf(" ✓ 初始状态: %v\n", status) + + // 暂停工作者 + fmt.Println(" 暂停工作者...") + err = PauseWorker(workerID) + if err != nil { + // 如果是"worker not paused"错误，可能是因为状态已经是暂停 + if strings.Contains(err.Error(), "not paused") { + fmt.Println(" ⚠ 工作者可能已经暂停") + } else { + t.Errorf("暂停工作者失败: %v", err) + return + } + } else { + fmt.Println(" ✓ 工作者已暂停") + } + + // 再次检查状态 + time.Sleep(200 * time.Millisecond) + status, err = GetWorkerStatus(workerID) + if err != nil { + t.Errorf("获取工作者状态失败: %v", err) + return + } + fmt.Printf(" ✓ 暂停后状态: %v\n", status) + + // 恢复工作者 + fmt.Println(" 恢复工作者...") + err = ResumeWorker(workerID) + if err != nil { + // 如果是"already running"错误，可以接受 + if strings.Contains(err.Error(), "already running") { + fmt.Println(" ⚠ 工作者已经在运行状态") + } else { + t.Errorf("恢复工作者失败: %v", err) + return + } + } else { + fmt.Println(" ✓ 工作者已恢复") + } + + // 等待恢复完成 + time.Sleep(200 * time.Millisecond) + status, err = GetWorkerStatus(workerID) + if err != nil { + t.Errorf("获取工作者状态失败: %v", err) + return + } + fmt.Printf(" ✓ 恢复后状态: %v\n", status) + + // 停止工作者 + fmt.Println(" 停止工作者...") + err = StopWorker(workerID) + if err != nil { + // 检查是否为可接受的错误 + acceptableErrors := []string{ + "worker not found", + "worker is stopped", + "worker is already stopped", + "worker context cancelled", + "timeout", + "control channel", + "no response", + } + + isAcceptable := false + errStr := strings.ToLower(err.Error()) + for _, acceptable := range acceptableErrors { + if strings.Contains(errStr, acceptable) { + isAcceptable = true + break + } + } + + if !isAcceptable { + t.Errorf("停止工作者失败: %v", err) + return + } + fmt.Printf(" ⚠ 工作者停止（可能已成功）: %v\n", err) + } else { + fmt.Println(" ✓ 工作者已停止") + } + + // 等待停止完成 + time.Sleep(500 * time.Millisecond) + + // 验证工作者确实已停止 + status, err = GetWorkerStatus(workerID) + if err != nil { + // 如果工作者不存在了，也是正常情况 + if strings.Contains(err.Error(), "worker not found") { + fmt.Println(" ✓ 工作者已从池中移除") + } else { + t.Errorf("获取工作者状态失败: %v", err) + } + } else { + // 如果工作者还存在，检查状态 + // 注意：status 已经是 map[string]interface{} 类型，不需要类型断言 + if s, ok := status["status"].(float64); ok && int(s) == WorkerStatusStopped { + fmt.Println(" ✓ 工作者状态为已停止") + } else { + fmt.Printf(" ⚠ 工作者状态为: %v\n", status["status"]) + } + } +} + +// testPoolControl 测试池控制 +func testPoolControl(t *testing.T) { + // 暂停所有工作者 + fmt.Println(" 暂停所有工作者...") + count, err := PauseAllWorkers() + if err != nil { + t.Errorf("暂停所有工作者失败: %v", err) + } else { + fmt.Printf(" ✓ 已暂停 %d 个工作者\n", count) + } + + // 等待确保所有工作者都已暂停 + time.Sleep(100 * time.Millisecond) + + // 检查工作者状态 + for i := 1; i <= 3; i++ { + status, err := GetWorkerStatus(i) + if err == nil { + if s, ok := status["status"].(float64); ok && int(s) == WorkerStatusPaused { + fmt.Printf(" ✓ 工作者 %d 已暂停\n", i) + } else { + fmt.Printf(" ! 工作者 %d 状态异常: %v\n", i, status) + } + } + } + + // 提交任务应该失败 + taskID, err := SubmitTask(0, "string_task", "During Pause", PriorityNormal) + if err != nil { + fmt.Printf(" ✓ 提交任务失败（预期）: %v\n", err) + } else { + t.Errorf("暂停状态下任务不应提交成功，但得到了任务ID: %d", taskID) + } + + // 恢复所有工作者 + fmt.Println(" 恢复所有工作者...") + count, err = ResumeAllWorkers() + if err != nil { + t.Errorf("恢复所有工作者失败: %v", err) + } else { + fmt.Printf(" ✓ 已恢复 %d 个工作者\n", count) + } + + // 等待工作者完全恢复 + time.Sleep(100 * time.Millisecond) + + // 检查工作者是否恢复 + for i := 1; i <= 3; i++ { + status, err := GetWorkerStatus(i) + if err == nil { + if s, ok := status["status"].(float64); ok && + (int(s) == WorkerStatusRunning || int(s) == WorkerStatusIdle) { + fmt.Printf(" ✓ 工作者 %d 已恢复\n", i) + } else { + fmt.Printf(" ! 工作者 %d 未正确恢复，状态: %v\n", i, status) + } + } + } + + // 现在工作者恢复了，提交一个新任务 + taskID, err = SubmitTask(0, "string_task", "After Resume", PriorityNormal) + if err != nil { + t.Errorf("恢复后提交任务失败: %v", err) + } else { + fmt.Printf(" ✓ 恢复后任务已提交: ID=%d\n", taskID) + + // 等待任务执行 + time.Sleep(500 * time.Millisecond) + + info, err := GetTaskInfo(taskID) + if err != nil { + fmt.Printf(" ! 获取任务信息失败: %v\n", err) + } else if info != nil { + if status, ok := info["status"].(int32); ok { + if status == TaskStatusCompleted { + fmt.Println(" ✓ 任务已执行完成") + } else if status == TaskStatusRunning { + fmt.Println(" ! 任务仍在运行中") + } else if status == TaskStatusPending { + fmt.Println(" ! 任务仍在排队") + } else { + fmt.Printf(" ! 任务状态异常: %d\n", status) + } + } + } + } +} + +// testPriorityTasks 测试优先级任务 +func testPriorityTasks(t *testing.T) { + fmt.Println(" 提交不同优先级的任务...") + + // 提交低优先级任务 + for i := 1; i <= 3; i++ { + param := fmt.Sprintf("Low Priority %d", i) + taskID, err := SubmitTask(0, "string_task", param, PriorityLow) + if err != nil { + t.Errorf("低优先级任务%d提交失败: %v", i, err) + } else { + fmt.Printf(" ✓ 低优先级任务%d: ID=%d\n", i, taskID) + } + } + + // 提交高优先级任务 + for i := 1; i <= 2; i++ { + param := fmt.Sprintf("High Priority %d", i) + taskID, err := SubmitTask(0, "string_task", param, PriorityHigh) + if err != nil { + t.Errorf("高优先级任务%d提交失败: %v", i, err) + } else { + fmt.Printf(" ✓ 高优先级任务%d: ID=%d\n", i, taskID) + } + } + + // 提交紧急优先级任务 + urgentTaskID, err := SubmitTask(0, "string_task", "Urgent Task", PriorityUrgent) + if err != nil { + t.Errorf("紧急优先级任务提交失败: %v", err) + } else { + fmt.Printf(" ✓ 紧急优先级任务: ID=%d\n", urgentTaskID) + } + + time.Sleep(300 * time.Millisecond) + fmt.Println(" 优先级验证完成（需检查执行时间戳）") +} + +// testErrorHandling 测试错误处理 +func testErrorHandling(t *testing.T) { + // 提交会失败的任务 + taskID, err := SubmitTask(0, "error_task", "Test Error", PriorityNormal) + if err != nil { + t.Errorf("错误任务提交失败: %v", err) + } else { + fmt.Printf(" ✓ 错误任务已提交: ID=%d\n", taskID) + } + + time.Sleep(50 * time.Millisecond) + + // 检查错误任务状态 + info, err := GetTaskInfo(taskID) + if err != nil { + t.Errorf("获取错误任务信息失败: %v", err) + } else { + status := info["status"].(int32) + if status == TaskStatusFailed { + fmt.Println(" ✓ 任务失败处理正确") + } + } + + // 测试取消任务 + taskID, err = SubmitTask(0, "long_task", "To be cancelled", PriorityNormal) + if err != nil { + t.Errorf("可取消任务提交失败: %v", err) + } else { + fmt.Printf(" ✓ 可取消任务已提交: ID=%d\n", taskID) + } + + // 立即取消 + err = CancelTask(taskID) + if err != nil { + t.Errorf("取消任务失败: %v", err) + } else { + fmt.Println(" ✓ 任务取消请求已发送") + } + + time.Sleep(100 * time.Millisecond) + + // 检查取消状态 + info, err = GetTaskInfo(taskID) + if err == nil { + status := info["status"].(int32) + if status == TaskStatusCancelled { + fmt.Println(" ✓ 任务已成功取消") + } + } +} + +// testMemoryAndPerformance 测试内存和性能 +func testMemoryAndPerformance(t *testing.T) { + fmt.Println(" 开始内存密集型任务测试...") + + startTime := time.Now() + const memoryTasks = 20 + var wg sync.WaitGroup + + for i := 0; i < memoryTasks; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + taskID, err := SubmitTask(0, "memory_task", fmt.Sprintf("Memory Test %d", index), PriorityNormal) + if err != nil { + fmt.Printf(" 内存任务%d提交失败: %v\n", index, err) + } else { + // 等待完成 + for retry := 0; retry < 20; retry++ { + info, err := GetTaskInfo(taskID) + if err == nil && (info["status"].(int32) == TaskStatusCompleted || + info["status"].(int32) == TaskStatusFailed) { + break + } + time.Sleep(50 * time.Millisecond) + } + } + }(i) + } + + wg.Wait() + duration := time.Since(startTime) + + fmt.Printf(" ✓ 内存测试完成: 耗时=%v\n", duration) +} + +// testGracefulShutdown 测试优雅关闭 +func testGracefulShutdown(t *testing.T) { + // 提交一些任务 + fmt.Println(" 提交一些任务以测试优雅关闭...") + + taskIDs := make([]int64, 0, 5) + for i := 1; i <= 5; i++ { + taskID, err := SubmitTask(0, "string_task", fmt.Sprintf("Shutdown Test %d", i), PriorityNormal) + if err != nil { + t.Errorf("关闭测试任务%d提交失败: %v", i, err) + } else { + fmt.Printf(" ✓ 关闭测试任务%d: ID=%d\n", i, taskID) + taskIDs = append(taskIDs, taskID) + } + } + + // 等待一小会儿让任务开始执行 + time.Sleep(200 * time.Millisecond) + + // 检查任务状态 + runningTasks := 0 + for _, taskID := range taskIDs { + info, err := GetTaskInfo(taskID) + if err == nil && info != nil { + if status, ok := info["status"].(int32); ok { + if status == TaskStatusRunning || status == TaskStatusPending { + runningTasks++ + } + } + } + } + fmt.Printf(" 当前有 %d 个任务正在运行或排队\n", runningTasks) + + // 开始优雅关闭（同步等待） + fmt.Println(" 开始优雅关闭...") + startTime := time.Now() + err := GracefulShutdown() + shutdownDuration := time.Since(startTime) + + if err != nil { + t.Errorf("优雅关闭失败: %v", err) + fmt.Printf(" 优雅关闭失败: %v (耗时: %v)\n", err, shutdownDuration) + } else { + fmt.Printf(" ✓ 优雅关闭成功 (耗时: %v)\n", shutdownDuration) + } + + // 等待一小段时间确保状态更新 + time.Sleep(100 * time.Millisecond) + + // 检查池状态 + if !IsInitialized() { + fmt.Println(" ✓ 池已正确关闭") + + // 检查任务是否都执行完成 + completedCount := 0 + for _, taskID := range taskIDs { + info, err := GetTaskInfo(taskID) + if err == nil && info != nil { + if status, ok := info["status"].(int32); ok { + if status == TaskStatusCompleted || status == TaskStatusFailed { + completedCount++ + } + } + } + } + fmt.Printf(" ✓ %d/%d 个任务执行完成\n", completedCount, len(taskIDs)) + } else { + // 如果池未关闭，可能是还在处理中，等待更长时间 + fmt.Println(" ! 池仍在运行，等待额外时间...") + time.Sleep(1 * time.Second) + + if !IsInitialized() { + fmt.Println(" ✓ 池已正确关闭（等待后）") + } else { + t.Error("池未正确关闭") + // 强制关闭 + Shutdown() + } + } +} + +// testReinitialization 测试重新初始化 +func testReinitialization(t *testing.T) { + fmt.Println(" 测试重新初始化...") + + // 重新初始化 + err := Initialize(2, 5, `{ + "minWorkers": 2, + "maxWorkers": 5, + "queueSize": 20, + "name": "ReinitializedPool" + }`) + if err != nil { + t.Fatalf("重新初始化失败: %v", err) + } + fmt.Println(" ✓ 池重新初始化成功") + + // 重新注册函数 + RegisterFunction("string_task", testStringTask) + RegisterFunction("number_task", testNumberTask) + + // 测试新池功能 + taskID, err := SubmitTask(0, "string_task", "Reinitialization Test", PriorityNormal) + if err != nil { + t.Errorf("重新初始化后提交任务失败: %v", err) + } else { + fmt.Printf(" ✓ 重新初始化后任务提交成功: ID=%d\n", taskID) + } + + time.Sleep(100 * time.Millisecond) + + // 获取任务信息 + info, err := GetTaskInfo(taskID) + if err == nil && info["status"].(int32) == TaskStatusCompleted { + fmt.Println(" ✓ 重新初始化后任务执行成功") + } +} + +// ==================== 高级测试 ==================== + +// TestAdvancedFeatures2 测试高级功能（重命名避免冲突） +func TestAdvancedFeatures2(t *testing.T) { + fmt.Println("\n========== 开始高级功能测试 ==========") + + // 初始化 + err := Initialize(4, 8, `{ + "minWorkers": 4, + "maxWorkers": 8, + "queueSize": 100, + "priorityQueue": true + }`) + if err != nil { + t.Fatalf("初始化失败: %v", err) + } + + RegisterFunction("string_task", testStringTask) + RegisterFunction("number_task", testNumberTask) + + // 1. 动态扩缩容测试 + fmt.Println("\n1. 动态扩缩容测试...") + testAutoScaling2(t) + + // 2. 故障恢复测试 + fmt.Println("\n2. 故障恢复测试...") + testFaultRecovery2(t) + + // 3. 配置热更新测试 + fmt.Println("\n3. 配置热更新测试...") + testHotConfigUpdate2(t) + + fmt.Println("\n========== 高级测试完成 ==========") + + // 确保清理 + if IsInitialized() { + Shutdown() + } +} + +// testAutoScaling2 测试自动扩缩容 +func testAutoScaling2(t *testing.T) { + // 获取初始统计 + stats, err := GetPoolStats() + if err != nil { + t.Errorf("获取统计失败: %v", err) + return + } + initialWorkers := stats["totalWorkers"].(int32) + fmt.Printf(" 初始工作者数: %d\n", initialWorkers) + + // 提交大量任务触发扩容 + const burstTasks = 100 + var wg sync.WaitGroup + + fmt.Printf(" 提交 %d 个任务触发扩容...\n", burstTasks) + for i := 0; i < burstTasks; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + SubmitTask(0, "number_task", strconv.Itoa(index), PriorityNormal) + }(i) + } + + wg.Wait() + time.Sleep(500 * time.Millisecond) + + // 检查扩容 + stats, err = GetPoolStats() + if err != nil { + t.Errorf("获取统计失败: %v", err) + return + } + finalWorkers := stats["totalWorkers"].(int32) + fmt.Printf(" 扩容后工作者数: %d (增加: %d)\n", finalWorkers, finalWorkers-initialWorkers) +} + +// testFaultRecovery2 测试故障恢复 +func testFaultRecovery2(t *testing.T) { + // 创建专门用于测试的工作者 + workerID, err := CreateWorker() + if err != nil { + t.Errorf("创建测试工作者失败: %v", err) + return + } + + fmt.Printf(" 创建测试工作者: ID=%d\n", workerID) + + // 注册一个会panic的任务函数 + panicFunc := func(param string) (string, error) { + panic("simulated panic in task") + } + RegisterFunction("panic_task", panicFunc) + + // 提交会panic的任务 + taskID, err := SubmitTask(workerID, "panic_task", "Trigger Panic", PriorityNormal) + if err != nil { + t.Errorf("提交panic任务失败: %v", err) + return + } + + fmt.Printf(" ✓ 提交panic任务: ID=%d\n", taskID) + + // 等待恢复 + time.Sleep(2 * time.Second) + + // 提交正常任务测试恢复后的功能 + taskID, err = SubmitTask(workerID, "string_task", "After Recovery", PriorityNormal) + if err == nil { + fmt.Printf(" ✓ 恢复后任务提交成功: ID=%d\n", taskID) + time.Sleep(100 * time.Millisecond) + } +} + +// testHotConfigUpdate2 测试配置热更新 +func testHotConfigUpdate2(t *testing.T) { + fmt.Println(" 更新池配置...") + + newConfig := `{ + "minWorkers": 6, + "maxWorkers": 12, + "queueSize": 200, + "taskTimeoutMs": 10000, + "workerIdleTimeoutMs": 60000, + "memoryLimitMB": 1024, + "name": "UpdatedPool" + }` + + err := UpdatePoolConfig(newConfig) + if err != nil { + t.Errorf("配置热更新失败: %v", err) + } else { + fmt.Println(" ✓ 配置热更新成功") + } +} + +// ==================== 集成测试示例 ==================== + +// TestExampleUsage 使用示例 +func TestExampleUsage(t *testing.T) { + fmt.Println("========== 协程池使用示例 ==========") + + // 1. 初始化 + err := Initialize(2, 5, "") + if err != nil { + t.Fatalf("初始化失败: %v", err) + } + defer Shutdown() + + // 2. 注册任务函数 + RegisterFunction("process_data", func(param string) (string, error) { + // 模拟数据处理 + time.Sleep(20 * time.Millisecond) + return "Processed: " + param, nil + }) + + // 3. 提交任务 + taskIDs := make([]int64, 0, 10) + for i := 0; i < 10; i++ { + taskID, err := SubmitTask(0, "process_data", + fmt.Sprintf("Data item %d", i), PriorityNormal) + if err == nil { + taskIDs = append(taskIDs, taskID) + } + } + + fmt.Printf("提交了 %d 个任务\n", len(taskIDs)) + + // 4. 等待任务完成 + time.Sleep(1 * time.Second) + + // 5. 获取统计 + stats, _ := GetPoolStats() + fmt.Printf("池统计: 总任务=%v, 成功=%v, 失败=%v\n", + stats["totalTasks"], stats["successTasks"], stats["failedTasks"]) + + fmt.Println("示例完成") +} + +// ==================== 主测试函数 ==================== + +// TestAll 运行所有测试 +func TestAll(t *testing.T) { + // 设置恢复 + defer func() { + if r := recover(); r != nil { + t.Errorf("测试发生panic: %v", r) + } + + // 最终清理 + if IsInitialized() { + Shutdown() + } + }() + + fmt.Println("开始运行协程池完整测试套件") + fmt.Println("======================================") + + // 运行基础功能测试 + TestFullFunctionality(t) + + // 运行高级功能测试 + TestAdvancedFeatures2(t) + + // 运行使用示例 + TestExampleUsage(t) + + fmt.Println("\n所有测试运行完成！") + fmt.Println("======================================") +} + +func TestDebugBasic(t *testing.T) { + fmt.Println("=== 调试测试开始 ===") + + // 1. 简单初始化 + err := Initialize(2, 2, `{ + "minWorkers": 2, + "maxWorkers": 2, + "queueSize": 10, + "taskTimeoutMs": 1000, + "workerIdleTimeoutMs": 1000, + "enableMetrics": false + }`) + if err != nil { + t.Fatalf("初始化失败: %v", err) + } + defer Shutdown() + + // 等待池启动 + time.Sleep(100 * time.Millisecond) + + // 2. 注册简单任务 + RegisterFunction("echo", func(param string) (string, error) { + fmt.Printf("任务执行: %s\n", param) + return "Echo: " + param, nil + }) + + // 3. 提交单个任务 + taskID, err := SubmitTask(0, "echo", "Test1", PriorityNormal) + if err != nil { + t.Fatalf("提交任务失败: %v", err) + } + fmt.Printf("任务提交成功: ID=%d\n", taskID) + + // 4. 等待执行 + time.Sleep(200 * time.Millisecond) + + // 5. 检查状态 + stats, err := GetPoolStats() + if err != nil { + t.Fatalf("获取统计失败: %v", err) + } + + fmt.Printf("池统计: %+v\n", stats) + + // 6. 获取任务信息 + info, err := GetTaskInfo(taskID) + if err != nil { + fmt.Printf("获取任务信息失败: %v\n", err) + } else { + fmt.Printf("任务信息: %+v\n", info) + } + + fmt.Println("=== 调试测试结束 ===") +} diff --git a/image/dll/image.dll b/image/dll/image.dll index d1331d6..171ae94 100644 Binary files a/image/dll/image.dll and b/image/dll/image.dll differ diff --git a/image/dll/image.h b/image/dll/image.h index f39459e..2bdd12d 100644 --- a/image/dll/image.h +++ b/image/dll/image.h @@ -88,7 +88,7 @@ extern "C" { // =================== C 导出函数 ======================= -// 处理图片成功 +// 检测图片纯白占比 // extern __declspec(dllexport) char* ProcessImage(char* jsonConfig); @@ -104,6 +104,22 @@ extern __declspec(dllexport) char* ResizeToHeightQuality(char* jsonConfig, int t // extern __declspec(dllexport) char* RemoveWhiteBorderAndPNG(char* jsonConfig); +// ResizeWTToHeightQuality 图片缩放 +// +extern __declspec(dllexport) char* ResizeWTToHeightQuality(char* jsonConfig, int dsWidth, int dsHeight); + +// CropImage 图片裁切 +// +extern __declspec(dllexport) char* CropImage(char* jsonConfig, int x, int y, int width, int height); + +// CreateChineseTextImage 创建带中文字体的文本图片，支持超出部分显示... +// +extern __declspec(dllexport) char* CreateChineseTextImage(char* text, int width, int height, char* fontSize, char* outputPath); + +// DrawChineseInfo 绘制书名，作者，出版社信息 +// +extern __declspec(dllexport) char* DrawChineseInfo(char* filePath, char* title, char* author, char* publisher, char* outputPath); + // 导出函数：释放C字符串内存 // extern __declspec(dllexport) void FreeCString(char* str); diff --git a/image/image.go b/image/image.go index 4b93ed3..5f0f3ed 100644 --- a/image/image.go +++ b/image/image.go @@ -5,20 +5,31 @@ import "C" import ( "encoding/json" "fmt" + "github.com/boombuler/barcode" + "github.com/boombuler/barcode/code128" + "github.com/boombuler/barcode/code39" + "github.com/boombuler/barcode/ean" "github.com/disintegration/imaging" + "github.com/fogleman/gg" + "github.com/golang/freetype" + "github.com/golang/freetype/truetype" "github.com/makiuchi-d/gozxing" "github.com/makiuchi-d/gozxing/qrcode" "github.com/nfnt/resize" "golang.org/x/image/draw" + "golang.org/x/image/math/fixed" "image" "image/color" "image/jpeg" _ "image/jpeg" "image/png" _ "image/png" + "io/ioutil" "math" "os" "path/filepath" + "runtime" + "strconv" "strings" "unsafe" ) @@ -706,20 +717,6 @@ func cropImage(config *Config, x, y, width, height int) (string, error) { return "", err } - //// 执行裁切 - //var cropped image.Image - //if width > 0 && height > 0 { - // cropped, err = SmartCrop(img, x, y, width, height int) - //} else { - // // 如果未指定尺寸，使用中心裁切 - // cropped, err = CropCenter(img, config.Width, config.Height) - //} - // - //if err != nil { - // return "", err - //} - // - // 保存图片到指定目录下 filename := filepath.Base(config.FileName) destPath := filepath.Join(config.OutputDir, config.CropDir, filename) @@ -766,77 +763,6 @@ func basicCrop(src image.Image, x, y, width, height int) (image.Image, error) { return cropped, nil } -//// smartCrop 智能裁切，处理边界和自动调整 -- 暂时没用 -//func smartCrop(src image.Image, x, y, width, height int) (image.Image, error) { -// srcBounds := src.Bounds() -// srcWidth := srcBounds.Dx() -// srcHeight := srcBounds.Dy() -// -// // 如果宽度或高度为0，使用图像的最大可能尺寸 -// if width == 0 { -// width = srcWidth - x -// } -// if height == 0 { -// height = srcHeight - y -// } -// -// // 调整裁切区域以确保在边界内 -// newX := max(0, min(x, srcWidth-1)) -// newY := max(0, min(x, srcHeight-1)) -// width = max(1, min(width, srcWidth-x)) -// height = max(1, min(height, srcHeight-y)) -// -// // 如果需要保持宽高比，调整裁切区域 -// if width > 0 && height > 0 { -// currentRatio := float64(width) / float64(height) -// originalRatio := float64(srcWidth) / float64(srcHeight) -// -// if math.Abs(currentRatio-originalRatio) > 0.01 { -// // 调整宽度以匹配原始宽高比 -// newWidth := int(float64(height) * originalRatio) -// if newWidth <= (srcWidth - newX) { -// width = newWidth -// } else { -// // 调整高度以匹配原始宽高比 -// newHeight := int(float64(width) / originalRatio) -// if newHeight <= (srcHeight - newY) { -// height = newHeight -// } -// } -// } -// } -// -// // 执行裁切 -// cropped, err := basicCrop(src, newX, newY, width, height) -// if err != nil { -// return nil, err -// } -// -// return cropped, nil -//} - -//// CropCenter 中心裁切 -- 暂时没用 -//func CropCenter(src image.Image, width, height int) (image.Image, error) { -// srcBounds := src.Bounds() -// srcWidth := srcBounds.Dx() -// srcHeight := srcBounds.Dy() -// -// // 如果裁切尺寸大于原图，返回原图或调整裁切尺寸 -// if width >= srcWidth && height >= srcHeight { -// return src, nil -// } -// -// // 计算中心裁切的起始点 -// x := (srcWidth - width) / 2 -// y := (srcHeight - height) / 2 -// -// // 确保不超出边界 -// x = max(0, min(x, srcWidth-width)) -// y = max(0, min(y, srcHeight-height)) -// -// return BasicCrop(src, x, y, width, height) -//} - // 识别二维码 func scanQRCode(fileName string) (bool, string, error) { file, err := os.Open(fileName) @@ -1107,6 +1033,652 @@ func generateQRCode(content string, width int, height int, fileName string) (str return fmt.Sprintf("生成二维码成功: %v", fileName), nil } +// 简单的自动换行实现（按字符数，适合中英文混合） +// 返回处理后的行列表和是否还有更多内容未显示 +func simpleAutoWrap(text string, maxCharsPerLine int) ([]string, bool) { + var lines []string + var currentLine strings.Builder + charCount := 0 + + for _, r := range text { + // 换行符处理 + if r == '\n' { + if currentLine.Len() > 0 { + lines = append(lines, currentLine.String()) + currentLine.Reset() + charCount = 0 + } + continue + } + + // 计算字符宽度（中文算2个字符，英文算1个） + charWidth := 1 + if r >= 0x4E00 && r <= 0x9FFF { // 中文范围 + charWidth = 2 + } else if r >= 0x3000 && r <= 0x303F { // 中文标点 + charWidth = 2 + } + + // 检查是否需要换行 + if charCount+charWidth > maxCharsPerLine && currentLine.Len() > 0 { + lines = append(lines, currentLine.String()) + currentLine.Reset() + charCount = 0 + } + + currentLine.WriteRune(r) + charCount += charWidth + } + + if currentLine.Len() > 0 { + lines = append(lines, currentLine.String()) + } + + return lines, false // 返回false表示所有内容都已处理 +} + +// 创建带中文字体的文本图片，支持超出部分显示... +// text 文本, width, height 宽度高度, fontSize 文字大小, outputPath 输入路径 +func createChineseTextImage(text string, width, height int, fontSize float64, outputPath string) (string, error) { + // 获取字体路径 + fontPath := getDefaultFontPath() + if fontPath == "" { + return "", fmt.Errorf("未找到系统字体文件，请手动指定字体路径") + } + + // 读取字体文件 + fontBytes, err := ioutil.ReadFile(fontPath) + if err != nil { + return "", fmt.Errorf("读取字体文件失败: %v", err) + } + + // 解析字体 + f, err := truetype.Parse(fontBytes) + if err != nil { + return "", fmt.Errorf("解析字体失败: %v", err) + } + + // 创建图片 + img := image.NewRGBA(image.Rect(0, 0, width, height)) + + // 白色背景 + white := color.RGBA{255, 255, 255, 255} + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + img.Set(x, y, white) + } + } + + // 创建freetype上下文 + c := freetype.NewContext() + c.SetDPI(72) + c.SetFont(f) + c.SetFontSize(fontSize) + c.SetClip(img.Bounds()) + c.SetDst(img) + c.SetSrc(image.NewUniform(color.Black)) + + // 计算可用的文本宽度（左右各留50像素边距） + availableWidth := width - 100 + + // 根据字体大小计算每行大约可以显示多少个字符 + charsPerLine := int(float64(availableWidth) / fontSize * 1.7) + if charsPerLine < 10 { + charsPerLine = 10 + } + + // 计算最大可显示行数 + // X坐标：从左侧20像素开始 + // Y坐标：从顶部20像素 + 字体高度 + lineSpacing := int(c.PointToFixed(fontSize*1.5) >> 6) + topMargin := 50 + int(c.PointToFixed(fontSize)>>6) + bottomMargin := 50 + maxLines := (height - topMargin - bottomMargin) / lineSpacing + if maxLines <= 0 { + maxLines = 1 + } + + // 使用简单的自动换行，获取所有行 + allLines, _ := simpleAutoWrap(text, charsPerLine) + + // 检查是否有超出图片的内容 + hasMore := len(allLines) > maxLines + displayLines := allLines + if hasMore { + // 只显示前 maxLines-1 行，最后一行添加 "..." + displayLines = allLines[:maxLines-1] + + // 获取最后一行文本，并确保能显示 "..." + lastLine := allLines[maxLines-1] + lastLineChars := 0 + var truncatedLine strings.Builder + + for _, r := range lastLine { + // 计算字符宽度 + charWidth := 1 + if r >= 0x4E00 && r <= 0x9FFF { + charWidth = 2 + } else if r >= 0x3000 && r <= 0x303F { + charWidth = 2 + } + + // 检查是否还能添加字符（留出3个字符给"..."） + if lastLineChars+charWidth > charsPerLine-3 { + break + } + + truncatedLine.WriteRune(r) + lastLineChars += charWidth + } + + // 添加 "..." + truncatedText := truncatedLine.String() + " ..." + displayLines = append(displayLines, truncatedText) + } else { + // 如果内容不多，直接显示所有行 + displayLines = allLines + if len(displayLines) > maxLines { + displayLines = displayLines[:maxLines] + } + } + + // 设置起始位置 + startX := 50 + startY := topMargin + + // 绘制每行文字 + pt := freetype.Pt(startX, startY) + for i, line := range displayLines { + // 确保不会超出图片底部 + if i*lineSpacing > height-bottomMargin { + break + } + + // 绘制文字 + _, err = c.DrawString(line, pt) + if err != nil { + return "", fmt.Errorf("绘制文字失败: %v", err) + } + + // 移动到下一行 + pt.Y += c.PointToFixed(fontSize * 1.5) + } + + // 保存图片 + file, err := os.Create(outputPath) + if err != nil { + return "", fmt.Errorf("创建文件失败: %v", err) + } + defer file.Close() + err = png.Encode(file, img) + if err != nil { + return "", fmt.Errorf("编码并写入失败: %v", err) + } + + return outputPath, nil +} + +// 获取系统字体路径 +func getDefaultFontPath() string { + switch runtime.GOOS { + case "windows": + paths := []string{ + "C:/Windows/Fonts/simhei.ttf", // 黑体 + "C:/Windows/Fonts/simsun.ttc", // 宋体 + "C:/Windows/Fonts/msyh.ttc", // 微软雅黑 + } + for _, path := range paths { + if _, err := os.Stat(path); err == nil { + return path + } + } + case "darwin": + return "/System/Library/Fonts/PingFang.ttc" + case "linux": + paths := []string{ + "/usr/share/fonts/truetype/wqy/wqy-microhei.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + } + for _, path := range paths { + if _, err := os.Stat(path); err == nil { + return path + } + } + } + return "" +} + +// 绘制中文文本（支持自动换行） +func drawChineseText(img *image.RGBA, title, author, publisher string) error { + // 获取字体路径 + fontPath := getDefaultFontPath() + if fontPath == "" { + return fmt.Errorf("未找到系统字体文件，请手动指定字体路径") + } + + // 读取字体文件 + fontBytes, err := ioutil.ReadFile(fontPath) + if err != nil { + return fmt.Errorf("读取字体文件失败: %v", err) + } + + // 解析字体 + fontObj, err := truetype.Parse(fontBytes) + if err != nil { + return fmt.Errorf("解析字体失败: %v", err) + } + + // 创建freetype上下文 + c := freetype.NewContext() + c.SetDPI(72) + c.SetFont(fontObj) + c.SetSrc(image.NewUniform(color.Black)) + c.SetClip(img.Bounds()) + c.SetDst(img) + + // 定义绘制区域的宽度 + maxWidth := 400 // 可根据需要调整 + + // 1. 绘制书名（使用较大字体，支持多行） + c.SetFontSize(45) + titleLines := wrapText(title, fontObj, 45, maxWidth) + // 限制书名最多显示3行 + if len(titleLines) > 3 { + titleLines = titleLines[:3] + } + titleY := 250 // 起始Y坐标 + titleLineHeight := 60 // 行高 + + for i, line := range titleLines { + lineWidth := calculateStringWidth(line, fontObj, 45) + pt := freetype.Pt((800-int(lineWidth))/2+30, titleY+i*titleLineHeight) + if _, err := c.DrawString(line, pt); err != nil { + return fmt.Errorf("绘制书名失败: %v", err) + } + } + + // 2. 绘制作者（使用中等字体，支持多行） + c.SetFontSize(28) + authorLines := wrapText(author, fontObj, 28, maxWidth) + // 限制作者最多显示2行 + if len(authorLines) > 2 { + authorLines = authorLines[:2] + } + authorY := 420 // 起始Y坐标 + authorLineHeight := 40 // 行高 + + for i, line := range authorLines { + lineWidth := calculateStringWidth(line, fontObj, 28) + pt := freetype.Pt((800-int(lineWidth))/2+30, authorY+i*authorLineHeight) + if _, err := c.DrawString(line, pt); err != nil { + return fmt.Errorf("绘制作者失败: %v", err) + } + } + + // 3. 绘制出版社（使用中等字体，支持多行） + c.SetFontSize(18) + publisherLines := wrapText(publisher, fontObj, 18, maxWidth) + // 限制出版社最多显示2行 + if len(publisherLines) > 1 { + publisherLines = publisherLines[:1] + } + publisherY := 700 // 起始Y坐标 + publisherLineHeight := 30 // 行高 + + for i, line := range publisherLines { + lineWidth := calculateStringWidth(line, fontObj, 18) + pt := freetype.Pt((800-int(lineWidth))/2+30, publisherY+i*publisherLineHeight) + if _, err := c.DrawString(line, pt); err != nil { + return fmt.Errorf("绘制出版社失败: %v", err) + } + } + + // 4. 绘制右下角文字 + err = drawBottomRightText(img, fontObj) + if err != nil { + return fmt.Errorf("绘制右下角文字失败: %v", err) + } + + return nil +} + +// 绘制右下角文字 +func drawBottomRightText(img *image.RGBA, fontObj *truetype.Font) error { + // 设置文字内容 + line1 := "此为实例图片" + line2 := "联系客服获取实图" + + // 设置字体大小 + fontSize := 8.0 + + // 计算右边界距 + rightMargin := 10 // 距离右边界的像素 + bottomMargin := 10 // 距离底部的像素 + + // 计算行高 + lineHeight := int(fontSize * 1.5) + + // 创建freetype上下文 + c := freetype.NewContext() + c.SetDPI(72) + c.SetFont(fontObj) + c.SetClip(img.Bounds()) + c.SetDst(img) + + // 计算第二行（最后一行）的位置 + line2Width := calculateStringWidth(line2, fontObj, fontSize) + x2 := img.Bounds().Dx() - line2Width - rightMargin + y2 := img.Bounds().Dy() - bottomMargin + + // 计算第一行的位置 + line1Width := calculateStringWidth(line1, fontObj, fontSize) + x1 := img.Bounds().Dx() - line1Width - rightMargin + y1 := y2 - lineHeight + + // 设置字体大小 + c.SetFontSize(fontSize) + + // 方法1：多层绘制实现描边效果 + strokeRadius := 1.0 // 描边半径 + + // 绘制描边（灰色，8个方向） + strokeColor := color.RGBA{128, 128, 128, 150} // 灰色 + c.SetSrc(image.NewUniform(strokeColor)) + + // 为第二行绘制描边 + offsets := []struct{ dx, dy float64 }{ + {-strokeRadius, -strokeRadius}, {0, -strokeRadius}, {strokeRadius, -strokeRadius}, + {-strokeRadius, 0}, {strokeRadius, 0}, + {-strokeRadius, strokeRadius}, {0, strokeRadius}, {strokeRadius, strokeRadius}, + } + + for _, offset := range offsets { + pt2 := freetype.Pt(int(float64(x2)+offset.dx), int(float64(y2)+offset.dy)) + if _, err := c.DrawString(line2, pt2); err != nil { + return fmt.Errorf("绘制第二行描边失败: %v", err) + } + + pt1 := freetype.Pt(int(float64(x1)+offset.dx), int(float64(y1)+offset.dy)) + if _, err := c.DrawString(line1, pt1); err != nil { + return fmt.Errorf("绘制第一行描边失败: %v", err) + } + } + + // 绘制白色文字（覆盖在中间） + textColor := color.RGBA{255, 255, 255, 255} // 白色 + c.SetSrc(image.NewUniform(textColor)) + + // 绘制第二行文字 + pt2 := freetype.Pt(x2, y2) + if _, err := c.DrawString(line2, pt2); err != nil { + return fmt.Errorf("绘制第二行文字失败: %v", err) + } + + // 绘制第一行文字 + pt1 := freetype.Pt(x1, y1) + if _, err := c.DrawString(line1, pt1); err != nil { + return fmt.Errorf("绘制第一行文字失败: %v", err) + } + + return nil +} + +// 文本换行函数 +func wrapText(text string, fontObj *truetype.Font, fontSize float64, maxWidth int) []string { + var lines []string + var currentLine string + var currentWidth int + + for _, ch := range text { + char := string(ch) + charWidth := calculateStringWidth(char, fontObj, fontSize) + + // 如果当前字符是换行符，直接换行 + if char == "\n" { + if currentLine != "" { + lines = append(lines, currentLine) + } + currentLine = "" + currentWidth = 0 + continue + } + + // 如果加上当前字符会超出宽度，开始新行 + if currentWidth+charWidth > maxWidth && currentLine != "" { + lines = append(lines, currentLine) + currentLine = char + currentWidth = charWidth + } else { + currentLine += char + currentWidth += charWidth + } + } + + // 添加最后一行 + if currentLine != "" { + lines = append(lines, currentLine) + } + + return lines +} + +// 计算字符串宽度 +func calculateStringWidth(text string, fontObj *truetype.Font, fontSize float64) int { + width := 0 + for _, ch := range text { + idx := fontObj.Index(ch) + horizAdvance := fontObj.HMetric(fixed.Int26_6(fontSize), idx).AdvanceWidth + width += int(horizAdvance) + } + return width +} + +// 加载PNG图片 +func loadPNG(filename string) (*image.RGBA, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + defer file.Close() + + img, err := png.Decode(file) + if err != nil { + return nil, err + } + + // 转换为RGBA + rgba := image.NewRGBA(img.Bounds()) + for y := 0; y < img.Bounds().Dy(); y++ { + for x := 0; x < img.Bounds().Dx(); x++ { + rgba.Set(x, y, img.At(x, y)) + } + } + + return rgba, nil +} + +// 保存为PNG文件 +func savePNG(img image.Image, filename string) error { + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + return png.Encode(file, img) +} + +const ( + CODE128 = "code128" + EAN13 = "ean13" + CODE39 = "code39" +) + +// 根据类型生成条形码 +func generateBarcode(barcodeType, content, filename string) (string, error) { + switch barcodeType { + case CODE128: + return generateCode128(content, filename) + case EAN13: + return generateEAN13(content, filename) + case CODE39: + return generateCode39(content, filename) + } + return "", fmt.Errorf("条形码类型不存在: %s", barcodeType) +} + +// 生成Code128条形码 +func generateCode128(content, filename string) (string, error) { + // 创建条形码 + code, err := code128.Encode(content) + if err != nil { + return "", fmt.Errorf("创建条形码失败： %v", err) + } + + // 缩放条形码尺寸 + scaledCode, err := barcode.Scale(code, 250, 85) + if err != nil { + return "", fmt.Errorf("缩放条形码尺寸失败： %v", err) + } + + // 创建带文字的图像 + imgWidth := 250 + imgHeight := 110 // 条形码高度 + 文字区域高度 + dc := gg.NewContext(imgWidth, imgHeight) + + // 设置白色背景 + dc.SetColor(color.White) + dc.Clear() + + dc.DrawImage(scaledCode, 0, 10) + + fontPath := getDefaultFontPath() + if fontPath == "" { + return "", fmt.Errorf("未找到系统字体文件，请手动指定字体路径") + } + + // 设置文字属性 + if err := dc.LoadFontFace("/System/Library/Fonts/Helvetica.ttc", 14); err != nil { + // 如果字体文件不存在，使用默认字体 + dc.LoadFontFace("Arial.ttf", 14) + } + + dc.SetColor(color.Black) + + // 在条形码下方居中绘制文字0 + textY := float64(85 + 20) // 条形码高度100 + 20像素间距 + textWidth, _ := dc.MeasureString(content) + textX := (float64(imgWidth) - textWidth) / 2 + dc.DrawString(content, textX, textY) + + // 保存图像 + err = dc.SavePNG(filename) + if err != nil { + return "", fmt.Errorf("保存图像失败: %v", err) + } + return filename, nil +} + +// 生成EAN13条形码 +func generateEAN13(content, filename string) (string, error) { + eanCode, err := ean.Encode(content) + if err != nil { + return "", fmt.Errorf("创建条形码失败： %v", err) + } + + scaledEanCode, err := barcode.Scale(eanCode, 250, 85) + if err != nil { + return "", fmt.Errorf("缩放条形码尺寸失败： %v", err) + } + + // 创建带文字的图像 + imgWidth := 250 + imgHeight := 110 // 条形码高度 + 文字区域高度 + dc := gg.NewContext(imgWidth, imgHeight) + + // 设置白色背景 + dc.SetColor(color.White) + dc.Clear() + + dc.DrawImage(scaledEanCode, 0, 10) + + fontPath := getDefaultFontPath() + if fontPath == "" { + return "", fmt.Errorf("未找到系统字体文件，请手动指定字体路径") + } + + // 设置文字属性 + if err := dc.LoadFontFace("/System/Library/Fonts/Helvetica.ttc", 14); err != nil { + // 如果字体文件不存在，使用默认字体 + dc.LoadFontFace("Arial.ttf", 14) + } + + dc.SetColor(color.Black) + + // 在条形码下方居中绘制文字0 + textY := float64(85 + 20) // 条形码高度100 + 20像素间距 + textWidth, _ := dc.MeasureString(content) + textX := (float64(imgWidth) - textWidth) / 2 + dc.DrawString(content, textX, textY) + + // 保存图像 + err = dc.SavePNG(filename) + if err != nil { + return "", fmt.Errorf("保存图像失败: %v", err) + } + return filename, nil +} + +// 生成Code39 +func generateCode39(content, filename string) (string, error) { + code, err := code39.Encode(content, true, true) + if err != nil { + return "", fmt.Errorf("创建条形码失败： %v", err) + } + + scaled, err := barcode.Scale(code, 250, 85) + if err != nil { + return "", fmt.Errorf("缩放条形码尺寸失败： %v", err) + } + + // 创建带文字的图像 + imgWidth := 250 + imgHeight := 110 // 条形码高度 + 文字区域高度 + dc := gg.NewContext(imgWidth, imgHeight) + + // 设置白色背景 + dc.SetColor(color.White) + dc.Clear() + + dc.DrawImage(scaled, 0, 10) + + fontPath := getDefaultFontPath() + if fontPath == "" { + return "", fmt.Errorf("未找到系统字体文件，请手动指定字体路径") + } + + // 设置文字属性 + if err := dc.LoadFontFace("/System/Library/Fonts/Helvetica.ttc", 14); err != nil { + // 如果字体文件不存在，使用默认字体 + dc.LoadFontFace("Arial.ttf", 14) + } + + dc.SetColor(color.Black) + + // 在条形码下方居中绘制文字0 + textY := float64(85 + 20) // 条形码高度100 + 20像素间距 + textWidth, _ := dc.MeasureString(content) + textX := (float64(imgWidth) - textWidth) / 2 + dc.DrawString(content, textX, textY) + + // 保存图像 + err = dc.SavePNG(filename) + if err != nil { + return "", fmt.Errorf("保存图像失败: %v", err) + } + return filename, nil +} + // =================== 辅助函数 ======================= // 辅助函数 @@ -1207,21 +1779,81 @@ func ResizeWTToHeightQuality(jsonConfig *C.char, dsWidth, dsHeight C.int) *C.cha //export CropImage func CropImage(jsonConfig *C.char, x, y, width, height C.int) *C.char { configStr := C.GoString(jsonConfig) - xStr := int(x) - yStr := int(y) - widthStr := int(width) - heightStr := int(height) + xInt := int(x) + yInt := int(y) + widthInt := int(width) + heightInt := int(height) var config *Config if err := json.Unmarshal([]byte(configStr), &config); err != nil { return C.CString(fmt.Sprintf("解析 config 失败: %v", err)) } - fileName, err := cropImage(config, xStr, yStr, widthStr, heightStr) + fileName, err := cropImage(config, xInt, yInt, widthInt, heightInt) if err != nil { return C.CString(fmt.Sprintf("%v", err)) } return C.CString(fileName) } +// CreateChineseTextImage 创建带中文字体的文本图片，支持超出部分显示... +// +//export CreateChineseTextImage +func CreateChineseTextImage(text *C.char, width, height C.int, fontSize *C.char, outputPath *C.char) *C.char { + textStr := C.GoString(text) + widthInt := int(width) + heightInt := int(height) + fontSizeStr := C.GoString(fontSize) + float, err := strconv.ParseFloat(fontSizeStr, 64) + if err != nil { + return C.CString(fmt.Sprintf("转换float64类型失败: %v", err)) + } + outputPathStr := C.GoString(outputPath) + textImage, err := createChineseTextImage(textStr, widthInt, heightInt, float, outputPathStr) + if err != nil { + return C.CString(fmt.Sprintf("%v", err)) + } + return C.CString(textImage) +} + +// DrawChineseInfo 绘制书名，作者，出版社信息 +// +//export DrawChineseInfo +func DrawChineseInfo(filePath, title, author, publisher, outputPath *C.char) *C.char { + filePathStr := C.GoString(filePath) + titleStr := C.GoString(title) + authorStr := C.GoString(author) + publishertr := C.GoString(publisher) + outputPathStr := C.GoString(outputPath) + + img, err := loadPNG(filePathStr) + if err != nil { + return C.CString(fmt.Sprintf("%v", err)) + } + err = drawChineseText(img, titleStr, authorStr, publishertr) + if err != nil { + return C.CString(fmt.Sprintf("%v", err)) + } + err = savePNG(img, outputPathStr) + if err != nil { + return C.CString(fmt.Sprintf("%v", err)) + } + return C.CString(fmt.Sprintf("图片保存成功，路径: %s", outputPath)) +} + +// GenerateBarcode 根据类型生成条形码 +// +//export GenerateBarcode +func GenerateBarcode(barcodeType, content, filename *C.char) *C.char { + barcodeTypeStr := C.GoString(barcodeType) + contentStr := C.GoString(content) + filenameStr := C.GoString(filename) + + filename, err := generateBarcode(barcodeTypeStr, contentStr, filenameStr) + if err != nil { + return C.CString(fmt.Sprintf("%v", err)) + } + return C.CString(filename) +} + // 导出函数：释放C字符串内存 // //export FreeCString diff --git a/image/imageDllTest.go b/image/imageDllTest.go index e6384de..30af96d 100644 --- a/image/imageDllTest.go +++ b/image/imageDllTest.go @@ -79,67 +79,67 @@ func cStr(ptr uintptr) string { return string(b) } -func main() { - - //config := &Config{ - // OutputDir: "D:\\isbn_images\\result", // 输出根目录 - // FileName: "D:\\isbn_images\\result\\97800079351851.jpg", - // //MatchDir: "matched", // 满足条件的图片目录 - // //UnmatchDir: "unmatched", // 不满足条件的图片目录 - // //WhiteDir: "white", - // //EqualHeightDir: "equalHeight", - // //WhiteHeightZoomDir: "whiteHeightZoom", - // CropDir: "crop", - // MinWhitePct: 0.1, // 纯白占比下限 10% - // MaxWhitePct: 0.65, // 纯白占比上限 90% - // Extensions: []string{"jpg", "jpeg", "png", "gif", "bmp", "webp"}, - //} - - //dll, err := InitImageDll() - //if err != nil { - // fmt.Println(err) - //} - ////err = dll.ProcessImage(config) - ////if err != nil { - //// fmt.Println(err) - ////} - // - //image, err := dll.CreateWhiteBottomCenteredImage(config, 800, 800) - //if err != nil { - // fmt.Println(err) - //} - //fmt.Println(image) - - // 图片缩放 - //quality, err := resizeWTToHeightQuality(config, 800, 500) - //if err != nil { - // fmt.Println(err) - //} - //fmt.Println(quality) - - //// 图片裁切 - //image, err := cropImage(config, 100, 0, 300, 300) - //if err != nil { - // fmt.Println(err) - //} - //fmt.Println(image) - - //file := "D:\\isbn_images\\result\\123\\qrcode.jpg" - // - //code, s, err := scanQRCodeNew(file) - //if err != nil { - // fmt.Println(err) - //} - //if code { - // fmt.Println(s) - //} - - code, err := generateQRCode("你好", 500, 500, "D:\\isbn_images\\result\\123\\qrcode.jpg") - if err != nil { - fmt.Println(err) - } - fmt.Println(code) -} +//func main() { +// +// //config := &Config{ +// // OutputDir: "D:\\isbn_images\\result", // 输出根目录 +// // FileName: "D:\\isbn_images\\result\\97800079351851.jpg", +// // //MatchDir: "matched", // 满足条件的图片目录 +// // //UnmatchDir: "unmatched", // 不满足条件的图片目录 +// // //WhiteDir: "white", +// // //EqualHeightDir: "equalHeight", +// // //WhiteHeightZoomDir: "whiteHeightZoom", +// // CropDir: "crop", +// // MinWhitePct: 0.1, // 纯白占比下限 10% +// // MaxWhitePct: 0.65, // 纯白占比上限 90% +// // Extensions: []string{"jpg", "jpeg", "png", "gif", "bmp", "webp"}, +// //} +// +// //dll, err := InitImageDll() +// //if err != nil { +// // fmt.Println(err) +// //} +// ////err = dll.ProcessImage(config) +// ////if err != nil { +// //// fmt.Println(err) +// ////} +// // +// //image, err := dll.CreateWhiteBottomCenteredImage(config, 800, 800) +// //if err != nil { +// // fmt.Println(err) +// //} +// //fmt.Println(image) +// +// // 图片缩放 +// //quality, err := resizeWTToHeightQuality(config, 800, 500) +// //if err != nil { +// // fmt.Println(err) +// //} +// //fmt.Println(quality) +// +// //// 图片裁切 +// //image, err := cropImage(config, 100, 0, 300, 300) +// //if err != nil { +// // fmt.Println(err) +// //} +// //fmt.Println(image) +// +// //file := "D:\\isbn_images\\result\\123\\qrcode.jpg" +// // +// //code, s, err := scanQRCodeNew(file) +// //if err != nil { +// // fmt.Println(err) +// //} +// //if code { +// // fmt.Println(s) +// //} +// +// code, err := generateQRCode("你好", 500, 500, "D:\\isbn_images\\result\\123\\qrcode.jpg") +// if err != nil { +// fmt.Println(err) +// } +// fmt.Println(code) +//} // 生成二维码 func generateQRCode1() { diff --git a/image/imageTest.go b/image/imageTest.go index 77f1418..d6aaf44 100644 --- a/image/imageTest.go +++ b/image/imageTest.go @@ -1,41 +1,39 @@ package main //func main() { -// // ==================== 在这里设置你的参数 ==================== -// config := &Config{ -// OutputDir: "D:\\isbn_images\\result", // 输出根目录 -// FileName: "D:\\isbn_images\\result\\9771671688095.jpg", -// MatchDir: "matched", // 满足条件的图片目录 -// UnmatchDir: "unmatched", // 不满足条件的图片目录 -// EqualHeightDir: "equalHeight", -// WhiteDir: "white", -// WhiteBorderPngDir: "whiteBorderPng", -// MinWhitePct: 0.1, // 纯白占比下限 10% -// MaxWhitePct: 0.65, // 纯白占比上限 90% -// Extensions: []string{"jpg", "jpeg", "png", "gif", "bmp", "webp"}, -// } -// -// //err := processImage(config) +// //// 测试长文本 +// //longText := "这是一段非常长的中文文本，需要测试自动换行功能。我们希望当文本超过图片宽度时，能够自动换行到下一行显示。这样就不需要手动添加换行符了。这段文本包含了中英文混合的内容，比如这里有一些English words mixed with中文。同时，我们也要测试标点符号的处理，例如逗号、句号、感叹号！问号？以及各种括号（包括圆括号、方括号[]、花括号{}）等等。最后，我们还要测试一下当文本非常长，超过图片高度时的处理情况。" + +// // "这是一段非常长的中文文本，需要测试自动换行功能。我们希望当文本超过图片宽度时，能够自动换行到下一行显示。这样就不需要手动添加换行符了。这段文本包含了中英文混合的内容，比如这里有一些English words mixed with中文。同时，我们也要测试标点符号的处理，例如逗号、句号、感叹号！问号？以及各种括号（包括圆括号、方括号[]、花括号{}）等等。最后，我们还要测试一下当文本非常长，超过图片高度时的处理情况。" + +// // "这是一段非常长的中文文本，需要测试自动换行功能。我们希望当文本超过图片宽度时，能够自动换行到下一行显示。这样就不需要手动添加换行符了。这段文本包含了中英文混合的内容，比如这里有一些English words mixed with中文。同时，我们也要测试标点符号的处理，例如逗号、句号、感叹号！问号？以及各种括号（包括圆括号、方括号[]、花括号{}）等等。最后，我们还要测试一下当文本非常长，超过图片高度时的处理情况。" + +// // "这是一段非常长的中文文本，需要测试自动换行功能。我们希望当文本超过图片宽度时，能够自动换行到下一行显示。这样就不需要手动添加换行符了。这段文本包含了中英文混合的内容，比如这里有一些English words mixed with中文。同时，我们也要测试标点符号的处理，例如逗号、句号、感叹号！问号？以及各种括号（包括圆括号、方括号[]、花括号{}）等等。最后，我们还要测试一下当文本非常长，超过图片高度时的处理情况。" + +// // "这是一段非常长的中文文本，需要测试自动换行功能。我们希望当文本超过图片宽度时，能够自动换行到下一行显示。这样就不需要手动添加换行符了。这段文本包含了中英文混合的内容，比如这里有一些English words mixed with中文。同时，我们也要测试标点符号的处理，例如逗号、句号、感叹号！问号？以及各种括号（包括圆括号、方括号[]、花括号{}）等等。最后，我们还要测试一下当文本非常长，超过图片高度时的处理情况。" + +// // "这是一段非常长的中文文本，需要测试自动换行功能。我们希望当文本超过图片宽度时，能够自动换行到下一行显示。这样就不需要手动添加换行符了。这段文本包含了中英文混合的内容，比如这里有一些English words mixed with中文。同时，我们也要测试标点符号的处理，例如逗号、句号、感叹号！问号？以及各种括号（包括圆括号、方括号[]、花括号{}）等等。最后，我们还要测试一下当文本非常长，超过图片高度时的处理情况。" +// // +// //text := longText +// // +// //err := createChineseTextImage(text, 800, 600, 20, "chinese_output.png") // //if err != nil { -// // fmt.Println(err) +// // fmt.Printf("生成图片失败: %v\n", err) +// // fmt.Println("\n解决方案：") +// // fmt.Println("1. 下载中文字体（如思源黑体）") +// // fmt.Println("2. 修改代码中的字体路径") +// // fmt.Println("3. 将字体文件放在项目目录中") +// // return // //} // // -// //file, err := resizeToHeightQuality(config, 700) -// //if err != nil { -// // fmt.Println(err) -// //} -// //fmt.Println(file) -// //config.FileName = file -// //fmt.Println(config) -// //file1, err := createWhiteBottomCenteredImage(config, 800, 800) -// //if err != nil { -// // fmt.Println(err) -// //} -// //fmt.Println(file1) +// //fmt.Println("图片已生成: chinese_output.png") // -// png, err := removeWhiteBorderAndPNG(config) +// img, err := loadPNG("image/123.png") +// if err != nil { +// fmt.Println(err) +// } +// // 绘制文本信息 +// err = drawChineseText(img, "Go语言高级编程Go语言高级编程Go语言高级编程Go语言高级编程Go语言高级编程Go语言高级编程", "作者：王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明王小明", "出版社：人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社人民邮电出版社") +// if err != nil { +// fmt.Println(err) +// } +// err = savePNG(img, "image/book_cover.png") // if err != nil { // fmt.Println(err) // } -// fmt.Println(png) //} diff --git a/kongfz/dll/kongfz.dll b/kongfz/dll/kongfz.dll index 4ccdeea..2457433 100644 Binary files a/kongfz/dll/kongfz.dll and b/kongfz/dll/kongfz.dll differ diff --git a/kongfz/dll/kongfz.h b/kongfz/dll/kongfz.h index 0d69b37..6e14cda 100644 --- a/kongfz/dll/kongfz.h +++ b/kongfz/dll/kongfz.h @@ -108,6 +108,10 @@ extern __declspec(dllexport) char* OutGetGoodsListMsgFromSelfShop(char* token, c // extern __declspec(dllexport) char* OutAddGoods(char* token, char* proxy, char* formData); +// OutAddGoodsAndFile 整合添加商品和获取孔网图片 +// +extern __declspec(dllexport) char* OutAddGoodsAndFile(char* token, char* proxy, char* filePath, char* formData); + // OutDelGoodsFromSelfShop 删除商品-已登的店铺 // extern __declspec(dllexport) char* OutDelGoodsFromSelfShop(char* token, char* proxy, char* itemId); @@ -144,6 +148,22 @@ extern __declspec(dllexport) char* KongfzOrderDeliver(int appId, char* appSecret // extern __declspec(dllexport) char* KongfzOrderSynchronization(int appId, char* appSecret, char* accessToken, char* shippingComName, int orderId, char* shippingId, char* shippingCom, char* shipmentNum, char* userDefined, char* moreShipmentNum); +// KongfzImageUpload 上传图片接口 +// +extern __declspec(dllexport) char* KongfzImageUpload(int appId, char* appSecret, char* accessToken, char* filePath, char* savePath); + +// KongfzShopItemAdd 添加店铺商品 +// +extern __declspec(dllexport) char* KongfzShopItemAdd(int appId, char* appSecret, char* accessToken, char* shopItemAddJson); + +// KongfzOrderList 查询订单列表 +// +extern __declspec(dllexport) char* KongfzOrderList(int appId, char* appSecret, char* accessToken, char* orderListJson); + +// KongfzOrderGet 查询单个订单 +// +extern __declspec(dllexport) char* KongfzOrderGet(int appId, char* appSecret, char* accessToken, char* userType, int orderId); + // Initialize 初始化配置 // extern __declspec(dllexport) char* Initialize(char* configJSON); diff --git a/kongfz/kongfz.go b/kongfz/kongfz.go index bdde7ba..4693e6c 100644 --- a/kongfz/kongfz.go +++ b/kongfz/kongfz.go @@ -5,14 +5,19 @@ package main */ import "C" import ( + "bytes" "crypto/md5" "encoding/json" "fmt" "io" "log" "math/rand" + "mime/multipart" "net/http" "net/url" + "os" + "path" + "path/filepath" "regexp" "sort" "strconv" @@ -614,17 +619,42 @@ func outAddGoods(token, proxy, formData string) (map[string]interface{}, error) } // 新增商品的URL url := "https://seller.kongfz.com/pc/itemInfo/add" + + var repData map[string]interface{} + err := json.Unmarshal([]byte(formData), &repData) + if err != nil { + return nil, fmt.Errorf("JSON解析失败: %v", err) + } + + //// 创建 multipart 请求体 + //body := &bytes.Buffer{} + //writer := multipart.NewWriter(body) + // + //// 写入所有表单字段 + //for key, value := range repData { + // if err := writer.WriteField(key, value); err != nil { + // return nil, fmt.Errorf("写入表单字段失败: %w", err) + // } + //} + // + //// 关闭writer以完成multipart写入 + //if err := writer.Close(); err != nil { + // return nil, fmt.Errorf("关闭multipart写入器失败: %w", err) + //} + // 创建请求 request := gorequest.New() // 设置代理（如果有提供代理URL） if proxy != "" { request.Proxy(proxy) } - resp, body, errs := request.Post(url). + resp, respBody, errs := request.Post(url). Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). Set("Accept", "application/json, text/plain, */*"). - Send(formData). + //Set("Content-Type", writer.FormDataContentType()). + Type("multipart"). + Send(repData). Timeout(15 * time.Second). End() if errs != nil { @@ -656,16 +686,219 @@ func outAddGoods(token, proxy, formData string) (map[string]interface{}, error) } // 解析JSON响应 var data map[string]interface{} - if err := json.Unmarshal([]byte(body), &data); err != nil { + if err := json.Unmarshal([]byte(respBody), &data); err != nil { return nil, fmt.Errorf("解析JSON失败: %w", err) } // 检查响应状态 - if val, ok := data["status"]; ok && val == 1 { + if val, ok := data["status"]; ok && fmt.Sprint(val) == "1" { return data, nil } return nil, fmt.Errorf("API返回错误: %+v", data) } +/* + * 获取图片key + * param token[string] 孔网token + * param proxy[string] 代理服务器IP + * return 获取图片key响应信息结构体,错误信息 + */ +func outKfzimgKey(token, proxy string) (map[string]interface{}, error) { + // 判断登录token + if token == "" { + return nil, fmt.Errorf("请先登录获取Token") + } + // 新增商品的URL + url := fmt.Sprintf("https://seller.kongfz.com/kfzimg-key?type=%s", "book") + // 创建请求 + request := gorequest.New() + // 设置代理（如果有提供代理URL） + if proxy != "" { + request.Proxy(proxy) + } + resp, body, errs := request.Get(url). + Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). + Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). + Set("Accept", "application/json, text/plain, */*"). + Timeout(15 * time.Second). + End() + if errs != nil { + // 检查是否是代理相关错误 + var isProxyError bool + var errorDetails []string + for _, e := range errs { + errorStr := e.Error() + errorDetails = append(errorDetails, errorStr) + if strings.Contains(errorStr, "Proxy Authentication Required") || + strings.Contains(errorStr, "connectex: A connection attempt failed") || + strings.Contains(errorStr, "connectex: No connection could be made") || + strings.Contains(errorStr, "proxyconnect tcp") || + strings.Contains(errorStr, "timeout") || + strings.Contains(errorStr, "connection refused") { + isProxyError = true + } + } + log.Printf("[ERROR] 请求错误详情: %v", errorDetails) + if isProxyError { + // 处理代理失败 + return nil, fmt.Errorf("代理连接失败: %s", errs) + } + return nil, fmt.Errorf("查询请求失败: %v", errs) + } + //检查HTTP状态码 + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP错误: %s", resp.Status) + } + + // 去除括号 + jsonStr := strings.TrimPrefix(body, "(") + jsonStr = strings.TrimSuffix(jsonStr, ")") + // 解析JSON响应 + var data map[string]interface{} + if err := json.Unmarshal([]byte(jsonStr), &data); err != nil { + return nil, fmt.Errorf("解析JSON失败: %w", err) + } + // 检查响应状态 + if val, ok := data["status"]; ok && val == "success" { + return data, nil + } + return nil, fmt.Errorf("API返回错误: %+v", data) +} + +/* + * 获取孔网图片 + * param token[string] 孔网token + * param proxy[string] 代理服务器IP + * return 获取孔网图片响应信息结构体,错误信息 + */ +func outKfzimgUpload(token, proxy, filePath string) (string, error) { + url := fmt.Sprint("https://seller.kongfz.com/kfzimg-upload") + fileName := filepath.Base(filePath) + // 创建multipart/form-data请求体 + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + // 读取文件内容 + fileData, err := os.ReadFile(filePath) + if err != nil { + return "", fmt.Errorf("读取文件失败: %v", err) + } + // 添加文件字段 + part, err := writer.CreateFormFile("img", filepath.Base(filePath)) + if err != nil { + return "", fmt.Errorf("创建表单文件失败: %v", err) + } + _, err = part.Write(fileData) + if err != nil { + return "", fmt.Errorf("写入文件数据失败: %v", err) + } + _ = writer.WriteField("autorotated", "1") + _ = writer.WriteField("name", fileName) + _ = writer.Close() + + key, err := outKfzimgKey(token, proxy) + if err != nil { + return "", err + } + imageKey := key["imagekey"] + + // 在发送请求前添加 + log.Printf("发送请求到: %s", url) + log.Printf("Cookie: PHPSESSID=%s", token) + log.Printf("文件: %s", fileName) + log.Printf("Content-Type: %s", writer.FormDataContentType()) + // 发送POST请求 + request := gorequest.New() + // 设置代理（如果有提供代理URL） + if proxy != "" { + request.Proxy(proxy) + } + resp, respBody, errs := request.Post(url). + //Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). + Set("Cookie", fmt.Sprintf("PHPSESSID=%s; IMGKEY=%s", token, imageKey)). + Set("Content-Type", writer.FormDataContentType()). // 设置Content-Type + Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). + Set("Accept", "*/*"). + Set("Host", "seller.kongfz.com"). + Set("Connection", "keep-alive"). + Send(body.String()). // 发送请求体 + Timeout(30 * time.Second). + End() + + if errs != nil { + // 检查是否是代理相关错误 + var isProxyError bool + var errorDetails []string + for _, e := range errs { + errorStr := e.Error() + errorDetails = append(errorDetails, errorStr) + if strings.Contains(errorStr, "Proxy Authentication Required") || + strings.Contains(errorStr, "connectex: A connection attempt failed") || + strings.Contains(errorStr, "connectex: No connection could be made") || + strings.Contains(errorStr, "proxyconnect tcp") || + strings.Contains(errorStr, "timeout") || + strings.Contains(errorStr, "connection refused") { + isProxyError = true + } + } + log.Printf("[ERROR] 请求错误详情: %v", errorDetails) + if isProxyError { + // 处理代理失败 + return "", fmt.Errorf("代理连接失败: %s", errs) + } + return "", fmt.Errorf("查询请求失败: %v", errs) + } + // 在得到响应后添加 + log.Printf("响应状态码: %d", resp.StatusCode) + log.Printf("响应头: %v", resp.Header) + //检查HTTP状态码 + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP错误: %s", resp.Status) + } + // 正则表达式匹配路径 + re := regexp.MustCompile(`(sw/.*?\.jpg)`) + matches := re.FindStringSubmatch(respBody) + if len(matches) > 0 { + return matches[1], nil + } else { + return "", fmt.Errorf("未找到匹配的路径") + } +} + +/* + * 整合添加商品和获取孔网图片 + * param token[string] 孔网token + * param proxy[string] 代理服务器IP + * param filePath[string] 图片路径 + * param formData[string] 商品JSON字符串 + * return 添加商品响应信息结构体,错误信息 + */ +func outAddGoodsAndFile(token, proxy, filePath, formData string) (string, error) { + upload, err := outKfzimgUpload(token, proxy, filePath) + if err != nil { + return "", err + } + var data map[string]interface{} + err = json.Unmarshal([]byte(formData), &data) + if err != nil { + return "", fmt.Errorf("JSON解析失败: %v", err) + } + data["images[0][imgUrl]"] = upload + + dataStr, err := json.Marshal(data) + if err != nil { + return "", fmt.Errorf("JSON序列化失败: %v", err) + } + fmt.Println(string(dataStr)) + goods, err := outAddGoods(token, proxy, string(dataStr)) + if err != nil { + return "", err + } + goodsStr, err := json.Marshal(goods) + if err != nil { + return "", fmt.Errorf("goodsStr JSON序列化失败: %v", err) + } + return string(goodsStr), nil +} + /* * 删除商品-已登的店铺 * param token[string] 孔网token @@ -952,7 +1185,21 @@ func outGetImageFilterShopId(token string, proxy string, isbn string, shopId int return nil, nil } -// 获取孔网商品图片和信息(官图和拍图) +/* + * 获取孔网商品图片和信息(官图和拍图) + * param token[string] 孔网token + * param proxy[string] 代理服务器IP + * param isbn[string] isbn + * param isLiveImage[int] 是否官图 0官图 1实拍图 + * param isReturnMsg[int] 是否有其他信息 + * return 孔网商品图片和信息响应信息结构体,错误信息 + * Error 请先登录获取Token + * Error 代理连接失败 + * Error 查询请求失败 + * Error 解析JSON失败 + * Error 错误信息: %w，状态码: %s + * Error 查询失败，没有数据！ + */ func outGetImageByIsbn(token string, proxy string, isbn string, isLiveImage int, isReturnMsg int) (*BookInfo, error) { // isLiveImage 1实拍图 0官图 ，isReturnMsg 0商品信息 bookInfo := &BookInfo{} @@ -1178,7 +1425,21 @@ func outGetImageByIsbn(token string, proxy string, isbn string, isLiveImage int, return nil, fmt.Errorf("查询失败，没有数据！") } -// 获取商品列表通过店铺ID +/* + * 获取商品列表通过店铺ID + * param shopId[int] 店铺ID + * param proxy[string] 代理服务器IP + * param retPrice[int] 是否需要价格 + * param sortType[string] 排序类型 + * param sort[string] 排序 + * param priceMin[float32] 价格区间-低 + * param priceMax[float32] 价格区间-高 + * param pageNum[int] 页数 + * param returnNum[int] 返回数量 + * return 商品列表响应信息结构体,商品总数,总页数,错误信息 + * Error 无效的排序类型: %s，可选值: sort, putDate, newItem, price + * Error 无效的排序类型: %s，可选值: desc, asc + */ func outGetGoodsListMsgByShopId(shopId int, proxy string, retPrice int, isImage int, sortType string, sort string, priceMin float32, priceMax float32, pageNum, returnNum int) (books []BookInfo, goodsNum string, pNum string, err error) { @@ -1323,7 +1584,14 @@ func outGetGoodsListMsgByShopId(shopId int, proxy string, retPrice int, isImage return books, goodsNum, pNum, nil } -// 获取商品信息通过商品详情链接 +/* + * 获取商品信息通过商品详情链接 + * param detailUrl[string] 详情页url + * param proxy[string] 代理服务器IP + * return 商品列表响应信息结构体,错误信息 + * Error 未找到指定的contentSpan信息 + * Error 无效的排序类型: %s，可选值: desc, asc + */ func outGetGoodsMsgByDetailUrl(detailUrl, proxy string) (*BookInfo, error) { // 发送请求获取响应 response, err := fetchResponse(detailUrl, proxy) @@ -1478,7 +1746,17 @@ func outGetGoodsMsgByDetailUrl(detailUrl, proxy string) (*BookInfo, error) { return &book, nil } -// 获取商品信息的快递费(定位到河南) +/* + * 获取商品信息的快递费(定位到河南) + * param url[string] 获取快递费url + * param proxy[string] 代理服务器IP + * return 商品列表响应信息结构体,错误信息 + * Error 无效的店铺编码: %s + * Error 无效的图书编码: %s + * Error 请求失败 + * Error HTTP错误 + * Error 解析JSON失败 + */ func getBookDetailShippingFee(url, proxy string) (string, error) { // 从URL中提取店铺ID和商品ID compile := regexp.MustCompile(`kongfz\.com/(\d+)/(\d+)`) @@ -1550,7 +1828,20 @@ func getBookDetailShippingFee(url, proxy string) (string, error) { return shippingFee, nil } -// 公用发送请求方法 +/* + * 公用发送请求方法 + * param url[string] url + * param proxy[string] 代理服务器IP + * return 商品列表响应信息结构体,错误信息 + * Error 请求失败 + * Error 响应为空 (尝试 %d/%d) + * Error HTTP状态码: %d,HTTP请求失败: %s (尝试 %d/%d) + * Error 代理认证失败 + * Error 请求超时，经过 %d 次尝试，超时网址:%s + * Error 网络连接错误，经过 %d 次尝试，错误网址:%s + * Error 查询请求失败，经过 %d 次尝试: %v，失败网址:%s + * Error HTTP错误 + */ func fetchResponse(url, proxy string) (*http.Response, error) { log.Printf("调用的URL: %s", url) maxRetries := 3 @@ -1652,7 +1943,16 @@ func fetchResponse(url, proxy string) (*http.Response, error) { return detailsResp, nil } -// 获取销量榜商品列表(带有Out的都非官放标准接口) +/* + * 获取销量榜商品列表 + * param catId[int] 分类ID + * param proxy[string] 代理服务器IP + * return isbn数组,错误信息 + * Error 请求失败 + * Error HTTP错误 + * Error 解析JSON失败 + * Error API返回错误: %s (代码: %d) + */ func outGetTopGoodsListMsg(catId int, proxy string) ([]string, error) { // 构建请求URL url := fmt.Sprintf("https://item.kongfz.com/api/pc/getSellWellListDetail?page=1&pageSize=100&timeRank=2&catId=%d", catId) @@ -1703,7 +2003,11 @@ func outGetTopGoodsListMsg(catId int, proxy string) ([]string, error) { return isbnList, nil } -// 去除重复的ISBN +/* + * 去除重复的ISBN + * param isbns[[]string] isbn数组 + * return isbn数组 + */ func removeDuplicateISBNs(isbns []string) []string { seen := make(map[string]bool) var result []string @@ -1716,7 +2020,12 @@ func removeDuplicateISBNs(isbns []string) []string { return result } -// generateSign 生成签名 +/* + * 生成签名 + * param params[map[string]interface{}] 生成签名的字段map + * param appSecret[string] app密钥 + * return 签名 + */ func generateSign(params map[string]interface{}, appSecret string) string { // 获取所有键并排序 keys := make([]string, 0, len(params)) @@ -1755,7 +2064,7 @@ func generateSign(params map[string]interface{}, appSecret string) string { data := appSecret + signString + appSecret hash := md5.Sum([]byte(data)) result := strings.ToUpper(fmt.Sprintf("%x", hash)) - fmt.Printf("Debug: MD5签名结果: %s\n", result) + //fmt.Printf("Debug: MD5签名结果: %s\n", result) return result } @@ -1770,6 +2079,7 @@ type KwAPIResponse struct { RequestMethod string `json:"requestMethod"` // 请求方法 } +// ErrorResponse 孔网API错误响应结构体 type ErrorResponse struct { Code int `json:"code"` // 错误码 Msg string `json:"msg"` // 错误信息 @@ -1790,7 +2100,13 @@ type Company struct { IsDefault bool `json:"isDefault"` // 是否默认 } -// 获取配送方式列表 +/* + * 获取配送方式列表 + * param appId[int] appkey + * param appSecret[string] app密钥 + * param accessToken[string] 访问token + * return 配送方式列表结构体字符串,错误信息 + */ func kongfzDeliveryMethodList(appId int, appSecret, accessToken string) (string, error) { kUrl := fmt.Sprint("https://open.kongfz.com/router/rest") dateTime := getCurrentTimeGMT8() @@ -1863,7 +2179,19 @@ func kongfzDeliveryMethodList(appId int, appSecret, accessToken string) (string, return string(responseJSON), nil } -// 订单发货 +/* + * 订单发货 + * param appId[int] appkey + * param appSecret[string] app密钥 + * param accessToken[string] 访问token + * param orderId[int] 订单编号 + * param shippingId[string] 配送方式 + * param shippingCom[string] 快递公司。当shippingId!=noLogistics时，此参数为必填。 + * param shipmentNum[string] 快递单号。当shippingId!=noLogistics时，此参数为必填。 + * param userDefined[string] 用户自定义物流公司。当shippingCom=other时，此参数为必填。 + * param moreShipmentNum[string] 填写更多的快递单号，以逗号分隔。 + * return 订单发货结构体字符串,错误信息 + */ func kongfzOrderDeliver(appId int, appSecret, accessToken string, orderId int, shippingId, shippingCom, shipmentNum, userDefined, moreShipmentNum string) (string, error) { kUrl := fmt.Sprint("https://open.kongfz.com/router/rest") @@ -1960,7 +2288,20 @@ func kongfzOrderDeliver(appId int, appSecret, accessToken string, return string(responseJSON), nil } -// 孔网订单同步 +/* + * 孔网订单同步 + * param appId[int] appkey + * param appSecret[string] app密钥 + * param accessToken[string] 访问token + * param shippingComName[string] 快递公司名称 + * param orderId[int] 订单编号 + * param shippingId[string] 配送方式 + * param shippingCom[string] 快递公司。当shippingId!=noLogistics时，此参数为必填。 + * param shipmentNum[string] 快递单号。当shippingId!=noLogistics时，此参数为必填。 + * param userDefined[string] 用户自定义物流公司。当shippingCom=other时，此参数为必填。 + * param moreShipmentNum[string] 填写更多的快递单号，以逗号分隔。 + * return 订单同步结构体字符串,错误信息 + */ func kongfzOrderSynchronization(appId int, appSecret, accessToken string, shippingComName string, orderId int, shippingId, shippingCom, shipmentNum, userDefined, moreShipmentNum string) (string, error) { // 获取配送方式列表 @@ -1986,6 +2327,7 @@ func kongfzOrderSynchronization(appId int, appSecret, accessToken string, shippi if shippingId == "" { shippingId = "express" shippingCom = "other" + userDefined = shippingComName } // 执行订单发货 orderDeliver, err := kongfzOrderDeliver(appId, appSecret, accessToken, orderId, shippingId, shippingCom, shipmentNum, userDefined, moreShipmentNum) @@ -1995,17 +2337,392 @@ func kongfzOrderSynchronization(appId int, appSecret, accessToken string, shippi return orderDeliver, err } -// =============== 辅助函数 ============== +/* + * 上传图片接口 + * param appId[int] appKey + * param appSecret[string] app密钥 + * param accessToken[string] 访问token + * param filePath[string] 图片url/本地图片路径 + * param savePath[string] 图片下载路径，如果是图片url需要下载到本地 + * return 上传图片响应结构体,错误信息 + * Error 解析图片URL失败 + * Error 读取文件失败 + * Error 创建表单文件失败 + * Error 写入文件数据失败 + */ +func kongfzImageUpload(appId int, appSecret, accessToken string, filePath, savePath string) (string, error) { + var image string + var needCleanup bool = false // 标记是否需要清理 + defer func() { + // 函数结束时删除临时图片 + if needCleanup && image != "" { + if err := os.Remove(image); err != nil { + log.Printf("警告：删除临时图片失败 %s: %v", image, err) + } else { + log.Printf("已清理临时图片: %s", image) + } + } + }() + if strings.HasPrefix(filePath, "http://") || strings.HasPrefix(filePath, "https://") { + // 解析URL + parse, err := url.Parse(filePath) + if err != nil { + return "", fmt.Errorf("解析图片URL失败: %v", err) + } + // 获取路径部分 + fPath := parse.Path + // 获取文件名 + fileName := path.Base(fPath) + savePath = filepath.Join(savePath, fileName) + image, err = downloadImage(filePath, savePath) + if err != nil { + return "", err + } + needCleanup = true // 标记需要清理 + } else { + // 本地文件 + image = filePath + // 本地文件不需要清理，由调用者管理 + } -// 获取GMT+8当前时间的字符串格式 + kUrl := fmt.Sprint("https://open.kongfz.com/router/image/upload") + dateTime := getCurrentTimeGMT8() + + // 生成签名参数 + params := map[string]interface{}{ + "method": "kongfz.image.upload", + "appId": appId, + "accessToken": accessToken, + "datetime": dateTime, + "format": "json", + "v": "1.0", + "signMethod": "md5", + "simplify": 0, + } + params["bucket"] = "book" + // 生成签名 + sign := generateSign(params, appSecret) + + // 创建multipart/form-data请求体 + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + // 读取文件内容 + fileData, err := os.ReadFile(image) + if err != nil { + return "", fmt.Errorf("读取文件失败: %v", err) + } + // 添加文件字段 + part, err := writer.CreateFormFile("image", filepath.Base(image)) + if err != nil { + return "", fmt.Errorf("创建表单文件失败: %v", err) + } + _, err = part.Write(fileData) + if err != nil { + return "", fmt.Errorf("写入文件数据失败: %v", err) + } + + // 添加其他参数到multipart表单 + for key, value := range params { + writer.WriteField(key, fmt.Sprintf("%v", value)) + } + writer.WriteField("sign", sign) // 添加签名 + writer.Close() // 关闭writer，完成表单构建 + + // 发送POST请求 + request := gorequest.New() + resp, respBody, errs := request.Post(kUrl). + Timeout(30*time.Second). + Set("Content-Type", writer.FormDataContentType()). // 设置Content-Type + Send(body.String()). // 发送请求体 + End() + if len(errs) > 0 { + return "", fmt.Errorf("请求失败: %v", errs) + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP状态码异常: %d, 响应: %s", resp.StatusCode, body) + } + + // 解析响应 + var response map[string]interface{} + if err := json.Unmarshal([]byte(respBody), &response); err != nil { + return "", fmt.Errorf("解析响应失败: %v, 响应内容: %s", err, body) + } + + // 转换成json字符串 + responseJSON, err := json.Marshal(response) + if err != nil { + return "", fmt.Errorf("JSON序列化失败: %v", err) + } + return string(responseJSON), nil +} + +/* + * 添加店铺商品 + * param appId[int] appKey + * param appSecret[string] app密钥 + * param accessToken[string] 访问token + * param shopItemAddJson[string] 店铺商品请求结构体字符串 + * return 添加店铺商品响应结构体,错误信息 + */ +func kongfzShopItemAdd(appId int, appSecret, accessToken string, shopItemAddJson string) (string, error) { + kUrl := fmt.Sprint("https://open.kongfz.com/router/rest") + dateTime := getCurrentTimeGMT8() + + // 生成签名参数 + params := map[string]interface{}{ + "method": "kongfz.shop.item.add", + "appId": appId, + "accessToken": accessToken, + "datetime": dateTime, + "format": "json", + "v": "1.0", + "signMethod": "md5", + "simplify": 0, + } + + if shopItemAddJson == "" { + return "", fmt.Errorf("shopItemAddJson 参数为空！") + } + toParams, err := addStructToParams(shopItemAddJson, params) + if err != nil { + return "", err + } + // 生成签名 + sign := generateSign(toParams, appSecret) + if sign == "" { + return "", fmt.Errorf("生成 sign 签名错误！") + } + params["sign"] = sign + + request := gorequest.New() + resp, body, errs := request.Get(kUrl). + Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). + Set("Accept", "application/json, text/plain, */*"). + Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). + Timeout(30 * time.Second). + Send(params). + End() + if len(errs) > 0 { + return "", fmt.Errorf("请求失败: %v", errs) + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP状态码异常: %d, 响应: %s", resp.StatusCode, body) + } + + // 解析响应 + var response map[string]interface{} + if err := json.Unmarshal([]byte(body), &response); err != nil { + return "", fmt.Errorf("解析响应失败: %v, 响应内容: %s", err, body) + } + + // 转换成json字符串 + responseJSON, err := json.Marshal(response) + if err != nil { + return "", fmt.Errorf("JSON序列化失败: %v", err) + } + return string(responseJSON), nil +} + +/* + * 查询订单列表 + * param appId[int] appKey + * param appSecret[string] app密钥 + * param accessToken[string] 访问token + * param orderListJson[string] 查询订单请求结构体字符串 + * return 添加店铺商品响应结构体,错误信息 + */ +func kongfzOrderList(appId int, appSecret, accessToken string, orderListJson string) (string, error) { + kUrl := fmt.Sprint("https://open.kongfz.com/router/rest") + dateTime := getCurrentTimeGMT8() + + // 生成签名参数 + params := map[string]interface{}{ + "method": "kongfz.order.list", + "appId": appId, + "accessToken": accessToken, + "datetime": dateTime, + "format": "json", + "v": "1.0", + "signMethod": "md5", + "simplify": 0, + } + if orderListJson == "" { + return "", fmt.Errorf("orderListJson 参数为空！") + } + toParams, err := addStructToParams(orderListJson, params) + if err != nil { + return "", err + } + // 生成签名 + sign := generateSign(toParams, appSecret) + if sign == "" { + return "", fmt.Errorf("生成 sign 签名错误！") + } + params["sign"] = sign + + request := gorequest.New() + resp, body, errs := request.Post(kUrl). + Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). + Set("Accept", "application/json, text/plain, */*"). + Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). + Set("Content-Type", "application/x-www-form-urlencoded"). + Timeout(30 * time.Second). + Send(params). + End() + if len(errs) > 0 { + return "", fmt.Errorf("请求失败: %v", errs) + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP状态码异常: %d, 响应: %s", resp.StatusCode, body) + } + + // 解析响应 + var response map[string]interface{} + if err := json.Unmarshal([]byte(body), &response); err != nil { + return "", fmt.Errorf("解析响应失败: %v, 响应内容: %s", err, body) + } + + // 转换成json字符串 + responseJSON, err := json.Marshal(response) + if err != nil { + return "", fmt.Errorf("JSON序列化失败: %v", err) + } + return string(responseJSON), nil +} + +/* + * 查询单个订单 + * param appId[int] appKey + * param appSecret[string] app密钥 + * param accessToken[string] 访问token + * param userType[string] 查询订单请求结构体字符串 + * param orderId[string] 查询订单请求结构体字符串 + * return 添加店铺商品响应结构体,错误信息 + */ +func kongfzOrderGet(appId int, appSecret, accessToken string, userType string, orderId int) (string, error) { + kUrl := fmt.Sprint("https://open.kongfz.com/router/rest") + dateTime := getCurrentTimeGMT8() + + // 生成签名参数 + params := map[string]interface{}{ + "method": "kongfz.order.get", + "appId": appId, + "accessToken": accessToken, + "datetime": dateTime, + "format": "json", + "v": "1.0", + "signMethod": "md5", + "simplify": 0, + } + params["userType"] = userType + params["orderId"] = strconv.Itoa(orderId) + // 生成签名 + sign := generateSign(params, appSecret) + if sign == "" { + return "", fmt.Errorf("生成 sign 签名错误！") + } + params["sign"] = sign + + request := gorequest.New() + resp, body, errs := request.Post(kUrl). + Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). + Set("Accept", "application/json, text/plain, */*"). + Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). + Set("Content-Type", "application/x-www-form-urlencoded"). + Timeout(30 * time.Second). + Send(params). + End() + if len(errs) > 0 { + return "", fmt.Errorf("请求失败: %v", errs) + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP状态码异常: %d, 响应: %s", resp.StatusCode, body) + } + + // 解析响应 + var response map[string]interface{} + if err := json.Unmarshal([]byte(body), &response); err != nil { + return "", fmt.Errorf("解析响应失败: %v, 响应内容: %s", err, body) + } + + // 转换成json字符串 + responseJSON, err := json.Marshal(response) + if err != nil { + return "", fmt.Errorf("JSON序列化失败: %v", err) + } + return string(responseJSON), nil +} + +// =============== 辅助函数 ============== +/* + * 将结构体的字段添加到 params 映射中 + * param req[string] 需要合并的json字符串 + * param params[map[string]interface{}] 合并的map + * return map响应结构体,错误信息 + */ +func addStructToParams(req string, params map[string]interface{}) (map[string]interface{}, error) { + // 将JSON字符串解析为map + var tempMap map[string]interface{} + if err := json.Unmarshal([]byte(req), &tempMap); err != nil { + return nil, fmt.Errorf("解析 req json失败：%v ", err) + } + // 合并到params + for k, v := range tempMap { + // 只添加非nil的值 + if v != nil { + params[k] = v + } + } + return params, nil +} + +/* + * 下载图片到本地 + * param filePath[string] 文件路径 + * param savePath[string] 需要保存的路径 + * return 本地图片路径,错误信息 + */ +func downloadImage(filePath, savePath string) (string, error) { + // 创建HTTP请求 + request := gorequest.New() + resp, body, err := request.Get(filePath). + Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"). + EndBytes() + if err != nil { + return "", fmt.Errorf("请求失败: %v", err) + } + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP 状态码错误: %d", resp.StatusCode) + } + errs := os.WriteFile(savePath, body, 0644) + if errs != nil { + return "", fmt.Errorf("保存文件失败: %v", errs) + } + return savePath, nil +} + +/* + * 获取GMT+8当前时间的字符串格式 + * return 返回时间字符串，格式：2006-01-02 15:04:05 + */ func getCurrentTimeGMT8() string { // 创建北京时间（GMT+8） - loc, _ := time.LoadLocation("Asia/Shanghai") + loc, err := time.LoadLocation("Asia/Shanghai") + if err != nil { + loc = time.FixedZone("GMT+8", 8*60*60) + } now := time.Now().In(loc) return now.Format("2006-01-02 15:04:05") } -// 替换所有空白字符为空格 +/* + * 替换所有空白字符为空格 + * return 返回字符串 + */ func cleanString(s string) string { s = strings.ReplaceAll(s, "\n", "") s = strings.ReplaceAll(s, "\r", "") @@ -2014,7 +2731,10 @@ func cleanString(s string) string { return removeDuplicates(s) } -// 字符串去重 +/* + * 字符串去重 + * return 返回字符串 + */ func removeDuplicates(s string) string { seen := make(map[rune]bool) var result strings.Builder @@ -2027,7 +2747,11 @@ func removeDuplicates(s string) string { return result.String() } -// 验证日期格式并转换为时间戳 +/* + * 验证日期格式并转换为时间戳 + * param dateStr[string] 时间字符串 + * return 时间戳 + */ func validateDateFormat(dateStr string) int64 { // 去除前后空格 dateStr = strings.TrimSpace(dateStr) @@ -2088,7 +2812,7 @@ func validateDateFormat(dateStr string) int64 { return parsedTime.Unix() } -// 初始化配置 +// 初始化配置--暂时没用 func initializeConfig(config Config) { // 设置全局配置 cf = config @@ -2283,6 +3007,21 @@ func OutAddGoods(token, proxy, formData *C.char) *C.char { return C.CString(string(jsonData)) } +// OutAddGoodsAndFile 整合添加商品和获取孔网图片 +// +//export OutAddGoodsAndFile +func OutAddGoodsAndFile(token, proxy, filePath, formData *C.char) *C.char { + goToken := C.GoString(token) + goProxy := C.GoString(proxy) + goFilePath := C.GoString(filePath) + goFormData := C.GoString(formData) + addGoodsAndFile, err := outAddGoodsAndFile(goToken, goProxy, goFilePath, goFormData) + if err != nil { + return C.CString(fmt.Sprint(err)) + } + return C.CString(addGoodsAndFile) +} + // OutDelGoodsFromSelfShop 删除商品-已登的店铺 // //export OutDelGoodsFromSelfShop @@ -2573,6 +3312,68 @@ func KongfzOrderSynchronization(appId C.int, appSecret, accessToken, shippingCom return C.CString(synchronization) } +// KongfzImageUpload 上传图片接口 +// +//export KongfzImageUpload +func KongfzImageUpload(appId C.int, appSecret, accessToken, filePath, savePath *C.char) *C.char { + goAppId := int(appId) + goAppSecret := C.GoString(appSecret) + goAccessToken := C.GoString(accessToken) + goFilePath := C.GoString(filePath) + goSavePath := C.GoString(savePath) + upload, err := kongfzImageUpload(goAppId, goAppSecret, goAccessToken, goFilePath, goSavePath) + if err != nil { + return C.CString(fmt.Sprint(err)) + } + return C.CString(upload) +} + +// KongfzShopItemAdd 添加店铺商品 +// +//export KongfzShopItemAdd +func KongfzShopItemAdd(appId C.int, appSecret, accessToken, shopItemAddJson *C.char) *C.char { + goAppId := int(appId) + goAppSecret := C.GoString(appSecret) + goAccessToken := C.GoString(accessToken) + goShopItemAddJson := C.GoString(shopItemAddJson) + upload, err := kongfzShopItemAdd(goAppId, goAppSecret, goAccessToken, goShopItemAddJson) + if err != nil { + return C.CString(fmt.Sprint(err)) + } + return C.CString(upload) +} + +// KongfzOrderList 查询订单列表 +// +//export KongfzOrderList +func KongfzOrderList(appId C.int, appSecret, accessToken *C.char, orderListJson *C.char) *C.char { + goAppId := int(appId) + goAppSecret := C.GoString(appSecret) + goAccessToken := C.GoString(accessToken) + goOrderListJson := C.GoString(orderListJson) + list, err := kongfzOrderList(goAppId, goAppSecret, goAccessToken, goOrderListJson) + if err != nil { + return C.CString(fmt.Sprint(err)) + } + return C.CString(list) +} + +// KongfzOrderGet 查询单个订单 +// +//export KongfzOrderGet +func KongfzOrderGet(appId C.int, appSecret, accessToken *C.char, userType *C.char, orderId C.int) *C.char { + goAppId := int(appId) + goAppSecret := C.GoString(appSecret) + goAccessToken := C.GoString(accessToken) + goUserType := C.GoString(userType) + goOrderId := int(orderId) + orderGet, err := kongfzOrderGet(goAppId, goAppSecret, goAccessToken, goUserType, goOrderId) + if err != nil { + return C.CString(fmt.Sprint(err)) + } + return C.CString(orderGet) +} + // Initialize 初始化配置 // //export Initialize diff --git a/kongfz/kongfzDll.go b/kongfz/kongfzDll.go new file mode 100644 index 0000000..0e085df --- /dev/null +++ b/kongfz/kongfzDll.go @@ -0,0 +1,190 @@ +package main + +import "fmt" + +func main() { + //upload, err := kongfzImageUpload(576, "256e10220c5b307f5172b1a49c11467a6cfa8038bbe2a7feccc42231852324f8", + // "306849d7541661989453102bc0e1a4f000c4469ab084beddc7b10a90ca55f6bb", + // "D:/work/image/9787511220660.jpg", + // "D:\\work\\project\\kfzgw-info\\kongfz") + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println(upload) + + //var q OrderQueryParams + //q.UserType = "buyer" + //marshal, err := json.Marshal(q) + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println(string(marshal)) + // + //list, err := kongfzOrderList(576, "256e10220c5b307f5172b1a49c11467a6cfa8038bbe2a7feccc42231852324f8", + // "306849d7541661989453102bc0e1a4f000c4469ab084beddc7b10a90ca55f6bb", string(marshal)) + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println(list) + // + //key, err := outKfzimgKey("de810e9e702de07c50601ef27bb51c93878c920a", "") + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println(key) + // + //upload, err := outKfzimgUpload("8b7f613a6f9de5dbaff8f6c419e1c1de8495a993", "", + // "D:/work/image/9787115460622.jpg") + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println(upload) + + //uploadd, err := outKfzimgUploadd("de810e9e702de07c50601ef27bb51c93878c920a", "", + // "D:/work/image/9787511220660.jpg") + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println(uploadd) + + appId := 576 + appSecret := "256e10220c5b307f5172b1a49c11467a6cfa8038bbe2a7feccc42231852324f8" + accessToken := "7c5ef2e492f82457898dd9a1bc2eb725df3a67b282721c9c52bfa24e460c4d92" + shippingComName := "韵达快递" + orderId := 275723461 + shippingId := "" + shippingCom := "" + shipmentNum := "312944151258647" + userDefined := "" + moreShipmentNum := "" + + synchronization, err := kongfzOrderSynchronization(appId, appSecret, accessToken, shippingComName, orderId, shippingId, shippingCom, shipmentNum, userDefined, moreShipmentNum) + if err != nil { + fmt.Println(err) + } + fmt.Println(synchronization) + + //data := ItemInfo{ + // ItemName: "至关重要的关系", + // ISBN: "9787550213869", + // Author: "[美]里德·霍夫曼、本·卡斯诺瓦 著；钱峰 译", + // Press: "北京联合出版公司", + // YearsGroup: "1", + // PubDate: "2013-04", + // PubDateYear: "2013", + // PubDateMonth: "4", + // Edition: "1", + // Binding: "2", // 2可能表示平装 + // Quality: "95", + // Price: "1500", + // Number: "2", + // ItemSn: "a1", + // MouldId: "909963", + // DeliverTime: "48h", + // IsDeliverTimeDefault: "48h", + // CatId: "43000000000000000", + // PageType: "add", + // OldCatId: "0000000000000000000", + // Tpl: "13", + // BearShipping: "buyer", + // WeightPiece: "1", + // IsOnSale: "1", + // Weight: "0.5", + // IsUseMould: "1", + // DeliverTimeGroup: "1", + //} + //marshal, err := json.Marshal(data) + //if err != nil { + // fmt.Println(err) + //} + // + //file, err := outAddGoodsAndFile("7feb7d192911b5440a4c83d81decd568e3cb4833", "", "D:/work/image/9787115460622.jpg", string(marshal)) + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println(string(file)) +} + +type ItemInfo struct { + ItemName string `json:"itemName"` + ISBN string `json:"isbn"` + Author string `json:"author"` + Press string `json:"press"` + YearsGroup string `json:"yearsGroup"` + PubDate string `json:"pubDate"` + PubDateYear string `json:"pubDateYear"` + PubDateMonth string `json:"pubDateMonth"` + Edition string `json:"edition"` + Binding string `json:"binding"` + Quality string `json:"quality"` + Price string `json:"price"` + Number string `json:"number"` + ItemSn string `json:"itemSn"` + Images []ImageInfo `json:"images"` + MouldId string `json:"mouldId"` + DeliverTime string `json:"deliverTime"` + IsDeliverTimeDefault string `json:"isDeliverTimeDefault"` + CatId string `json:"catId"` + PageType string `json:"pageType"` + OldCatId string `json:"oldCatId"` + Tpl string `json:"tpl"` + BearShipping string `json:"bearShipping"` + WeightPiece string `json:"weightPiece"` + IsOnSale string `json:"isOnSale"` + Weight string `json:"weight"` + IsUseMould string `json:"isUseMould"` + DeliverTimeGroup string `json:"deliverTimeGroup"` +} + +type ImageInfo struct { + ImgUrl string `json:"imgUrl"` + IsMain int `json:"isMain"` +} + +// OrderQueryParams 订单查询参数结构体 +type OrderQueryParams struct { + // UserType 用户类型。取值 seller: 卖家；buyer: 买家。 + UserType string `json:"userType" binding:"required"` + + // OrderStatus 订单状态。默认为全部订单。 + // Trading：交易中； + // Pending：待确认； + // ConfirmedToPay：待付款； + // PaidToShip：待发货； + // ShippedToReceipt：待确认收货； + // Refund：退货退款中； + // sellerReviewed：待评价； + // Successful：成功完成； + // RefundDeald：已退货退款； + // BuyerCancelled：买家取消； + // SellerCancelledBeforeConfirm：卖家取消； + // AdminClosedBeforeConfirm：管理员关闭。 + OrderStatus string `json:"orderStatus,omitempty"` + + // PageNum 页码。默认为：1 + PageNum int `json:"pageNum,omitempty"` + + // PageSize 每页最大条数。默认为：20，最大值为：100 + PageSize int `json:"pageSize,omitempty"` + + // IsDelete 是否已删除。默认查询未删除的 + IsDelete int `json:"isDelete,omitempty"` + + // StartDate 根据订单生成时间查询的起始日期，格式为yyyy-mm-dd，时区为GMT+8 + StartDate string `json:"startDate,omitempty"` + + // EndDate 根据订单生成时间查询的截止日期，格式为yyyy-mm-dd，时区为GMT+8 + EndDate string `json:"endDate,omitempty"` + + // StartTime 根据订单生成时间查询的起始时间，格式为yyyy-mm-dd hh:mm:ss，时区为GMT+8 + StartTime string `json:"startTime,omitempty"` + + // EndTime 根据订单生成时间查询的截止时间，格式为yyyy-mm-dd hh:mm:ss，时区为GMT+8 + EndTime string `json:"endTime,omitempty"` + + // StartUpdateTime 根据订单更新时间查询的起始时间，格式为yyyy-mm-dd hh:mm:ss，时区为GMT+8 + StartUpdateTime string `json:"startUpdateTime,omitempty"` + + // EndUpdateTime 根据订单更新时间查询的截止时间，格式为yyyy-mm-dd hh:mm:ss，时区为GMT+8 + EndUpdateTime string `json:"endUpdateTime,omitempty"` +} diff --git a/main.go b/main.go index 65910f0..b606fc3 100644 --- a/main.go +++ b/main.go @@ -1,3308 +1,153 @@ package main -/* -#include - -// proxyConfig.dll 函数声明 -extern char* ProxyTypeManager(char* proxyType, char* username, char* password, char* machineCode); -extern void FreeCString(char* str); -*/ -import "C" import ( - "context" - "database/sql" - "encoding/json" + "errors" "fmt" - "io" - "log" - "math/rand" - "net/http" - "net/url" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "sync" - "syscall" + "testing" "time" - "unsafe" - - "github.com/PuerkitoBio/goquery" - "github.com/chromedp/chromedp" - _ "github.com/go-sql-driver/mysql" - "github.com/parnurzeal/gorequest" ) -const ( - UseProxy = "proxy" // 使用代理 - NotProxy = "direct" // 不用代理代理 -) +// 回调函数的类型定义 +type Callback func(int) int -type Config struct { - App struct { - MaxRetryTimes int `ini:"app.max_retry_times" json:"max_retry_times" default:"3"` - RateLimitDelay time.Duration `ini:"app.rate_limit_delay" json:"rate_limit_delay" default:"500ms"` - Size int `ini:"app.size" json:"size" default:"5"` - DefaultUserAgent string `ini:"app.default_user_agent" json:"default_user_agent" default:"Mozilla/5.0"` - } `ini:"app" json:"app"` - - API struct { - LoginURL string `ini:"api.login_url" json:"login_url" default:"https://login.kongfz.com/Pc/Login/account"` - BookSearchURL string `ini:"api.book_search_url" json:"book_search_url" default:"https://search.kongfz.com/pc-gw/search-web/client/pc/bookLib/keyword/list"` - ProductSearchURL string `ini:"api.product_search_url" json:"product_search_url" default:"https://search.kongfz.com/pc-gw/search-web/client/pc/product/keyword/list"` - } `ini:"api" json:"api"` - - Proxy struct { - Servers string `ini:"proxy.servers" json:"servers" default:"http-dynamic.xiaoxiangdaili.com,http-dynamic-S02.xiaoxiangdaili.com,http-dynamic-S03.xiaoxiangdaili.com,http-dynamic-S04.xiaoxiangdaili.com"` - Username string `ini:"proxy.username" json:"username" default:"1297757178467602432"` - Password string `ini:"proxy.password" json:"password" default:"QgQBvP7f"` - TailMachineCode string `ini:"proxy.tail_machine_code" json:"tail_machine_code" default:"b7bf22a237ec692f13fcc2c43ee63252"` - TailCardKey string `ini:"proxy.tail_card_key" json:"tail_card_key" default:"DL_20_YK_1920acb2129844c2aabade3896560a9b"` - ProxyFilePath string `ini:"proxy.proxy_file_path" json:"proxy_file_path" default:"dll/proxyConfig.dll"` - } `ini:"proxy" json:"proxy"` - - Database struct { - Username string `ini:"database.username" json:"username" default:"newAdmin"` - Password string `ini:"database.password" json:"password" default:"bYPp8SbBe5F7nz2i"` - Host string `ini:"database.host" json:"host" default:"146.56.227.42:3306"` - Name string `ini:"database.name" json:"name" default:"newadmin"` - } `ini:"database" json:"database"` -} - -// 条目详情结构体 -type BookInfo struct { - BookName string `json:"book_name"` // 书名 - Author string `json:"author"` // 作者 - Publisher string `json:"publisher"` // 出版社 - ISBN string `json:"isbn"` // ISBN - PublicationTime int64 `json:"publication_time"` // 出版时间 - Edition string `json:"edition"` // 版次 - PrintTime string `json:"print_time"` // 印刷时间 - FixPrice string `json:"fix_price"` // 定价 - BindingLayout string `json:"binding_layout"` // 装帧 - Format string `json:"format"` // 开本 - Paper string `json:"paper"` // 纸张 - Pages string `json:"pages"` // 页数 - Wordage string `json:"wordage"` // 字数 - Languages string `json:"languages"` // 语种 - Era string `json:"era"` // 年代 - EngravingMethod string `json:"engraving_method"` // 刻印方式 - Dimensions string `json:"dimensions"` // 尺寸 - VolumeNumber string `json:"volume_number"` // 册数 - BookPic string `json:"book_pic"` // 图书封面图(官图) - BookPicS string `json:"book_pic_s"` // 图书封面图(实拍图) - SellingPrice string `json:"selling_price"` // 售价 - Condition string `json:"condition"` // 品相 - ExpressDeliveryFee string `json:"express_delivery_fee"` // 快递费 - Editor string `json:"editor"` // 编辑 - Category string `json:"category"` // 分类 - BuyCount string `json:"buy_count"` // 买过 - SellCount string `json:"sell_count"` // 在卖 - Content string `json:"content"` // 内容 - Mid int64 `json:"mid"` // 商家id - ItemId int64 `json:"item_id"` // 商品id - ShopId int64 `json:"shop_id"` // 店铺id - DetailUrl string `json:"detail_url"` // 商品详情url -} - -// API响应结构 -type APIResp struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` - GoodsNum string `json:"goods_num,omitempty"` - PNum string `json:"pnum,omitempty"` - Data interface{} `json:"data,omitempty"` - Error string `json:"error,omitempty"` -} - -// APIResponse -type APIResponse struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` - Data interface{} `json:"data,omitempty"` -} - -// 账号凭证结构 -type AccountCredential struct { - ID int64 - Username string - Password string - Token string -} - -// 全局变量 -var ( - cf Config // 配置信息 - tailProxyMu sync.Mutex // 互斥锁 - proxyFailCount int - mu sync.RWMutex // 数据库锁 - - // 全局代理管理器 - globalProxyManager *ProxyConfigManager - proxyManagerOnce sync.Once - proxyManagerInitErr error -) - -// ProductInfo 商品信息结构 -type ProductInfo struct { - ItemID string `json:"itemId"` - BookName string `json:"bookName"` - Price string `json:"price"` - ShippingFee string `json:"shippingFee"` -} - -// ProductResponse 响应结构 -type ProductResponse struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` - Data []ProductInfo `json:"data,omitempty"` -} - -// 并行获取详情的结果结构 -type DetailResult struct { - URL string - Doc *goquery.Document - Error error - Index int -} - -// 第一阶段：收集基本信息并识别需要详情的项目 -type BookItem struct { - Book BookInfo - Selection *goquery.Selection - HasDetail bool - DetailURL string - Index int -} - -// 分类项结构体 -type SalesCategory struct { - Key string `json:"key"` - Value int `json:"value"` -} - -// 图书详情响应结构体 -type BookDetailResponse struct { - Status bool `json:"status"` - Result BookList `json:"result"` - ErrMessage string `json:"errMessage"` - ErrCode int `json:"errCode"` -} - -// 图书列表结构体 -type BookList struct { - Current int `json:"current"` - Data []BookInformation `json:"data"` - Total int `json:"total"` -} - -// 图书信息结构体 -type BookInformation struct { - Author string `json:"author"` - BookName string `json:"bookName"` - ContentIntroduction string `json:"contentIntroduction"` - ImgUrl string `json:"imgUrl"` - Isbn string `json:"isbn"` - ItemUrls ItemUrls `json:"itemUrls"` - Mid int `json:"mid"` - NewMinPrice string `json:"newMinPrice"` - OldMinPrice string `json:"oldMinPrice"` - Press string `json:"press"` - Price string `json:"price"` - PubDate string `json:"pubDate"` - RiseTag string `json:"riseTag"` - AuthorArr []AuthorInfo `json:"authorArr"` - PressUrl string `json:"pressUrl"` -} - -// 作者信息结构体 -type AuthorInfo struct { - Name string `json:"name"` - OriName string `json:"oriName"` - Nationality string `json:"nationality"` - Role string `json:"role"` - Url string `json:"url"` -} - -// 商品链接结构体 -type ItemUrls struct { - AppUrl string `json:"appUrl"` - MUrl string `json:"mUrl"` - MiniUrl string `json:"miniUrl"` - PcUrl string `json:"pcUrl"` -} - -type UserInfo struct { - UserID int64 `json:"userId"` - Nickname string `json:"nickname"` - Mobile string `json:"mobile"` -} - -// 店铺快递费参数 -type ParamsInfo struct { - Params []Param `json:"params"` - Area string `json:"area"` -} - -type Param struct { - UserId string `json:"userId"` - ItemId string `json:"itemId"` -} - -// ProxyConfigManager 代理配置DLL管理器 -type ProxyConfigManager struct { - dll *syscall.DLL -} - -func NewProxyConfigManager(dllPath string) (*ProxyConfigManager, error) { - dll, err := syscall.LoadDLL(dllPath) - if err != nil { - return nil, fmt.Errorf("加载代理配置DLL失败: %v", err) - } - return &ProxyConfigManager{dll: dll}, nil -} - -func (m *ProxyConfigManager) Close() { - if m.dll != nil { - m.dll.Release() - } -} - -// ProxyTypeManager 调用代理类型管理器 -func (m *ProxyConfigManager) ProxyTypeManager(proxyType, username, password, machineCode string) (string, error) { - proc, err := m.dll.FindProc("ProxyTypeManager") - if err != nil { - return "", fmt.Errorf("找不到函数 ProxyTypeManager: %v", err) - } - // 准备参数 - proxyTypePtr, _ := syscall.BytePtrFromString(proxyType) - usernamePtr, _ := syscall.BytePtrFromString(username) - passwordPtr, _ := syscall.BytePtrFromString(password) - machineCodePtr, _ := syscall.BytePtrFromString(machineCode) - // 调用函数 - r1, _, err := proc.Call( - uintptr(unsafe.Pointer(proxyTypePtr)), - uintptr(unsafe.Pointer(usernamePtr)), - uintptr(unsafe.Pointer(passwordPtr)), - uintptr(unsafe.Pointer(machineCodePtr)), - ) - - if err != nil && err.Error() != "The operation completed successfully." { - return "", fmt.Errorf("调用 ProxyTypeManager 失败: %v", err) - } - // 转换结果 - result := (*byte)(unsafe.Pointer(r1)) - var resultBytes []byte - for i := 0; ; i++ { - bytePtr := (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i))) - if *bytePtr == 0 { - break - } - resultBytes = append(resultBytes, *bytePtr) - } - // 释放内存 - freeProc, _ := m.dll.FindProc("FreeCString") - if freeProc != nil { - freeProc.Call(r1) - } - return string(resultBytes), nil -} - -// 登录 -func outLogin(username, password string) (string, error) { - if username == "" || password == "" { - return "", fmt.Errorf("请输入用户名和密码!") - } - formData := map[string]string{ - "loginName": username, - "loginPass": password, - "returnUrl": "http://user.kongfz.com/", - } - resp, body, errs := gorequest.New(). - Post(cf.API.LoginURL). - Set("Content-Type", "application/x-www-form-urlencoded"). - Set("User-Agent", cf.App.DefaultUserAgent). - Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"). - Send(formData). - Timeout(15 * time.Second).End() - if len(errs) > 0 { - return "", fmt.Errorf("登录请求失败: %v", errs) - } - // 检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("登录失败(HTTP状态码: %d)", resp.StatusCode) - } - // 提取Cookie - cookie := resp.Header.Get("Set-Cookie") - if strings.Contains(body, "window.location.href='https://login.kongfz.cn/Pc/Session/rsync") { - if cookie == "" { - return "", fmt.Errorf("登录成功但未获取到Cookie") - } - // 登录成功 - if strings.Contains(cookie, "PHPSESSID=") { - token := strings.Split(strings.Split(cookie, "PHPSESSID=")[1], ";")[0] - return token, nil - } - return "", fmt.Errorf("登录失败: 未找到PHPSESSID") - } - // 错误信息 - var res struct { - Status bool `json:"status"` - ErrCode int `json:"errCode"` - ErrInfo string `json:"errInfo"` - } - if err := json.Unmarshal([]byte(body), &res); err == nil { - if res.ErrCode == 1001 || res.ErrCode == 1005 { - return "", fmt.Errorf("账号或密码错误！") - } - if res.ErrInfo != "" { - return "", fmt.Errorf("登录失败: %s", res.ErrInfo) - } - } - return "", fmt.Errorf("登录失败，未知错误！") -} - -// 获取用户信息(带有Out的都非官方标准接口) -func outGetUserMsg(token string) (*UserInfo, error) { - url := "https://user.kongfz.com/User/Index/getUserInfo/" - resp, body, errs := gorequest.New(). - Get(url). - Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Timeout(15 * time.Second). - End() - if len(errs) > 0 { - return nil, fmt.Errorf("查询请求失败: %v", errs) - } - //检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", resp.Status) - } - var userInfo struct { - Status bool `json:"status"` - Data struct { - UserID int64 `json:"userId"` - Nickname string `json:"nickname"` - Mobile string `json:"mobile"` - } - } - if err := json.Unmarshal([]byte(body), &userInfo); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - user := &UserInfo{} - if !userInfo.Status { - return nil, fmt.Errorf("获取用户失败！") - } - user.UserID = userInfo.Data.UserID - user.Nickname = userInfo.Data.Nickname - user.Mobile = userInfo.Data.Mobile - return user, nil -} - -// 获取商品模版 -func outGetGoodsTplMsg(token, itemId, proxy string) (map[string]interface{}, error) { - if token == "" { - return nil, fmt.Errorf("请先登录获取Token") - } - url := fmt.Sprintf("https://seller.kongfz.com/pc/itemInfo/getTplFields?itemId=%s&isClone=1&v=%d", - itemId, time.Now().Unix()) - // 创建HTTP客户端 - request := gorequest.New() - // 设置代理（如果有提供代理URL） - if proxy != "" { - request.Proxy(proxy) - } - resp, body, errs := request.Get(url). - Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Set("Referer", "https://seller.kongfz.com/"). - Set("Origin", "https://seller.kongfz.com"). - Timeout(15 * time.Second). - End() - if errs != nil { - // 检查是否是代理相关错误 - var isProxyError bool - var errorDetails []string - for _, e := range errs { - errorStr := e.Error() - errorDetails = append(errorDetails, errorStr) - if strings.Contains(errorStr, "Proxy Authentication Required") || - strings.Contains(errorStr, "connectex: A connection attempt failed") || - strings.Contains(errorStr, "connectex: No connection could be made") || - strings.Contains(errorStr, "proxyconnect tcp") || - strings.Contains(errorStr, "timeout") || - strings.Contains(errorStr, "connection refused") { - isProxyError = true - } - } - log.Printf("[ERROR] 请求错误详情: %v", errorDetails) - if isProxyError { - // 处理代理失败 - return nil, fmt.Errorf("代理连接失败") - } - return nil, fmt.Errorf("查询请求失败: %v", errs) - } - //检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", resp.Status) - } - var data map[string]interface{} - if err := json.Unmarshal([]byte(body), &data); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - if val, ok := data["status"].(float64); ok && val == 1 { - return data, nil - } - return nil, fmt.Errorf("API返回错误: %+v", data) -} - -// 获取商品列表-已登的店铺 -func outGetGoodsListMsgFromSelfShop(token string, proxy string, itemSn string, priceMin string, priceMax string, startCreateTime string, - endCreateTime string, requestType string, isItemSnEqual int, page int, size int) (map[string]interface{}, error) { - if token == "" { - return nil, fmt.Errorf("请先登录获取Token") - } - url := "https://seller.kongfz.com/pc-gw/book-manage-service/client/pc/goods/unSold/list" - formData := struct { - ItemSn string `json:"itemSn"` - PriceMin string `json:"priceMin"` - PriceMax string `json:"priceMax"` - StartCreateTime string `json:"startCreateTime"` - EndCreateTime string `json:"endCreateTime"` - RequestType string `json:"requestType,omitempty"` - IsItemSnEqual int `json:"isItemSnEqual,omitempty"` - Page int `json:"page,omitempty"` - Size int `json:"size,omitempty"` - }{ - ItemSn: itemSn, - PriceMin: priceMin, - PriceMax: priceMax, - StartCreateTime: startCreateTime, - EndCreateTime: endCreateTime, - RequestType: requestType, - IsItemSnEqual: isItemSnEqual, - Page: page, - Size: size, - } - request := gorequest.New() - if proxy != "" { - request.Proxy(proxy) - } - resp, body, errs := request.Post(url). - Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Send(formData). - Timeout(15 * time.Second). - End() - if errs != nil { - // 检查是否是代理相关错误 - var isProxyError bool - var errorDetails []string - for _, e := range errs { - errorStr := e.Error() - errorDetails = append(errorDetails, errorStr) - if strings.Contains(errorStr, "Proxy Authentication Required") || - strings.Contains(errorStr, "connectex: A connection attempt failed") || - strings.Contains(errorStr, "connectex: No connection could be made") || - strings.Contains(errorStr, "proxyconnect tcp") || - strings.Contains(errorStr, "timeout") || - strings.Contains(errorStr, "connection refused") { - isProxyError = true - } - } - log.Printf("[ERROR] 请求错误详情: %v", errorDetails) - if isProxyError { - // 处理代理失败 - return nil, fmt.Errorf("代理连接失败") - } - return nil, fmt.Errorf("查询请求失败: %v", errs) - } - //检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", resp.Status) - } - var data map[string]interface{} - if err := json.Unmarshal([]byte(body), &data); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - if _, ok := data["status"]; ok { - return data, nil - } - return nil, fmt.Errorf("API返回错误: %+v", data) -} - -// 删除商品-已登的店铺(带有Out的都非官方标准接口) -func outDelGoodsFromSelfShop(token, proxy, itemId string) (map[string]interface{}, error) { - if token == "" { - return nil, fmt.Errorf("请先登录获取Token") - } - url := "https://seller.kongfz.com/pc-gw/book-manage-service/client/pc/goods/quickUpdate" - formData := map[string]string{ - "itemId": itemId, - "updateType": "delete", - "value": "1", - } - request := gorequest.New() - if proxy != "" { - request.Proxy(proxy) - } - resp, body, errs := request.Post(url). - Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Send(formData). - Timeout(15 * time.Second). - End() - if errs != nil { - // 检查是否是代理相关错误 - var isProxyError bool - var errorDetails []string - for _, e := range errs { - errorStr := e.Error() - errorDetails = append(errorDetails, errorStr) - if strings.Contains(errorStr, "Proxy Authentication Required") || - strings.Contains(errorStr, "connectex: A connection attempt failed") || - strings.Contains(errorStr, "connectex: No connection could be made") || - strings.Contains(errorStr, "proxyconnect tcp") || - strings.Contains(errorStr, "timeout") || - strings.Contains(errorStr, "connection refused") { - isProxyError = true - } - } - log.Printf("[ERROR] 请求错误详情: %v", errorDetails) - if isProxyError { - // 处理代理失败 - return nil, fmt.Errorf("代理连接失败") - } - return nil, fmt.Errorf("查询请求失败: %v", errs) - } - //检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", resp.Status) - } - var data map[string]interface{} - if err := json.Unmarshal([]byte(body), &data); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - if status, ok := data["status"].(bool); ok && status { - return data, nil - } - if status, ok := data["status"].(float64); ok && status == 1 { - return data, nil - } - return nil, fmt.Errorf("API返回错误: %+v", data) -} - -// 新增商品(带有Out的都非官方标准接口) -func outAddGoods(token, proxy, formData string) (map[string]interface{}, error) { - if token == "" { - return nil, fmt.Errorf("请先登录获取Token") - } - url := "https://seller.kongfz.com/pc/itemInfo/add" - request := gorequest.New() - if proxy != "" { - request.Proxy(proxy) - } - resp, body, errs := request.Post(url). - Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Send(formData). - Timeout(15 * time.Second). - End() - if errs != nil { - // 检查是否是代理相关错误 - var isProxyError bool - var errorDetails []string - for _, e := range errs { - errorStr := e.Error() - errorDetails = append(errorDetails, errorStr) - if strings.Contains(errorStr, "Proxy Authentication Required") || - strings.Contains(errorStr, "connectex: A connection attempt failed") || - strings.Contains(errorStr, "connectex: No connection could be made") || - strings.Contains(errorStr, "proxyconnect tcp") || - strings.Contains(errorStr, "timeout") || - strings.Contains(errorStr, "connection refused") { - isProxyError = true - } - } - log.Printf("[ERROR] 请求错误详情: %v", errorDetails) - if isProxyError { - // 处理代理失败 - return nil, fmt.Errorf("代理连接失败") - } - return nil, fmt.Errorf("查询请求失败: %v", errs) - } - //检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", resp.Status) - } - var data map[string]interface{} - if err := json.Unmarshal([]byte(body), &data); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - if val, ok := data["status"]; ok && val == 1 { - return data, nil - } - return nil, fmt.Errorf("API返回错误: %+v", data) -} - -// 获取图片URL(官图和拍图)带有店铺过滤 -func outGetImageFilterShopId(token string, isbn string, shopId int, proxy string, isLiveImage int, isReturnMsg int) (map[string]string, error) { - if isLiveImage == 0 { - gtUrl := fmt.Sprintf("%s?keyword=%s", cf.API.BookSearchURL, isbn) - request := gorequest.New() - if proxy != "" { - request.Proxy(proxy) - } - resp, body, errs := request.Get(gtUrl). - Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Set("Referer", "https://item.kongfz.com/"). - Timeout(30 * time.Second). - End() - if len(errs) > 0 { - // 检查是否是代理相关错误 - var isProxyError bool - var errorDetails []string - for _, e := range errs { - errorStr := e.Error() - errorDetails = append(errorDetails, errorStr) - if strings.Contains(errorStr, "Proxy Authentication Required") || - strings.Contains(errorStr, "connectex: A connection attempt failed") || - strings.Contains(errorStr, "connectex: No connection could be made") || - strings.Contains(errorStr, "proxyconnect tcp") || - strings.Contains(errorStr, "timeout") || - strings.Contains(errorStr, "connection refused") { - isProxyError = true - } - } - log.Printf("[ERROR] 请求错误详情: %v", errorDetails) - if isProxyError { - // 处理代理失败 - return nil, fmt.Errorf("代理连接失败") - } - return nil, fmt.Errorf("查询请求失败: %v", errs) - } - //检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", resp.Status) - } - // 解析响应 - var apiGtResp struct { - Status int `json:"status"` - ErrType string `json:"errType"` - Message string `json:"message"` - SystemTime int64 `json:"systemTime"` - Data struct { - ItemResponse struct { - Total int `json:"total"` - List []struct { - BookName string `json:"bookName"` - Mid int64 `json:"mid"` - ImgUrlEntity struct { - BigImgUrl string `json:"bigImgUrl"` - } `json:"imgUrlEntity"` - Isbn string `json:"isbn"` - BookShowInfo []string `json:"bookShowInfo"` - } `json:"list"` - } `json:"itemResponse"` - } `json:"data"` - } - if err := json.Unmarshal([]byte(body), &apiGtResp); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - if apiGtResp.ErrType == "102" { - return nil, fmt.Errorf("错误信息: %w，状态码: %s", apiGtResp.Message, apiGtResp.ErrType) - } - var info map[string]string - // 如果找到条目，返回图片URL - if apiGtResp.Data.ItemResponse.Total > 0 && len(apiGtResp.Data.ItemResponse.List) > 0 { - list := apiGtResp.Data.ItemResponse.List[0] - info = map[string]string{ - "book_name": list.BookName, - "book_pic": list.ImgUrlEntity.BigImgUrl, - "isbn": list.Isbn, - } - } - return info, nil - } - if isLiveImage == 1 { - // 实拍图 - sptUrl := fmt.Sprintf("%s?dataType=0&keyword=%s&page=1&size=%d&sortType=7&actionPath=quality,sortType&quality=85~&quaSelect=2&userArea=13003000000", cf.API.ProductSearchURL, isbn, cf.App.Size) - //创建HTTP客户端 - requestSpt := gorequest.New() - //设置代理(如果有提供代理URL) - if proxy != "" { - requestSpt.Proxy(proxy) - } - // 发送请求 - respSpt, bodySpt, errsSpt := requestSpt.Get(sptUrl). - Proxy(proxy). - Set("Cookie", token). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Set("Referer", "https://item.kongfz.com/"). - Timeout(30 * time.Second). - End() - // 错误处理 - if len(errsSpt) > 0 { - return nil, fmt.Errorf("请求失败: %v", errsSpt) - } - // 检查HTTP状态码 - if respSpt.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", respSpt.Status) - } - // 解析响应 - var apiSptResp struct { - Status int `json:"status"` - ErrType string `json:"errType"` - Message string `json:"message"` - SystemTime int64 `json:"systemTime"` - Data struct { - ItemResponse struct { - Total int `json:"total"` - List []struct { - ItemId int64 `json:"itemId"` - Title string `json:"title"` - ImgUrl string `json:"imgUrl"` - ImgBigUrl string `json:"imgBigUrl"` - Author string `json:"author"` - PubDateText string `json:"pubDateText"` - Isbn string `json:"isbn"` - Press string `json:"press"` - ShopId int64 `json:"shopId"` - TplRecords []struct { - Key string `json:"key"` - Value string `json:"value"` - } `json:"tplRecords"` - } `json:"list"` - } `json:"itemResponse"` - } `json:"data"` - } - // 解析JSON - if err := json.Unmarshal([]byte(bodySpt), &apiSptResp); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - if apiSptResp.ErrType == "102" { - return nil, fmt.Errorf("错误信息: %w，状态码: %s", apiSptResp.Message, apiSptResp.ErrType) - } - var info map[string]string - if apiSptResp.Data.ItemResponse.Total > 0 && len(apiSptResp.Data.ItemResponse.List) > 0 { - // 确定起始索引 - var startIndex int - if cf.App.Size >= apiSptResp.Data.ItemResponse.Total { - startIndex = apiSptResp.Data.ItemResponse.Total - 1 - } else { - startIndex = cf.App.Size - 1 - } - for attempt := 0; attempt < 3; attempt++ { - currentIndex := startIndex - attempt - // 检查索引是否有效 - if currentIndex < 0 || currentIndex >= len(apiSptResp.Data.ItemResponse.List) { - log.Printf("[DEBUG] 索引 %d 超出范围，跳过", currentIndex) - continue - } - randomNum := rand.Intn(currentIndex + 1) - item := apiSptResp.Data.ItemResponse.List[randomNum] - // 检查图片URL是否存在 - if item.ImgBigUrl == "" { - log.Printf("[DEBUG] 索引 %d 的图片URL为空，跳过", randomNum) - continue - } - // 过滤店铺 - if item.ShopId == int64(shopId) { - log.Printf("[DEBUG] 索引 %d 的店铺ID需要过滤，跳过", randomNum) - continue - } - info = map[string]string{ - "book_name": item.Title, - "book_pic": item.ImgUrl, - "isbn": item.Isbn, - } - return info, nil - } - } - } - return nil, nil -} - -// 获取图片URL(官图和拍图) -func outGetImageByIsbn(token string, isbn string, proxy string, isLiveImage int, isReturnMsg int) (*BookInfo, error) { - fmt.Println("[DEBUG] 使用的ISBN: ", isbn) - // isLiveImage 1实拍图 0官图 ，isReturnMsg 0商品信息 - bookInfo := &BookInfo{} - if isLiveImage == 0 { - // 孔网官图请求 - gtUrl := fmt.Sprintf("%s?keyword=%s", cf.API.BookSearchURL, isbn) - // 创建HTTP客户端 - requestGt := gorequest.New() - // 设置代理(如果有提供代理URL) - if proxy != "" { - requestGt.Proxy(proxy) - } - // 发送请求 - respGt, bodyGt, errsGt := requestGt.Get(gtUrl). - Set("Cookie", fmt.Sprintf("PHPSESSID=%s", token)). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Set("Referer", "https://item.kongfz.com/"). - Timeout(30 * time.Second). - End() - if len(errsGt) > 0 { - // 检查是否是代理相关错误 - var isProxyError bool - var errorDetails []string - for _, e := range errsGt { - errorStr := e.Error() - errorDetails = append(errorDetails, errorStr) - if strings.Contains(errorStr, "Proxy Authentication Required") || - strings.Contains(errorStr, "connectex: A connection attempt failed") || - strings.Contains(errorStr, "connectex: No connection could be made") || - strings.Contains(errorStr, "proxyconnect tcp") || - strings.Contains(errorStr, "timeout") || - strings.Contains(errorStr, "connection refused") { - isProxyError = true - } - } - log.Printf("[ERROR] 请求错误详情: %v", errorDetails) - if isProxyError { - // 处理代理失败 - return nil, fmt.Errorf("代理连接失败") - } - return nil, fmt.Errorf("查询请求失败: %v", errsGt) - } - //检查HTTP状态码 - if respGt.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", respGt.Status) - } - // 解析响应 - var apiGtResp struct { - Status int `json:"status"` - ErrType string `json:"errType"` - Message string `json:"message"` - SystemTime int64 `json:"systemTime"` - Data struct { - ItemResponse struct { - Total int `json:"total"` - List []struct { - BookName string `json:"bookName"` - Mid int64 `json:"mid"` - ImgUrlEntity struct { - BigImgUrl string `json:"bigImgUrl"` - } `json:"imgUrlEntity"` - Isbn string `json:"isbn"` - BookShowInfo []string `json:"bookShowInfo"` - } `json:"list"` - } `json:"itemResponse"` - } `json:"data"` - } - if err := json.Unmarshal([]byte(bodyGt), &apiGtResp); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - if apiGtResp.ErrType == "102" { - return nil, fmt.Errorf("错误信息: %w，状态码: %s", apiGtResp.Message, apiGtResp.ErrType) - } - // 如果找到条目，返回图片URL - if apiGtResp.Data.ItemResponse.Total > 0 && len(apiGtResp.Data.ItemResponse.List) > 0 { - list := apiGtResp.Data.ItemResponse.List[0] - bookShowInfo := list.BookShowInfo - bookInfo.BookName = list.BookName - bookInfo.BookPic = list.ImgUrlEntity.BigImgUrl - bookInfo.ISBN = list.Isbn - // 根据长度安全填充字段 - if isReturnMsg == 0 { - if len(bookShowInfo) > 0 { - bookInfo.Author = bookShowInfo[0] - } - if len(bookShowInfo) > 1 { - bookInfo.Publisher = bookShowInfo[1] - } - if len(bookShowInfo) > 2 { - bookInfo.PublicationTime = validateDateFormat(bookShowInfo[2]) - } - if len(bookShowInfo) > 3 { - bookInfo.BindingLayout = bookShowInfo[3] - } - if len(bookShowInfo) > 4 { - bookInfo.FixPrice = bookShowInfo[4] - } else { - log.Printf("[WARN] BookShowInfo 长度不足 (仅 %d 项): %v", len(bookShowInfo), bookShowInfo) - } - } - } - return bookInfo, nil - } - if isLiveImage == 1 { - // 实拍图 - sptUrl := fmt.Sprintf("%s?dataType=0&keyword=%s&page=1&size=%d&sortType=7&actionPath=quality,sortType&quality=85~&quaSelect=2&userArea=13003000000", cf.API.ProductSearchURL, isbn, cf.App.Size) - //创建HTTP客户端 - requestSpt := gorequest.New() - //设置代理(如果有提供代理URL) - if proxy != "" { - requestSpt.Proxy(proxy) - } - // 发送请求 - respSpt, bodySpt, errsSpt := requestSpt.Get(sptUrl). - Proxy(proxy). - Set("Cookie", token). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Set("Referer", "https://item.kongfz.com/"). - Timeout(30 * time.Second). - End() - // 错误处理 - if len(errsSpt) > 0 { - return nil, fmt.Errorf("请求失败: %v", errsSpt) - } - // 检查HTTP状态码 - if respSpt.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", respSpt.Status) - } - // 解析响应 - var apiSptResp struct { - Status int `json:"status"` - ErrType string `json:"errType"` - Message string `json:"message"` - SystemTime int64 `json:"systemTime"` - Data struct { - ItemResponse struct { - Total int `json:"total"` - List []struct { - ItemId int64 `json:"itemId"` - Title string `json:"title"` - ImgUrl string `json:"imgUrl"` - ImgBigUrl string `json:"imgBigUrl"` - Author string `json:"author"` - PubDateText string `json:"pubDateText"` - Isbn string `json:"isbn"` - Press string `json:"press"` - ShopId int64 `json:"shopId"` - TplRecords []struct { - Key string `json:"key"` - Value string `json:"value"` - } `json:"tplRecords"` - } `json:"list"` - } `json:"itemResponse"` - } `json:"data"` - } - // 解析JSON - if err := json.Unmarshal([]byte(bodySpt), &apiSptResp); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - if apiSptResp.ErrType == "102" { - return nil, fmt.Errorf("错误信息: %w，状态码: %s", apiSptResp.Message, apiSptResp.ErrType) - } - if apiSptResp.Data.ItemResponse.Total > 0 && len(apiSptResp.Data.ItemResponse.List) > 0 { - // 确定其实索引 - var startIndex int - if cf.App.Size >= apiSptResp.Data.ItemResponse.Total { - startIndex = apiSptResp.Data.ItemResponse.Total - 1 - } else { - startIndex = cf.App.Size - 1 - } - for attempt := 0; attempt < 3; attempt++ { - currentIndex := startIndex - attempt - // 检查索引是否有效 - if currentIndex < 0 || currentIndex >= len(apiSptResp.Data.ItemResponse.List) { - log.Printf("[DEBUG] 索引 %d 超出范围，跳过", currentIndex) - continue - } - randomNum := rand.Intn(currentIndex + 1) - item := apiSptResp.Data.ItemResponse.List[randomNum] - // 检查图片URL是否存在 - if item.ImgBigUrl == "" { - log.Printf("[DEBUG] 索引 %d 的图片URL为空，跳过", randomNum) - continue - } - // 响应信息 - bookInfo.BookPicS = item.ImgUrl - bookInfo.ISBN = item.Isbn - if bookInfo.BookName == "" { - bookInfo.BookName = item.Title - if isReturnMsg == 0 { - // 安全地获取TplRecords中的值 - if len(item.TplRecords) > 0 { - bookInfo.Author = item.TplRecords[0].Value - } - if len(item.TplRecords) > 1 { - bookInfo.Publisher = item.TplRecords[1].Value - } - if len(item.TplRecords) > 2 { - bookInfo.PublicationTime = validateDateFormat(item.TplRecords[2].Value) - } - if len(item.TplRecords) > 3 { - bookInfo.BindingLayout = item.TplRecords[3].Value - } - } - } - return bookInfo, nil - } - } - } - return nil, fmt.Errorf("查询失败，没有数据！") -} - -// 获取商品列表通过店铺ID -func outGetGoodsListMsgByShopId(shopId int, proxy string, retPrice int, isImage int, sortType string, sort string, priceMin float32, priceMax float32, pageNum, returnNum int) (books []BookInfo, goodsNum string, pNum string, err error) { - // 判断店铺ID - if shopId == 0 { - return nil, "", "", fmt.Errorf("店铺编码为空!") - } - // 判断是否有图片，设置默认值0 - var isImageStr string - if isImage == 0 { - isImageStr = "0" - } else { - isImageStr = "1" - } - // 判断一页图书数量，设置默认值100 - if returnNum == 0 { - returnNum = 100 - } - // 判断页数，设置默认值1 - if pageNum == 0 { - pageNum = 1 - } - // 判断排序类型，设置默认值sort - if sortType == "" { - sortType = "sort" - } else { - validSortTypes := map[string]bool{ - "sort": true, - "putDate": true, - "newItem": true, - "price": true, - } - if !validSortTypes[sortType] { - return nil, "", "", fmt.Errorf("无效的排序类型: %s，可选值: sort, putDate, newItem, price", sortType) - } - } - // 判断排序，设置默认值desc - if sort == "" { - sort = "desc" - } else { - validSorts := map[string]bool{ - "desc": true, - "asc": true, - } - if !validSorts[sort] { - return nil, "", "", fmt.Errorf("无效的排序类型: %s，可选值: desc, asc", sort) - } - } - var url string - var pMin int - pMin = 0 - var pMax int - pMax = 0 - // 判断价格下限，设置默认值0 - if priceMin == 0 && priceMax == 0 { - // 调用的url - url = fmt.Sprintf("https://shop.kongfz.com/%d/all/%s_%d_0_0_%d_%s_%s_%d_%d", - shopId, isImageStr, returnNum, pageNum, sortType, sort, pMin, pMax) - } else if priceMin == 0 { - url = fmt.Sprintf("https://shop.kongfz.com/%d/all/%s_%d_0_0_%d_%s_%s_%d_%.2f", - shopId, isImageStr, returnNum, pageNum, sortType, sort, pMin, priceMax) - } else if priceMax == 0 { - url = fmt.Sprintf("https://shop.kongfz.com/%d/all/%s_%d_0_0_%d_%s_%s_%.2f_%d", - shopId, isImageStr, returnNum, pageNum, sortType, sort, priceMin, pMax) - } else { - url = fmt.Sprintf("https://shop.kongfz.com/%d/all/%s_%d_0_0_%d_%s_%s_%.2f_%.2f", - shopId, isImageStr, returnNum, pageNum, sortType, sort, priceMin, priceMax) - } - // 发送请求 - response, err := fetchResponse(url, proxy) - if err != nil { - return nil, "", "", err - } - body, err := io.ReadAll(response.Body) - if err != nil { - return nil, "", "", err - } - doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(body))) - // 全部商品数量 - num := doc.Find("div.crumbs-nav-main.clearfix").Find("span") - if match := regexp.MustCompile(`\d+`).FindString(num.Text()); match != "" { - goodsNum = match - } - // 商品页数 - pg := doc.Find("li.pull-right.page_num").Find("span") - _, split, found := strings.Cut(strings.TrimSpace(pg.Text()), "/") - if found { - pNum = split - } else { - log.Printf("未找到页数！") - } - infoDiv := doc.Find("div.list-content") - var params ParamsInfo - if infoDiv.Length() > 0 { - item := infoDiv.Find("div.item.clearfix") - for i := 0; i < item.Length(); i++ { - s := item.Eq(i) - book := BookInfo{} - // 书名 - book.BookName = strings.TrimSpace(s.Find("div.title a.link").Text()) - // 提取ISBN - book.ISBN = strings.TrimSpace(s.AttrOr("isbn", "")) - // 店铺ID - shopid, _ := strconv.Atoi(strings.TrimSpace(s.AttrOr("shopid", ""))) - book.ShopId = int64(shopid) - // 商品ID - itemid, _ := strconv.Atoi(strings.TrimSpace(s.AttrOr("itemid", ""))) - book.ItemId = int64(itemid) - // 详情URL - book.DetailUrl = s.Find("div.item-img a.img-box").AttrOr("href", "") - // 价格 - if retPrice == 0 { - book.SellingPrice = s.Find("div.f_right.red.price").Find("span.bold").Text() - params.Params = append(params.Params, struct { - UserId string `json:"userId"` - ItemId string `json:"itemId"` - }{UserId: strings.TrimSpace(s.AttrOr("userid", "")), ItemId: strings.TrimSpace(s.AttrOr("itemid", ""))}) - } - books = append(books, book) - } - if params.Params != nil { - params.Area = "13003000000" - dataItem, err := getGoodsListShippingFee(params, proxy) - if err != nil { - return nil, "", "", err - } - for i := 0; i < len(books); i++ { - itemId := fmt.Sprintf("%d", books[i].ItemId) - for _, data := range dataItem { - if itemId == data.ItemID { - books[i].ExpressDeliveryFee = data.Fee[0].TotalFee - } - } - } - } - } - return books, goodsNum, pNum, nil -} - -// 定义对应的结构体 -type FeeInfo struct { - ShippingID string `json:"shippingId"` - ShippingName string `json:"shippingName"` - TotalFee string `json:"totalFee"` - ShippingText string `json:"shippingText"` - FilterTotalFee string `json:"filterTotalFee"` -} - -type DataItem struct { - Fee []FeeInfo `json:"fee"` - IsSeller bool `json:"isSeller"` - ItemID string `json:"itemId"` - UserID string `json:"userId"` -} - -type ResponseStruct struct { - Status bool `json:"status"` - Data []DataItem `json:"data"` - Message string `json:"message"` - ErrType string `json:"errType"` -} - -// 获取店铺里商品的快递费(定位到河南) -func getGoodsListShippingFee(params ParamsInfo, proxy string) ([]DataItem, error) { - if params.Params == nil { - return nil, fmt.Errorf("没有商品信息！") - } - paramsJSON, err := json.Marshal(params) - if err != nil { - return nil, fmt.Errorf("参数序列化失败: %v", err) - } - // URL 编码参数 - encodedParams := url.QueryEscape(string(paramsJSON)) - url := fmt.Sprintf("https://shop.kongfz.com/book/shopsearch/getShippingFee?params=%s", encodedParams) - request := gorequest.New() - if proxy != "" { - request.Proxy(proxy) - } - // 发送请求 - resp, body, errs := request.Get(url). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - End() - // 错误处理 - if len(errs) > 0 { - return nil, fmt.Errorf("请求失败: %v", errs) - } - // 检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", resp.Status) - } - responseStruct := ResponseStruct{} - err = json.Unmarshal([]byte(body), &responseStruct) - if err != nil { - return nil, fmt.Errorf("解析JSON失败: %v", err) - } - var dataItem []DataItem - for i := range responseStruct.Data { - dataItem = append(dataItem, responseStruct.Data[i]) - } - return dataItem, nil -} - -// 获取商品信息通过商品详情链接 -func outGetGoodsMsgByDetailUrl(detailUrl, proxy string) (*BookInfo, error) { - response, err := fetchResponse(detailUrl, proxy) - if err != nil { - return nil, err - } - body, err := io.ReadAll(response.Body) - if err != nil { - return nil, err - } - // 解析HTML文档 - document, err := goquery.NewDocumentFromReader(strings.NewReader(string(body))) - if err != nil { - return nil, err - } - // 获取商品信息的快递费 - fee, err := getBookDetailShippingFee(detailUrl, proxy) - if err != nil { - return nil, err - } - book := BookInfo{} - //书名 - book.BookName = strings.TrimSpace(document.Find("h1.title").Text()) - //作者等信息 - topDiv := document.Find("div.keywords-define.keywords-define-1000.clear-fix") - if topDiv.Length() > 0 { - topDiv.Find("li").Each(func(i int, li *goquery.Selection) { - titleSpan := li.Find("span.keywords-define-title") - contentSpan := li.Find("span.keywords-define-txt") - if contentSpan.Length() == 0 { - fmt.Printf("未找到指定的contentSpan信息") - } - titleText := strings.TrimSpace(titleSpan.Text()) - contentText := strings.TrimSpace(contentSpan.Text()) - titleText = strings.TrimSpace(titleText) - if strings.Contains(titleText, "作者") { - book.Author = cleanString(contentText) - } - if strings.Contains(titleText, "出版社") { - book.Publisher = contentText - } - if strings.Contains(titleText, "出版人") { - book.Publisher = contentText - } - if strings.Contains(titleText, "ISBN") { - book.ISBN = contentText - } - if strings.Contains(titleText, "出版时间") { - book.PublicationTime = validateDateFormat(contentText) - } - if strings.Contains(titleText, "版次") { - book.Edition = contentText - } - if strings.Contains(titleText, "装帧") { - book.BindingLayout = contentText - } - if strings.Contains(titleText, "开本") { - book.Format = contentText - } - if strings.Contains(titleText, "页数") { - book.Pages = contentText - } - if strings.Contains(titleText, "字数") { - book.Wordage = contentText - } - if strings.Contains(titleText, "纸张") { - book.Paper = contentText - } - if strings.Contains(titleText, "年代") { - book.Era = contentText - } - if strings.Contains(titleText, "刻印方式") { - book.EngravingMethod = contentText - } - if strings.Contains(titleText, "尺寸") { - book.Dimensions = contentText - } - if strings.Contains(titleText, "册数") { - book.VolumeNumber = contentText - } - }) - } else { - botDiv := document.Find("div.detail-lists.clear-fix") - botDiv.Find("li").Each(func(i int, li *goquery.Selection) { - spanText := strings.TrimSpace(li.Find("span").Text()) - spanText = strings.TrimSpace(spanText) - if strings.Contains(li.Text(), "作者") { - book.Author = cleanString(li.Text()) - book.Author = strings.ReplaceAll(book.Author, "作者:", "") - book.Author = strings.ReplaceAll(book.Author, "著", "") - } - if strings.Contains(li.Text(), "出版社") { - book.Publisher = spanText - } - if strings.Contains(li.Text(), "出版时间") { - book.PublicationTime = validateDateFormat(spanText) - } - if strings.Contains(li.Text(), "ISBN") { - book.ISBN = spanText - } - if strings.Contains(li.Text(), "装帧") { - book.BindingLayout = spanText - } - if strings.Contains(li.Text(), "开本") { - book.Format = spanText - } - if strings.Contains(li.Text(), "纸张") { - book.Paper = spanText - } - if strings.Contains(li.Text(), "版次") { - book.Edition = spanText - } - if strings.Contains(li.Text(), "页数") { - book.Pages = spanText - } - if strings.Contains(li.Text(), "字数") { - book.Wordage = spanText - } - }) - } - - //图片 - var imgUrls []string - tpUl := document.Find("ul.lg-list") - tpUl.Find("img").Each(func(i int, s *goquery.Selection) { - dataImgUrl, exists := s.Attr("data-imgurl") - if exists && dataImgUrl != "" { - imgUrls = append(imgUrls, dataImgUrl) - } - }) - book.BookPicS = strings.Join(imgUrls, ",") - // 售价 - price := document.Find("i.now-price-text").Text() - priceN := regexp.MustCompile(`(\d+\.?\d*)`) - if match := priceN.FindStringSubmatch(price); len(match) > 0 { - book.SellingPrice = match[1] - } - // 定价价 - fixPrice := document.Find("span.origin-price-text.clearfix").Text() - fixPriceN := regexp.MustCompile(`(\d+\.?\d*)`) - if match := fixPriceN.FindStringSubmatch(fixPrice); len(match) > 0 { - book.FixPrice = match[1] - } - // 品相 - text := document.Find("span.quality-text-cot.clearfix i").Text() - book.Condition = strings.TrimSpace(text) - // 快递费 - book.ExpressDeliveryFee = fee - return &book, nil -} - -// 获取商品信息的快递费(定位到河南) -func getBookDetailShippingFee(url, proxy string) (string, error) { - compile := regexp.MustCompile(`kongfz\.com/(\d+)/(\d+)`) - match := compile.FindStringSubmatch(url) - var shippingFee string - var shopId int - var itemId int - if len(match) == 3 { - firstNum, err := strconv.Atoi(match[1]) // 店铺ID - if err != nil { - return "", fmt.Errorf("无效的店铺编码: %s", match[1]) - } - shopId = firstNum - secondNum, err := strconv.Atoi(match[2]) // 图书ID - if err != nil { - return "", fmt.Errorf("无效的图书编码: %s", match[2]) - } - itemId = secondNum - } - shippingFeeUrl := fmt.Sprintf("https://book.kongfz.com/store-web/pc/v1/mould/calculateFee?area=13003000000&itemId=%d&shopId=%d", itemId, shopId) - // 创建HTTP客户端 - request := gorequest.New() - // 设置代理（如果有提供代理URL） - if proxy != "" { - request.Proxy(proxy) - } - // 设置超时和其他配置 - request.Timeout(30 * time.Second) - // 发送请求 - resp, body, errs := request.Get(shippingFeeUrl). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - End() - // 错误处理 - if len(errs) > 0 { - return "", fmt.Errorf("请求失败: %v", errs) - } - // 检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("HTTP错误: %s", resp.Status) - } - calculateFee := struct { - ErrCode int `json:"errCode"` - ErrMessage string `json:"errMessage"` - Result struct { - FeeList []struct { - FreeCondition string `json:"freeCondition"` - ShippingID string `json:"shippingId"` - ShippingName string `json:"shippingName"` - ShippingValue string `json:"shippingValue"` - } `json:"feeList"` - FeeText string `json:"feeText"` - } `json:"result"` - Status bool `json:"status"` - }{} - - err := json.Unmarshal([]byte(body), &calculateFee) - if err != nil { - return "", fmt.Errorf("解析JSON失败: %v", err) - } - for _, fee := range calculateFee.Result.FeeList { - shippingFee = fee.ShippingValue - } - return shippingFee, nil -} - -// 公用发送请求方法 -func fetchResponse(url, proxy string) (*http.Response, error) { - log.Printf("调用的URL: %s", url) - maxRetries := cf.App.MaxRetryTimes - var detailsResp *http.Response - var errors []error - for attempt := 0; attempt < maxRetries; attempt++ { - if attempt > 0 { - log.Printf("第 %d 次重试请求...", attempt) - // 重试前等待，使用指数退避策略 - waitTime := time.Duration(attempt*attempt) * cf.App.RateLimitDelay // 平方退避 - log.Printf("等待 %v 后重试", waitTime) - time.Sleep(waitTime) - } - if proxy != "" { - detailsResp, _, errors = gorequest.New(). - Get(url). - Proxy(proxy). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"). - Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Timeout(120 * time.Second). - End() - } - if proxy == "" { - detailsResp, _, errors = gorequest.New(). - Get(url). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"). - Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Timeout(120 * time.Second). - End() - } - // 检查是否需要重试 - shouldRetry := false - if len(errors) > 0 { - shouldRetry = true - log.Printf("请求失败 (尝试 %d/%d): %v", attempt+1, maxRetries+1, errors) - } else if detailsResp == nil { - shouldRetry = true - log.Printf("响应为空 (尝试 %d/%d)", attempt+1, maxRetries+1) - } else if detailsResp.StatusCode != http.StatusOK { - // 只对服务器错误进行重试，不对客户端错误重试 - if detailsResp.StatusCode >= 500 { - shouldRetry = true - } - log.Printf("HTTP状态码: %d,HTTP请求失败: %s (尝试 %d/%d)", detailsResp.StatusCode, detailsResp.Body, attempt+1, maxRetries+1) - } - // 如果不需要重试，跳出循环 - if !shouldRetry { - break - } - // 如果是最后一次尝试，不继续重试 - if attempt == maxRetries { - break - } - // 关闭响应体（如果存在） - if detailsResp != nil && detailsResp.Body != nil { - detailsResp.Body.Close() - } - } - // 检测请求是否错误 - if len(errors) > 0 { - var proxyAuthFailed bool - var timeoutError bool - var connectionError bool - for _, e := range errors { - errStr := e.Error() - if strings.Contains(errStr, "Proxy Authentication Required") { - proxyAuthFailed = true - } - if strings.Contains(errStr, "timeout") || strings.Contains(errStr, "i/o timeout") { - timeoutError = true - } - if strings.Contains(errStr, "connection") || strings.Contains(errStr, "connect") { - connectionError = true - } - } - if proxyAuthFailed { - return nil, fmt.Errorf("代理认证失败") - } - if timeoutError { - return nil, fmt.Errorf("请求超时，经过 %d 次尝试，超时网址:%s", maxRetries+1, url) - } - if connectionError { - return nil, fmt.Errorf("网络连接错误，经过 %d 次尝试，错误网址:%s", maxRetries+1, url) - } - return nil, fmt.Errorf("查询请求失败，经过 %d 次尝试: %v，失败网址:%s", maxRetries+1, errors, url) - } - if detailsResp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", detailsResp.Status) - } - return detailsResp, nil -} - -// 获取销量榜商品列表(带有Out的都非官放标准接口) -func outGetTopGoodsListMsg(catId int, proxy string) ([]string, error) { - // 构建请求URL - url := fmt.Sprintf("https://item.kongfz.com/api/pc/getSellWellListDetail?page=1&pageSize=100&timeRank=2&catId=%d", catId) - // 创建HTTP客户端 - request := gorequest.New() - // 设置代理（如果有提供代理URL） - if proxy != "" { - request.Proxy(proxy) - } - // 设置超时和其他配置 - request.Timeout(30 * time.Second) - // 发送请求 - resp, body, errs := request.Get(url). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). - Set("Accept", "application/json, text/plain, */*"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Set("Referer", "https://item.kongfz.com/"). - End() - // 错误处理 - if len(errs) > 0 { - return nil, fmt.Errorf("请求失败: %v", errs) - } - // 检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", resp.Status) - } - var bookDetailResponse BookDetailResponse - err := json.Unmarshal([]byte(body), &bookDetailResponse) - if err != nil { - return nil, fmt.Errorf("解析JSON失败: %v", err) - } - if !bookDetailResponse.Status { - return nil, fmt.Errorf("API返回错误: %s (代码: %d)", bookDetailResponse.ErrMessage, bookDetailResponse.ErrCode) - } - - var isbnList []string - for _, item := range bookDetailResponse.Result.Data { - if item.Isbn != "" { - isbnList = append(isbnList, item.Isbn) - } - } - isbnList = removeDuplicateISBNs(isbnList) - return isbnList, nil -} - -// 去除重复的ISBN -func removeDuplicateISBNs(isbns []string) []string { - seen := make(map[string]bool) - var result []string - for _, isbn := range isbns { - if !seen[isbn] { - seen[isbn] = true - result = append(result, isbn) - } +// 接受回调函数作为参数的函数 +func process(nums []int, callback Callback) []int { + result := make([]int, len(nums)) + for i, v := range nums { + result[i] = callback(v) // 调用回调函数 } return result } -// 获取Document文档(带重试机制) -func fetchDocument(fetchMode, proxyType, username, password, machineCode, url string) (*goquery.Document, error) { - log.Printf("调用的URL: %s", url) - maxRetries := cf.App.MaxRetryTimes - var detailsResp *http.Response - var errors []error - var detailsBody []byte - for attempt := 0; attempt < maxRetries; attempt++ { - if attempt > 0 { - log.Printf("第 %d 次重试请求...", attempt) - // 重试前等待，使用指数退避策略 - waitTime := time.Duration(attempt*attempt) * cf.App.RateLimitDelay // 平方退避 - log.Printf("等待 %v 后重试", waitTime) - time.Sleep(waitTime) +// 模拟异步操作 +func fetchData(callback func(string)) { + go func() { + time.Sleep(2 * time.Second) + data := "从服务器获取的数据" + callback(data) // 操作完成后调用回调 + }() +} + +// 事件处理器 +type EventHandler struct { + callbacks map[string][]func(interface{}) +} + +func NewEventHandler() *EventHandler { + return &EventHandler{ + callbacks: make(map[string][]func(interface{})), + } +} + +// 注册事件回调 +func (h *EventHandler) On(event string, callback func(interface{})) { + h.callbacks[event] = append(h.callbacks[event], callback) +} + +// 触发事件 +func (h *EventHandler) Emit(event string, data interface{}) { + if callbacks, exists := h.callbacks[event]; exists { + for _, callback := range callbacks { + callback(data) // 执行所有注册的回调 } - if fetchMode == NotProxy { - detailsResp, _, errors = gorequest.New(). - Get(url). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"). - Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Timeout(120 * time.Second). - End() - } else if fetchMode == UseProxy { - detailsReq := XxProxyRequest(proxyType, username, password, machineCode). - Get(url). - Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"). - Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"). - Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). - Set("Connection", "close") // 每次请求关闭连接 - detailsResp, _, errors = detailsReq.End() + } +} + +func main() { + //// 定义回调函数 + //square := func(x int) int { + // return x * x + //} + //nums := []int{1, 2, 3, 4, 5} + //result := process(nums, square) + //fmt.Println(result) // [1 4 9 16 25] + + //fmt.Println("开始请求数据...") + // + //fetchData(func(data string) { + // fmt.Println("收到数据:", data) + //}) + // + //time.Sleep(3 * time.Second) // 等待goroutine完成 + + //handler := NewEventHandler() + // + //// 注册点击事件回调 + //handler.On("click", func(data interface{}) { + // fmt.Println("点击事件1:", data) + //}) + // + //handler.On("click", func(data interface{}) { + // fmt.Println("点击事件2:", data) + //}) + // + //// 触发事件 + //handler.Emit("click", "按钮被点击") + + // 协程返回值 + //resultChan := make(chan int, 3) + // + //// 启动多个协程 + //for i := 1; i <= 3; i++ { + // go worker(i, resultChan) + //} + // + //// 收集结果 + //for i := 1; i <= 3; i++ { + // result := <-resultChan + // fmt.Printf("Received result: %d\n", result) + //} + + // 带错误处理的返回值 + values := []int{1, -2, 3, 4} + resultChan := make(chan Result, len(values)) + + for _, v := range values { + go asyncCalculate(v, resultChan) + } + + for range values { + result := <-resultChan + if result.Err != nil { + fmt.Printf("Error: %v\n", result.Err) } else { - errors = append(errors, fmt.Errorf("请求失败，选择模式错误！")) - } - // 检查是否需要重试 - shouldRetry := false - if len(errors) > 0 { - shouldRetry = true - log.Printf("请求失败 (尝试 %d/%d): %v", attempt+1, maxRetries+1, errors) - } else if detailsResp == nil { - shouldRetry = true - log.Printf("响应为空 (尝试 %d/%d)", attempt+1, maxRetries+1) - } else if detailsResp.StatusCode != http.StatusOK { - // 只对服务器错误进行重试，不对客户端错误重试 - if detailsResp.StatusCode >= 500 { - shouldRetry = true - } - log.Printf("HTTP状态码: %d,HTTP请求失败: %s (尝试 %d/%d)", detailsResp.StatusCode, detailsResp.Body, attempt+1, maxRetries+1) - } - // 如果不需要重试，跳出循环 - if !shouldRetry { - break - } - // 如果是最后一次尝试，不继续重试 - if attempt == maxRetries { - break - } - // 关闭响应体（如果存在） - if detailsResp != nil && detailsResp.Body != nil { - detailsResp.Body.Close() + fmt.Printf("Result: %d\n", result.Value) } } - // 检测请求是否错误 - if len(errors) > 0 { - var proxyAuthFailed bool - var timeoutError bool - var connectionError bool - for _, e := range errors { - errStr := e.Error() - if strings.Contains(errStr, "Proxy Authentication Required") { - proxyAuthFailed = true - } - if strings.Contains(errStr, "timeout") || strings.Contains(errStr, "i/o timeout") { - timeoutError = true - } - if strings.Contains(errStr, "connection") || strings.Contains(errStr, "connect") { - connectionError = true - } - } - if proxyAuthFailed { - return nil, fmt.Errorf("代理认证失败") - } - if timeoutError { - return nil, fmt.Errorf("请求超时，经过 %d 次尝试，超时网址:%s", maxRetries+1, url) - } - if connectionError { - return nil, fmt.Errorf("网络连接错误，经过 %d 次尝试，错误网址:%s", maxRetries+1, url) - } - return nil, fmt.Errorf("查询请求失败，经过 %d 次尝试: %v，失败网址:%s", maxRetries+1, errors, url) - } - if detailsResp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP错误: %s", detailsResp.Status) - } - - detailsBody, err := io.ReadAll(detailsResp.Body) - if err != nil { - return nil, err - } - - return goquery.NewDocumentFromReader(strings.NewReader(string(detailsBody))) } -// 合并基本信息和详情 -func mergeBookDetails(bookItems []BookItem, detailResults map[string]*DetailResult) []BookInfo { - var books []BookInfo - - for _, item := range bookItems { - book := item.Book - - // 如果有详情且获取成功，补充详细信息 - if item.HasDetail && item.DetailURL != "" { - if result, exists := detailResults[item.DetailURL]; exists && result.Error == nil && result.Doc != nil { - detailsDiv := result.Doc.Find("div.detail-lists.clear-fix") - if detailsDiv.Length() > 0 { - selection := detailsDiv.Find("li") - book.Author = extractNormalDetails(selection, "作者") - book.Publisher = extractNormalDetails(selection, "出版社") - detailsTime := extractNormalDetails(selection, "出版时间") - if detailsTime != "" { - formats := []string{"2006-01", "2006-01-02", "2006"} - for _, format := range formats { - if t, err := time.Parse(format, detailsTime); err == nil { - book.PublicationTime = t.Unix() - break - } - } - } - book.ISBN = extractNormalDetails(selection, "ISBN") - book.Edition = extractNormalDetails(selection, "版次") - book.FixPrice = extractNormalDetails(selection, "定价") - book.BindingLayout = extractNormalDetails(selection, "装帧") - book.Format = extractNormalDetails(selection, "开本") - book.Pages = extractNormalDetails(selection, "页数") - book.Wordage = extractNormalDetails(selection, "字数") - } - } - } - - books = append(books, book) - } - return books +type Result struct { + Value int + Err error } -// 根据url获取单个图书详情信息 -func getUrlBookDetails(fetchMode, proxyType, username, password, machineCode, url string) (books []BookInfo, err error) { - document, err := fetchDocument(fetchMode, proxyType, username, password, machineCode, url) - if err != nil { - return nil, err +func calculate(x int) (int, error) { + if x < 0 { + return 0, errors.New("negative value not allowed") } - - fee, err := FetchBookDetailsShippingFee(url) - if err != nil { - return nil, err - } - book := BookInfo{} - //书名 - book.BookName = strings.TrimSpace(document.Find("h1.title").Text()) - - //作者等信息 - topDiv := document.Find("div.keywords-define.keywords-define-1000.clear-fix") - if topDiv.Length() > 0 { - topDiv.Find("li").Each(func(i int, li *goquery.Selection) { - titleSpan := li.Find("span.keywords-define-title") - contentSpan := li.Find("span.keywords-define-txt") - if contentSpan.Length() == 0 { - fmt.Printf("未找到指定的contentSpan信息") - } - titleText := strings.TrimSpace(titleSpan.Text()) - contentText := strings.TrimSpace(contentSpan.Text()) - titleText = strings.TrimSpace(titleText) - if strings.Contains(titleText, "作者") { - book.Author = cleanString(contentText) - } - if strings.Contains(titleText, "出版社") { - book.Publisher = contentText - } - if strings.Contains(titleText, "出版人") { - book.Publisher = contentText - } - if strings.Contains(titleText, "ISBN") { - book.ISBN = contentText - } - if strings.Contains(titleText, "出版时间") { - book.PublicationTime = validateDateFormat(contentText) - } - if strings.Contains(titleText, "版次") { - book.Edition = contentText - } - if strings.Contains(titleText, "装帧") { - book.BindingLayout = contentText - } - if strings.Contains(titleText, "开本") { - book.Format = contentText - } - if strings.Contains(titleText, "页数") { - book.Pages = contentText - } - if strings.Contains(titleText, "字数") { - book.Wordage = contentText - } - if strings.Contains(titleText, "纸张") { - book.Paper = contentText - } - if strings.Contains(titleText, "年代") { - book.Era = contentText - } - if strings.Contains(titleText, "刻印方式") { - book.EngravingMethod = contentText - } - if strings.Contains(titleText, "尺寸") { - book.Dimensions = contentText - } - if strings.Contains(titleText, "册数") { - book.VolumeNumber = contentText - } - }) - } else { - botDiv := document.Find("div.detail-lists.clear-fix") - botDiv.Find("li").Each(func(i int, li *goquery.Selection) { - spanText := strings.TrimSpace(li.Find("span").Text()) - spanText = strings.TrimSpace(spanText) - if strings.Contains(li.Text(), "作者") { - book.Author = cleanString(li.Text()) - book.Author = strings.ReplaceAll(book.Author, "作者:", "") - book.Author = strings.ReplaceAll(book.Author, "著", "") - } - if strings.Contains(li.Text(), "出版社") { - book.Publisher = spanText - } - if strings.Contains(li.Text(), "出版时间") { - book.PublicationTime = validateDateFormat(spanText) - } - if strings.Contains(li.Text(), "ISBN") { - book.ISBN = spanText - } - if strings.Contains(li.Text(), "装帧") { - book.BindingLayout = spanText - } - if strings.Contains(li.Text(), "开本") { - book.Format = spanText - } - if strings.Contains(li.Text(), "纸张") { - book.Paper = spanText - } - if strings.Contains(li.Text(), "版次") { - book.Edition = spanText - } - if strings.Contains(li.Text(), "页数") { - book.Pages = spanText - } - if strings.Contains(li.Text(), "字数") { - book.Wordage = spanText - } - }) - } - - //图片 - var imgUrls []string - tpUl := document.Find("ul.lg-list") - tpUl.Find("img").Each(func(i int, s *goquery.Selection) { - dataImgUrl, exists := s.Attr("data-imgurl") - if exists && dataImgUrl != "" { - imgUrls = append(imgUrls, dataImgUrl) - } - }) - book.BookPicS = strings.Join(imgUrls, ",") - // 售价 - price := document.Find("i.now-price-text").Text() - priceN := regexp.MustCompile(`(\d+\.?\d*)`) - if match := priceN.FindStringSubmatch(price); len(match) > 0 { - book.SellingPrice = match[1] - } - // 定价价 - fixPrice := document.Find("span.origin-price-text.clearfix").Text() - fixPriceN := regexp.MustCompile(`(\d+\.?\d*)`) - if match := fixPriceN.FindStringSubmatch(fixPrice); len(match) > 0 { - book.FixPrice = match[1] - } - // 品相 - text := document.Find("span.quality-text-cot.clearfix i").Text() - book.Condition = strings.TrimSpace(text) - // 快递费 - if fee != "" { - if fee == "包邮" { - book.ExpressDeliveryFee = "0" - } else { - book.ExpressDeliveryFee = fee - } - } - books = append(books, book) - return books, nil + return x * 2, nil } -// 获取店铺页面的所有快递费用 -func FetchProductInfoWithChromedp(url string) (*ProductResponse, error) { - // 创建上下文 - opts := append(chromedp.DefaultExecAllocatorOptions[:], - chromedp.Flag("headless", true), - chromedp.Flag("disable-gpu", true), - chromedp.Flag("no-sandbox", true), - chromedp.Flag("disable-dev-shm-usage", true), - chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"), - // 禁用图片加载，加快速度 - chromedp.Flag("blink-settings", "imagesEnabled=false"), - ) - // 启动浏览器 - allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) - defer cancel() - // 创建浏览器上下文 - ctx, cancel := chromedp.NewContext(allocCtx) - defer cancel() - // 设置超时 - ctx, cancel = context.WithTimeout(ctx, 60*time.Second) - defer cancel() - log.Printf("[CHROMEDP] 开始导航到URL: %s", url) - // 第一步：先导航到页面并等待完全加载 - err := chromedp.Run(ctx, - // 导航到目标页面 - chromedp.Navigate(url), - - // 等待页面初步加载 - chromedp.WaitReady("body", chromedp.ByQuery), - chromedp.Sleep(3*time.Second), - - // 第二步：刷新页面，重新加载动态内容 - chromedp.Reload(), - - // 等待刷新后页面加载 - chromedp.WaitReady("body", chromedp.ByQuery), - chromedp.Sleep(5*time.Second), - - // 等待关键元素出现 - chromedp.WaitVisible(".item.clearfix", chromedp.ByQuery), - - // 滚动页面以触发动态加载 - chromedp.Evaluate(`window.scrollTo(0, document.body.scrollHeight / 3)`, nil), - chromedp.Sleep(2*time.Second), - chromedp.Evaluate(`window.scrollTo(0, document.body.scrollHeight / 2)`, nil), - chromedp.Sleep(2*time.Second), - chromedp.Evaluate(`window.scrollTo(0, document.body.scrollHeight)`, nil), - chromedp.Sleep(3*time.Second), - chromedp.Evaluate(`window.scrollTo(0, 0)`, nil), - chromedp.Sleep(1*time.Second), - ) - if err != nil { - log.Printf("[CHROMEDP] 页面导航失败: %v", err) - return nil, fmt.Errorf("页面导航失败: %v", err) - } - log.Printf("[CHROMEDP] 页面导航完成，开始提取数据") - var evalResult map[string]interface{} - // 执行任务 - err = chromedp.Run(ctx, - // 导航到目标页面 - chromedp.Navigate(url), - - // 执行 JavaScript 来提取商品信息 - chromedp.Evaluate(`(function() { - console.log("开始提取商品信息..."); - try { - let productData = []; - // 查找所有商品项 - 针对孔夫子旧书网的特定选择器 - const itemSelectors = [ - '.item.clearfix', - '.list-item', - '.product-item', - '.goods-item', - '[class*="item"]' - ]; - let items = []; - for (let selector of itemSelectors) { - const found = document.querySelectorAll(selector); - if (found.length > 0) { - items = found; - console.log("使用选择器:", selector, "找到项目数:", items.length); - break; - } - } - - if (items.length === 0) { - // 如果没有找到特定选择器，尝试查找任何看起来像商品的项目 - items = document.querySelectorAll('div[class*="item"], li[class*="item"]'); - console.log("使用通用选择器找到项目数:", items.length); - } - console.log("总共找到商品项:", items.length); - // 遍历每个商品项 - items.forEach((item, index) => { - // 提取 itemid - let itemId = ''; - - // 方法1: 从元素属性获取 - if (item.hasAttribute('itemid')) { - itemId = item.getAttribute('itemid'); - } - - // 方法2: 从数据属性获取 - if (!itemId && item.hasAttribute('data-itemid')) { - itemId = item.getAttribute('data-itemid'); - } - - // 方法3: 从ID属性获取 - if (!itemId && item.hasAttribute('id')) { - const id = item.getAttribute('id'); - if (id.includes('item') || id.includes('product')) { - itemId = id; - } - } - - // 方法4: 从链接中提取itemid - if (!itemId) { - const links = item.querySelectorAll('a[href*="item"], a[href*="product"]'); - for (let link of links) { - const href = link.getAttribute('href'); - if (href) { - const itemMatch = href.match(/(?:item|product)[_-]?(\d+)/i); - if (itemMatch) { - itemId = itemMatch[1]; - break; - } - } - } - } - - // 方法5: 从子元素中查找itemid - if (!itemId) { - const childWithItemId = item.querySelector('[itemid], [data-itemid]'); - if (childWithItemId) { - if (childWithItemId.hasAttribute('itemid')) { - itemId = childWithItemId.getAttribute('itemid'); - } else if (childWithItemId.hasAttribute('data-itemid')) { - itemId = childWithItemId.getAttribute('data-itemid'); - } - } - } - - // 提取书名 - let bookName = ''; - const titleSelectors = [ - '.title a', - '.book-title', - '.item-title', - '.name a', - 'h3 a', - 'h4 a', - '.link', - 'a[title]' - ]; - - for (let selector of titleSelectors) { - const titleEl = item.querySelector(selector); - if (titleEl) { - bookName = titleEl.textContent.trim(); - if (bookName && bookName.length > 1) { - break; - } - } - } - - // 如果没找到，尝试在item内查找任何文本作为书名 - if (!bookName) { - const textContent = item.textContent; - // 尝试提取看起来像书名的文本（较长的文本块） - const lines = textContent.split('\n').map(line => line.trim()).filter(line => line.length > 5); - if (lines.length > 0) { - bookName = lines[0]; - } - } - - // 提取价格 - let price = ''; - const priceSelectors = [ - '.price span', - '.price .bold', - '.current-price', - '.sell-price', - '.cost', - '.money', - '[class*="price"]', - 'strong' - ]; - - for (let selector of priceSelectors) { - const priceEl = item.querySelector(selector); - if (priceEl) { - let priceText = priceEl.textContent.trim(); - // 清理价格文本，保留数字和小数点 - priceText = priceText.replace(/[^\d\.]/g, ''); - if (priceText && !isNaN(parseFloat(priceText))) { - price = '¥' + priceText; - break; - } - } - } - - // 如果没找到价格，尝试在文本中查找价格模式 - if (!price) { - const itemText = item.textContent; - const priceMatch = itemText.match(/[¥￥]?\s*(\d+\.?\d*)/); - if (priceMatch) { - price = '¥' + priceMatch[1]; - } - } - - // 提取快递费用 - let shippingFee = ''; - const shippingSelectors = [ - '.ship-fee', - '.shipping-fee', - '.express-fee', - '.freight', - '.postage', - '[class*="fee"]', - '[class*="快递"]', - '[class*="运费"]' - ]; - - for (let selector of shippingSelectors) { - const shippingEl = item.querySelector(selector); - if (shippingEl) { - shippingFee = shippingEl.textContent.trim(); - if (shippingFee) break; - } - } - - // 如果没找到运费元素，尝试在文本中查找运费关键词 - if (!shippingFee) { - const itemText = item.textContent; - const feeMatches = itemText.match(/(运费[：:]\s*[^\s\n]+)|(快递[：:]\s*[^\s\n]+)|(邮费[：:]\s*[^\s\n]+)/); - if (feeMatches) { - shippingFee = feeMatches[0]; - } else if (itemText.includes('包邮') || itemText.includes('免运费')) { - shippingFee = '包邮'; - } else { - shippingFee = '运费待确认'; - } - } - - // 清理书名（移除过长的文本） - if (bookName && bookName.length > 100) { - bookName = bookName.substring(0, 100) + '...'; - } - - // 如果还没有itemid，生成一个基于索引的ID - if (!itemId) { - itemId = 'item_' + (index + 1); - } - - // 添加到数据中 - productData.push({ - itemId: itemId, - bookName: bookName || '商品${index + 1}', - price: price || '价格待确认', - shippingFee: shippingFee - }); - }); - - // 过滤掉明显无效的数据 - productData = productData.filter(item => - (item.bookName !== '商品1' || item.price !== '价格待确认') && - item.bookName && item.bookName.length > 0 - ); - - console.log("处理后商品数量:", productData.length); - if (productData.length > 0) { - console.log("前3个商品示例:", productData.slice(0, 3)); - } - - return { - success: true, - message: "成功提取商品信息", - data: productData - }; - - } catch(error) { - console.error("提取商品信息时出错:", error); - return { - success: false, - message: "提取商品信息时出错: " + error.message, - data: [] - }; - } - })()`, &evalResult), - ) - - if err != nil { - return nil, fmt.Errorf("chromedp执行失败: %v", err) - } - - // 处理 JavaScript 执行结果 - response := &ProductResponse{} - - // 转换结果数据 - if success, ok := evalResult["success"].(bool); ok { - response.Success = success - } - if message, ok := evalResult["message"].(string); ok { - response.Message = message - } - if data, ok := evalResult["data"].([]interface{}); ok { - for _, item := range data { - if productMap, ok := item.(map[string]interface{}); ok { - product := ProductInfo{} - if itemId, ok := productMap["itemId"].(string); ok { - product.ItemID = itemId - } - if bookName, ok := productMap["bookName"].(string); ok { - product.BookName = bookName - } - if price, ok := productMap["price"].(string); ok { - product.Price = price - } - if shippingFee, ok := productMap["shippingFee"].(string); ok { - info := extractShippingInfo(shippingFee) - product.ShippingFee = info - } - response.Data = append(response.Data, product) - } - } - } - - return response, nil +func asyncCalculate(x int, resultChan chan<- Result) { + value, err := calculate(x) + resultChan <- Result{Value: value, Err: err} } -// 获取详情页的快递费用 -func FetchBookDetailsShippingFee(url string) (string, error) { - // 创建上下文 - opts := append(chromedp.DefaultExecAllocatorOptions[:], - chromedp.Flag("headless", true), - chromedp.Flag("disable-gpu", true), - chromedp.Flag("no-sandbox", true), - chromedp.Flag("disable-dev-shm-usage", true), - chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"), - // 禁用图片加载，加快速度 - chromedp.Flag("blink-settings", "imagesEnabled=false"), - ) - // 启动浏览器 - allocCtx, cancel := chromedp.NewExecAllocator(context.Background(), opts...) - defer cancel() - // 创建浏览器上下文 - ctx, cancel := chromedp.NewContext(allocCtx) - defer cancel() - // 设置超时 - ctx, cancel = context.WithTimeout(ctx, 60*time.Second) - defer cancel() - log.Printf("[CHROMEDP] 开始导航到URL: %s", url) - var result string - - // 增强的 JavaScript 脚本 - enhancedJSScript := ` - (function() { - // 查找包含快递费用的元素 - const elements = document.querySelectorAll('*'); - let shippingFee = null; - - elements.forEach(element => { - if (element.textContent.includes('快递￥')) { - const match = element.textContent.match(/快递￥(\d+\.\d{2})/); - if (match) { - shippingFee = match[1]; - console.log('找到快递费用: ￥' + shippingFee); - } - } - // 匹配格式2: 快递：包邮 - else if (element.textContent.includes('快递：包邮') || element.textContent.includes('快递:包邮')) { - shippingFee = '包邮'; - console.log('找到快递费用: 包邮'); - } - // 匹配格式3: 快递 包邮 (包含空格变体) - else if (element.textContent.match(/快递[：:\s]包邮/)) { - shippingFee = '包邮'; - console.log('找到快递费用: 包邮'); - } - }); - - if (!shippingFee) { - // 如果没找到，尝试搜索整个页面 - const pageText = document.body.innerText; - const match = pageText.match(/快递￥(\d+\.\d{2})/); - if (match) { - shippingFee = match[1]; - console.log('找到快递费用: ￥' + shippingFee); - } - // 尝试匹配包邮 - else if (pageText.match(/快递[：:\s]包邮/)) { - shippingFee = '包邮'; - console.log('找到快递费用: 包邮'); - } - // 尝试其他包邮表述 - else if (pageText.includes('包邮') && pageText.includes('快递')) { - shippingFee = '包邮'; - console.log('找到快递费用: 包邮'); - } - else { - console.log('未找到快递费用信息'); - } - } - return shippingFee; - })();` - // 先导航到页面并等待完全加载 - err := chromedp.Run(ctx, - // 导航到目标页面 - chromedp.Navigate(url), - // 刷新页面，重新加载动态内容 - chromedp.Reload(), - // 等待刷新后页面加载 - chromedp.WaitReady("body", chromedp.ByQuery), - chromedp.Sleep(8*time.Second), - // 执行 JavaScript 来提取商品信息 - chromedp.Evaluate(enhancedJSScript, &result), - ) - if err != nil { - log.Printf("[CHROMEDP] 页面导航失败: %v", err) - return "", fmt.Errorf("页面导航失败: %v", err) - } - log.Printf("[CHROMEDP] 页面导航完成，开始提取数据") - - if err != nil { - return "", fmt.Errorf("chromedp执行失败: %v", err) - } - return result, nil +func worker(id int, resultChan chan<- int) { + fmt.Printf("Worker %d starting\n", id) + time.Sleep(time.Second) + fmt.Printf("Worker %d done\n", id) + resultChan <- id * 10 // 发送结果到通道 } -// 获取孔网实拍图(查询商品列表) -func getKFZSPTImageURL(proxyType, username, password, machineCode, isbn string) (*BookInfo, error) { - log.Printf("[DEBUG] ", isbn, " 获取图书ISBN: %s", isbn) - url := fmt.Sprintf("%s?dataType=0&keyword=%s&page=1&size=%d&sortType=7&actionPath=quality,sortType&quality=85~&quaSelect=2&userArea=13003000000", cf.API.ProductSearchURL, isbn, cf.App.Size) - - req := XxProxyRequest(proxyType, username, password, machineCode). - Get(url). - Set("User-Agent", cf.App.DefaultUserAgent). - Set("Accept", "*/*") - resp, body, errs := req.End() - log.Printf("[DEBUG] 获取图书列表 URL: %s", url) - - // 只在有响应内容时才打印，避免日志过长 - if len(body) > 0 && len(body) < 500 { - log.Printf("[DEBUG] 获取图书列表响应: %s", body) - } else if len(body) > 0 { - log.Printf("[DEBUG] 获取图书列表响应长度: %d 字符", len(body)) - } else { - log.Printf("[DEBUG] 获取图书列表响应为空") +func TestAdd(t *testing.T) { + result := Add(2, 3) + expected := 5 + if result != expected { + t.Errorf("Add(2, 3) = %d; want %d", result, expected) } - - // 处理请求错误 - if len(errs) > 0 { - // 检查是否是代理认证失败 - var proxyAuthFailed bool - for _, e := range errs { - if strings.Contains(e.Error(), "Proxy Authentication Required") { - proxyAuthFailed = true - break - } - } - if proxyAuthFailed { - return nil, fmt.Errorf("代理认证失败") - } - return nil, fmt.Errorf("查询请求失败: %v", errs) - } - - // 检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - space := strings.TrimSpace(resp.Status) - return nil, fmt.Errorf("HTTP错误: %s", space) - } - - // 解析响应 - var apiResp struct { - Status int `json:"status"` - ErrType string `json:"errType"` - Message string `json:"message"` - SystemTime int64 `json:"systemTime"` - Data struct { - ItemResponse struct { - Total int `json:"total"` - List []struct { - Title string `json:"title"` - ImgUrl string `json:"imgUrl"` - ImgBigUrl string `json:"imgBigUrl"` - ItemId int64 `json:"itemId"` - ShopId int64 `json:"shopId"` - TplRecords []struct { - Key string `json:"key"` - Value string `json:"value"` - } `json:"tplRecords"` - } `json:"list"` - } `json:"itemResponse"` - } `json:"data"` - } - - if err := json.Unmarshal([]byte(body), &apiResp); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - - // 如果找到商品，返回图片URL - if apiResp.Data.ItemResponse.Total > 0 && len(apiResp.Data.ItemResponse.List) > 0 { - // 确定起始索引 - var startIndex int - if cf.App.Size >= apiResp.Data.ItemResponse.Total { - startIndex = apiResp.Data.ItemResponse.Total - 1 - } else { - startIndex = cf.App.Size - 1 - } - - // 从指定索引开始，向前查找有效图片，最多重试3次 - for attempt := 0; attempt < 3; attempt++ { - currentIndex := startIndex - attempt - - // 检查索引是否有效 - if currentIndex < 0 || currentIndex >= len(apiResp.Data.ItemResponse.List) { - log.Printf("[DEBUG] 索引 %d 超出范围，跳过", currentIndex) - continue - } - - item := apiResp.Data.ItemResponse.List[currentIndex] - - // 检查图片URL是否存在 - if item.ImgBigUrl == "" { - log.Printf("[DEBUG] 索引 %d 的图片URL为空，跳过", currentIndex) - continue - } - - info := &BookInfo{} - info.BookName = item.Title - info.BookPicS = item.ImgUrl - info.ItemId = item.ItemId - info.ShopId = item.ShopId - - // 安全地获取TplRecords中的值 - if len(item.TplRecords) > 0 { - info.Author = item.TplRecords[0].Value - } - if len(item.TplRecords) > 1 { - info.Publisher = item.TplRecords[1].Value - } - if len(item.TplRecords) > 2 { - info.PublicationTime = validateDateFormat(item.TplRecords[2].Value) - } - if len(item.TplRecords) > 3 { - info.BindingLayout = item.TplRecords[3].Value - } - - // 成功时重置代理失败计数器 - resetProxyFailCount() - return info, nil - } - // 如果所有尝试都失败了，返回错误 - log.Printf("[WARN] 经过3次尝试，未找到有效图片") - return nil, fmt.Errorf("未找到有效图片，已尝试3次") - } - return nil, nil } -// 获取孔网官图(查询图书条目) -func getKFZGTImageURL(proxyType, username, password, machineCode, isbn string) (*BookInfo, error) { - url := fmt.Sprintf("%s?keyword=%s", cf.API.BookSearchURL, isbn) - - info := &BookInfo{} - - req := XxProxyRequest(proxyType, username, password, machineCode). - Get(url). - Set("User-Agent", cf.App.DefaultUserAgent). - Set("Accept", "*/*") - resp, body, errs := req.End() - log.Printf("[DEBUG] 获取图书条目 URL: %s", url) - // 只在有响应内容时才打印，避免日志过长 - if len(body) > 0 && len(body) < 500 { - log.Printf("[DEBUG] 获取图书条目响应: %s", body) - } else if len(body) > 0 { - log.Printf("[DEBUG] 获取图书条目响应长度: %d 字符", len(body)) - } else { - log.Printf("[DEBUG] 获取图书条目响应为空") - } - - // 处理请求错误 - if len(errs) > 0 { - // 检查是否是代理相关错误 - var isProxyError bool - var errorDetails []string - for _, e := range errs { - errorStr := e.Error() - errorDetails = append(errorDetails, errorStr) - if strings.Contains(errorStr, "Proxy Authentication Required") || - strings.Contains(errorStr, "connectex: A connection attempt failed") || - strings.Contains(errorStr, "connectex: No connection could be made") || - strings.Contains(errorStr, "proxyconnect tcp") || - strings.Contains(errorStr, "timeout") || - strings.Contains(errorStr, "connection refused") { - isProxyError = true - } - } - - log.Printf("[ERROR] 请求错误详情: %v", errorDetails) - - if isProxyError { - // 处理代理失败 - return nil, fmt.Errorf("代理连接失败") - } - - return nil, fmt.Errorf("查询请求失败: %v", errs) - } - - //检查HTTP状态码 - if resp.StatusCode != http.StatusOK { - space := strings.TrimSpace(resp.Status) - return nil, fmt.Errorf("HTTP错误: %s", space) - } - - // 解析响应 - var apiResp struct { - Status int `json:"status"` - ErrType string `json:"errType"` - Message string `json:"message"` - SystemTime int64 `json:"systemTime"` - Data struct { - ItemResponse struct { - Total int `json:"total"` - List []struct { - BookName string `json:"bookName"` - Mid int64 `json:"mid"` - ImgUrlEntity struct { - BigImgUrl string `json:"bigImgUrl"` - } `json:"imgUrlEntity"` - BookShowInfo []string `json:"bookShowInfo"` - } `json:"list"` - } `json:"itemResponse"` - } `json:"data"` - } - - if err := json.Unmarshal([]byte(body), &apiResp); err != nil { - return nil, fmt.Errorf("解析JSON失败: %w", err) - } - - // 如果找到条目，返回图片URL - if apiResp.Data.ItemResponse.Total > 0 && len(apiResp.Data.ItemResponse.List) > 0 { - list := apiResp.Data.ItemResponse.List[0] - info := list.BookShowInfo - - bookItem := &BookInfo{ - BookName: list.BookName, - BookPic: list.ImgUrlEntity.BigImgUrl, - Mid: list.Mid, - } - - // 根据长度安全填充字段 - if len(info) > 0 { - bookItem.Author = info[0] - } - if len(info) > 1 { - bookItem.Publisher = info[1] - } - if len(info) > 2 { - bookItem.PublicationTime = validateDateFormat(info[2]) - } - if len(info) > 3 { - bookItem.BindingLayout = info[3] - } - if len(info) > 4 { - bookItem.FixPrice = info[4] - } else { - log.Printf("[WARN] BookShowInfo 长度不足 (仅 %d 项): %v", len(info), info) - } - - fmt.Println("bookItem: ", bookItem) - return bookItem, nil - } - - return info, nil +func Add(a, b int) int { + return a + b } - -// 替换所有空白字符为空格 -func cleanString(s string) string { - s = strings.ReplaceAll(s, "\n", "") - s = strings.ReplaceAll(s, "\r", "") - s = strings.ReplaceAll(s, "\t", "") - s = strings.ReplaceAll(s, " ", "") - return removeDuplicates(s) -} - -// 字符串去重 -func removeDuplicates(s string) string { - seen := make(map[rune]bool) - var result strings.Builder - for _, r := range s { - if !seen[r] { - seen[r] = true - result.WriteRune(r) - } - } - return result.String() -} - -// 检测快递费用格式 -func extractShippingInfo(shippingFee string) string { - // 匹配 "快递￥数字" 格式 - re1 := regexp.MustCompile(`快递￥(\d+\.?\d*)`) - // 匹配 "包邮" 格式 - re2 := regexp.MustCompile(`(包邮)`) - - if matches := re1.FindStringSubmatch(shippingFee); len(matches) > 1 { - return matches[1] // 返回数字部分 - } - - if matches := re2.FindStringSubmatch(shippingFee); len(matches) > 1 { - return "0" // 返回"包邮" - } - - return "" -} - -// 改进字段 -func extractNormalDetails(s *goquery.Selection, fieldName string) string { - var info string - s.Each(func(i int, selection *goquery.Selection) { - if strings.Contains(selection.Text(), fieldName) { - all := strings.ReplaceAll(selection.Text(), " ", "") - all = strings.ReplaceAll(all, "\n", "") - all = strings.ReplaceAll(all, " ", "") - split := strings.SplitN(all, ":", 2) - if len(split) > 1 { - info = split[1] - } else { - return - } - } - }) - return info -} - -// 改进的字段提取函数 -func extractNormalText(s *goquery.Selection, fieldName string) string { - // 在左右两个分区中查找 - selectors := []string{"div.f_left", "div.f_right"} - for _, selector := range selectors { - fieldItem := s.Find(selector + " div.normal-item").FilterFunction(func(i int, sel *goquery.Selection) bool { - title := sel.Find("span.normal-title").Text() - return strings.Contains(title, fieldName) - }) - - if fieldItem.Length() > 0 { - text := fieldItem.Find("span.normal-text").Text() - return strings.TrimSpace(text) - } - } - return "" -} - -// 重置代理失败计数器（成功时调用） -func resetProxyFailCount() { - tailProxyMu.Lock() - defer tailProxyMu.Unlock() - proxyFailCount = 0 -} - -// 验证日期格式 -func validateDateFormat(dateStr string) int64 { - // 去除前后空格 - dateStr = strings.TrimSpace(dateStr) - - // 替换各种分隔符为统一的分隔符"-" - dateStr = regexp.MustCompile(`[/_\\.,\s]+`).ReplaceAllString(dateStr, "-") - - // 处理纯年份格式 (4位数字) - if regexp.MustCompile(`^\d{4}$`).MatchString(dateStr) { - dateStr += "-01-01" - } - - // 处理年月格式 (4位数字-1或2位数字) - if matches := regexp.MustCompile(`^(\d{4})-(\d{1,2})$`).FindStringSubmatch(dateStr); len(matches) == 3 { - year := matches[1] - month := matches[2] - if len(month) == 1 { - month = "0" + month - } - if monthNum, _ := strconv.Atoi(month); monthNum >= 1 && monthNum <= 12 { - dateStr = year + "-" + month + "-01" - } else { - return 0 - } - } - - // 处理年月日格式 (4位数字-1或2位数字-1或2位数字) - if matches := regexp.MustCompile(`^(\d{4})-(\d{1,2})-(\d{1,2})`).FindStringSubmatch(dateStr); len(matches) >= 4 { - year := matches[1] - month := matches[2] - day := matches[3] - - // 标准化月份和日期为两位数 - if len(month) == 1 { - month = "0" + month - } - if len(day) == 1 { - day = "0" + day - } - - dateStr = year + "-" + month + "-" + day - } - - // 加载上海时区 - loc, err := time.LoadLocation("Asia/Shanghai") - if err != nil { - // 如果加载时区失败，使用UTC+8作为后备方案 - loc = time.FixedZone("CST", 8*60*60) - } - - // 尝试解析为标准日期格式，并指定上海时区 - parsedTime, err := time.ParseInLocation("2006-01-02", dateStr, loc) - if err != nil { - return 0 - } - - // 返回秒级时间戳（UTC时间，但解析时已经考虑了时区偏移） - return parsedTime.Unix() -} - -// 代理请求(小象代理) -func XxProxyRequest(proxyType, username, password, machineCode string) *gorequest.SuperAgent { - proxyManagerOnce.Do(func() { - log.Printf("[INFO] 初始化代理管理器，DLL路径: %s", cf.Proxy.ProxyFilePath) - - // 检查DLL文件是否存在 - if _, err := os.Stat(cf.Proxy.ProxyFilePath); os.IsNotExist(err) { - // 尝试在可执行文件目录查找 - exePath, _ := os.Executable() - exeDir := filepath.Dir(exePath) - dllPath := filepath.Join(exeDir, cf.Proxy.ProxyFilePath) - - if _, err := os.Stat(dllPath); err == nil { - cf.Proxy.ProxyFilePath = dllPath - } else { - proxyManagerInitErr = fmt.Errorf("代理DLL文件不存在: %s (也尝试了: %s)", cf.Proxy.ProxyFilePath, dllPath) - return - } - } - - globalProxyManager, proxyManagerInitErr = NewProxyConfigManager(cf.Proxy.ProxyFilePath) - if proxyManagerInitErr != nil { - log.Printf("[ERROR] 代理管理器初始化失败: %v", proxyManagerInitErr) - } else { - log.Printf("[INFO] 代理管理器初始化成功") - } - }) - typeManager, err := globalProxyManager.ProxyTypeManager( - proxyType, - username, - password, - machineCode, - ) - if err != nil { - fmt.Printf("获取代理服务器信息失败: %v", err) - } - return gorequest.New().Proxy(typeManager).Timeout(120*time.Second).Retry(2, 3*time.Second) -} - -// 初始化 -func initializeConfig(config Config) { - // 设置全局配置 - cf = config -} - -// 连接数据库(选品) -func connectDBXP() (*sql.DB, error) { - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local", - cf.Database.Username, cf.Database.Password, cf.Database.Host, cf.Database.Name)) - if err != nil { - return nil, fmt.Errorf("打开数据库连接失败: %v", err) - } - // 设置连接池参数 - db.SetMaxOpenConns(20) // 最大打开连接数 - db.SetMaxIdleConns(10) // 最大空闲连接数 - err = db.Ping() - if err != nil { - return nil, fmt.Errorf("数据库连接测试失败: %v", err) - } - return db, nil -} - -// 从数据库随机获取一个可用账号 -func getRandomAccount() (*AccountCredential, error) { - mu.RLock() - defer mu.RUnlock() - // sql语句 - query := ` - SELECT id, credential_account, credential_password, token - FROM t_credential - WHERE credential_type = 'KWCredential' - AND status = 0 - AND is_del = 0 - ORDER BY RAND() - LIMIT 1` - - db, err := connectDBXP() - if err != nil { - return nil, err - } - var ac AccountCredential - err = db.QueryRow(query).Scan(&ac.ID, &ac.Username, &ac.Password, &ac.Token) - if err != nil { - if err == sql.ErrNoRows { - return nil, fmt.Errorf("没有可用的账号") - } - return nil, fmt.Errorf("查询账号失败: %v", err) - } - return &ac, nil -} - -// 修改账号为不可用 -func updateAccountInvalid(id int64) error { - mu.RLock() - defer mu.RUnlock() - db, err := connectDBXP() - if err != nil { - return err - } - query := `UPDATE t_credential SET is_del = 1 WHERE id = ?` - _, err = db.Exec(query, id) - return fmt.Errorf("更新失败: %v", err) -} - -// 修改账号为需要验证 -func updateAccountNeedVerify(id int64) error { - mu.RLock() - defer mu.RUnlock() - db, err := connectDBXP() - if err != nil { - return err - } - query := `UPDATE t_credential SET status = 1 WHERE id = ?` - _, err = db.Exec(query, id) - return err -} - -// 更新账号token -func updateAccountToken(id int64, token string) error { - mu.RLock() - defer mu.RUnlock() - db, err := connectDBXP() - if err != nil { - return err - } - query := `UPDATE t_credential SET token = ? WHERE id = ?` - _, err = db.Exec(query, token, id) - return err -} - -// =================== C 导出函数 ======================= -// 登录(带有Out的都非官方标准接口) -// -//export OutLogin -func OutLogin(username, password *C.char) *C.char { - goUsername := C.GoString(username) - goPassword := C.GoString(password) - respToken, err := outLogin(goUsername, goPassword) - resp := struct { - Token string `json:"token"` - }{ - Token: respToken, - } - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: resp, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 获取用户信息(带有Out的都非官方标准接口) -// -//export OutGetUserMsg -func OutGetUserMsg(token *C.char) *C.char { - goToken := C.GoString(token) - userInfo, err := outGetUserMsg(goToken) - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: userInfo, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 获取商品模版(带有Out的都非官方标准接口) -// -//export OutGetGoodsTplMsg -func OutGetGoodsTplMsg(token, itemId, proxy *C.char) *C.char { - goToken := C.GoString(token) - goItemId := C.GoString(itemId) - goProxy := C.GoString(proxy) - info, err := outGetGoodsTplMsg(goToken, goItemId, goProxy) - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: info, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 获取商品列表-已登的店铺(带有Out的都非官方标准接口) -// -//export OutGetGoodsListMsgFromSelfShop -func OutGetGoodsListMsgFromSelfShop(token, proxy, itemSn, priceMin, priceMax *C.char, startCreateTime *C.char, - endCreateTime *C.char, requestType *C.char, isItemSnEqual C.int, page C.int, size C.int) *C.char { - goToken := C.GoString(token) - goProxy := C.GoString(proxy) - goItemSn := C.GoString(itemSn) - goPriceMin := C.GoString(priceMin) - goPriceMax := C.GoString(priceMax) - goStartCreateTime := C.GoString(startCreateTime) - goEndCreateTime := C.GoString(endCreateTime) - goRequestType := C.GoString(requestType) - goIsItemSnEqual := int(isItemSnEqual) - goPage := int(page) - goSize := int(size) - info, err := outGetGoodsListMsgFromSelfShop(goToken, goProxy, goItemSn, goPriceMin, goPriceMax, goStartCreateTime, goEndCreateTime, goRequestType, goIsItemSnEqual, goPage, goSize) - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: info, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 删除商品-已登的店铺(带有Out的都非官方标准接口) -// -//export OutDelGoodsFromSelfShop -func OutDelGoodsFromSelfShop(token, proxy, itemId *C.char) *C.char { - goToken := C.GoString(token) - goProxy := C.GoString(proxy) - goItemId := C.GoString(itemId) - info, err := outDelGoodsFromSelfShop(goToken, goProxy, goItemId) - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: info, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 新增商品-已登的店铺(带有Out的都非官方标准接口) -// -//export OutAddGoods -func OutAddGoods(token, proxy, formData *C.char) *C.char { - goToken := C.GoString(token) - goProxy := C.GoString(proxy) - goFormData := C.GoString(formData) - info, err := outAddGoods(goToken, goProxy, goFormData) - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: info, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 获取图片URL(官图和拍图)带有店铺过滤 -// -//export OutGetImageFilterShopId -func OutGetImageFilterShopId(token, isbn *C.char, shopId C.int, proxy *C.char, isLiveImage C.int, isReturnMsg C.int) *C.char { - goToken := C.GoString(token) - goIsbn := C.GoString(isbn) - goShopId := int(shopId) - goProxy := C.GoString(proxy) - goIsLiveImage := int(isLiveImage) - goIsReturnMsg := int(isReturnMsg) - info, err := outGetImageFilterShopId(goToken, goIsbn, goShopId, goProxy, goIsLiveImage, goIsReturnMsg) - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: info, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 获取商品图片(带有Out的都非官方标准接口) -// -//export OutGetImageByIsbn -func OutGetImageByIsbn(token, isbn, proxy *C.char, isLiveImage C.int, isReturnMsg C.int) *C.char { - goToken := C.GoString(token) - goIsbn := C.GoString(isbn) - goProxy := C.GoString(proxy) - goIsLiveImage := int(isLiveImage) - goIsReturnMsg := int(isReturnMsg) - bookInfo, err := outGetImageByIsbn(goToken, goIsbn, goProxy, goIsLiveImage, goIsReturnMsg) - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: bookInfo, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 获取商品列表通过店铺ID(带有Out的都非官方标准接口) -// -//export OutGetGoodsListMsgByShopId -func OutGetGoodsListMsgByShopId(shopId C.int, proxy *C.char, retPrice C.int, isImage C.int, sortType *C.char, sort *C.char, priceMin C.float, priceMax C.float, pageNum, returnNum C.int) *C.char { - goShopId := int(shopId) - goProxy := C.GoString(proxy) - goRetPrice := int(retPrice) - goIsImage := int(isImage) - goSortType := C.GoString(sortType) - goSort := C.GoString(sort) - goPriceMin := float32(priceMin) - goPriceMax := float32(priceMax) - goPageNum := int(pageNum) - goReturnNum := int(returnNum) - books, num, pNum, err := outGetGoodsListMsgByShopId(goShopId, goProxy, goRetPrice, goIsImage, goSortType, goSort, goPriceMin, goPriceMax, goPageNum, goReturnNum) - // 构建统一格式的响应 - bookInfo := struct { - GoodsNum string `json:"goods_num,omitempty"` - PNum string `json:"pnum,omitempty"` - Data interface{} `json:"data,omitempty"` - }{ - GoodsNum: num, - PNum: pNum, - Data: books, - } - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: bookInfo, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 获取商品信息通过商品详情链接(带有0ut的都非官方标准接口) -// -//export OutGetGoodsMsgByDetailUrl -func OutGetGoodsMsgByDetailUrl(detailUrl, proxy *C.char) *C.char { - goDetailUrl := C.GoString(detailUrl) - goProxy := C.GoString(proxy) - response, err := outGetGoodsMsgByDetailUrl(goDetailUrl, goProxy) - // 构建统一格式的响应 - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: response, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 获取销量榜商品列表(带有Out的都非官放标准接口) -// -//export OutGetTopGoodsListMsg -func OutGetTopGoodsListMsg(catId C.int, proxy *C.char) *C.char { - goCatId := int(catId) - goProxy := C.GoString(proxy) - response, err := outGetTopGoodsListMsg(goCatId, goProxy) - // 构建统一格式的响应 - var apiResponse APIResponse - if err != nil { - apiResponse = APIResponse{ - Success: false, - Message: err.Error(), - } - } else { - apiResponse = APIResponse{ - Success: true, - Data: response, - } - } - // 转换为JSON字符串 - jsonData, marshalErr := json.Marshal(apiResponse) - if marshalErr != nil { - // 如果JSON序列化失败，返回错误信息 - apiResponse = APIResponse{ - Success: false, - Message: fmt.Sprintf("JSON序列化失败: %v", marshalErr), - } - errorJson, _ := json.Marshal(apiResponse) - return C.CString(string(errorJson)) - } - return C.CString(string(jsonData)) -} - -// 初始化配置 -// -//export Initialize -func Initialize(configJSON *C.char) *C.char { - configStr := C.GoString(configJSON) - log.Printf("[DEBUG] 接收到的配置JSON: %s", configStr) - - var config Config - if err := json.Unmarshal([]byte(configStr), &config); err != nil { - return C.CString(fmt.Sprintf(`{"success":false,"message":"配置解析失败: %v"}`, err)) - } - initializeConfig(config) - - return C.CString(`{"success":true,"message":"初始化成功"}`) -} - -// 导出函数：释放C字符串内存 -// -//export FreeCString -func FreeCString(str *C.char) { - C.free(unsafe.Pointer(str)) -} - -//// 空main函数，编译DLL时需要 -//func main() { -//} diff --git a/md/csv.md b/md/csv.md index 6268a47..042f586 100644 --- a/md/csv.md +++ b/md/csv.md @@ -149,10 +149,11 @@ dll.AppendRows(handle,rowsData,rowCount) |--|--------|--|----------------------------| | handle | int64 | 是 | 文件句柄 | | rowsData | []byte | 是 | 写入的数据，将[][]string转换成[]byte | +| dataSize | int | 是 | 写入数据的大小 | | rowCount | int | 是 | 数据行数 | ### 响应示例 ``` -文件数据行数：int64 +文件是否追加成功：int64 0：成功 ``` ## 5.获取总行数--GetRowCount diff --git a/md/kongfz.md b/md/kongfz.md index 4ce0ae9..67d4c5c 100644 --- a/md/kongfz.md +++ b/md/kongfz.md @@ -1696,7 +1696,7 @@ orderId, shippingId, shippingCom, shipmentNum, userDefined, moreShipmentNum) | accessToken | string | 否 | 用户登录授权成功后，开放平台颁发给应用的授权信息 | | shippingComName | string | 是 | 快递名称 | | orderId | int | 是 | 订单编号 | -| shippingId | string | 是 | 配送方式 | +| shippingId | string | 否 | 配送方式 | | shippingCom | string | 否 | 快递公司。当shippingId!=noLogistics时，此参数为必填。取值参考kongfz.delivery.method.list接口的返回值 | | shipmentNum | string | 否 | 快递单号。当shippingId!=noLogistics时，此参数为必填。 | | userDefined | string | 否 | 用户自定义物流公司。当shippingCom=other时，此参数为必填。 | @@ -1717,6 +1717,300 @@ orderId, shippingId, shippingCom, shipmentNum, userDefined, moreShipmentNum) } ``` +## 15.上传图片接口--KongfzImageUpload +### 请求信息 +```gotemplate +dll.KongfzImageUpload(appId, appSecret, accessToken, filePath, savePath) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|----|-------------------------------| +| appId | int | 是 | 开放平台分配给应用的AppId | +| appSecret | string | 是 | App密钥 | +| accessToken | string | 否 | 用户登录授权成功后，开放平台颁发给应用的授权信息 | +| filePath | string | 是 | 图片url/本地图片路径 | +| savePath | string | 是 | 图片下载路径，如果是图片url需要下载到本地 | +### 响应示例 +```json +{ + "requestId": "bAo3ZN679gaPgR9Y", + "requestMethod": "kongfz.image.upload", + "successResponse": { + "image": { + "url": "https://www.kfzimg.com/sw/kfzimg/2868/02698c12fec32cc69c_s.jpg" //图片地址 + } + }, + "errorResponse": null +} +``` + +## 16.添加店铺商品--KongfzShopItemAdd +### 请求信息 +```gotemplate +dll.KongfzShopItemAdd(appId, appSecret, accessToken, shopItemAddJson) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|----|-------------------------------| +| appId | int | 是 | 开放平台分配给应用的AppId | +| appSecret | string | 是 | App密钥 | +| accessToken | string | 否 | 用户登录授权成功后，开放平台颁发给应用的授权信息 | +| shopItemAddJson | string | 是 | 店铺商品请求结构体字符串 | +#### shopItemAddJson 请求结构体 +添加商品的业务参数比较复杂，根据商品分类的不同，适配不同的模板。以下参数将分别列出模板公共参数和模板特有参数。 +可以查看：https://open.kongfz.com/doc/api/shop/kongfz-shop-item-add +### 成功响应示例 +```json +{ + "kongfzShopItemAddResponse": { + "requestId": "mKPxiCD0dgaPSRib", + "requestMethod": "kongfz.shop.item.add", + "successResponse": { + "item": { + "itemId": 1343492600, //商品编号 + "addTime": "2019-06-22 16:12:17" //添加时间 + } + }, + "errorResponse": null + } +} +``` +### 失败响应示例 +```json +{ + "requestId": "8lDbCNSxzH4UHRx8", + "requestMethod": "kongfz.shop.item.add", + "successResponse": null, + "errorResponse": { + "code": 3003, + "msg": "Service error", + "subCode": "Failed To Add Item", + "subMsg": "添加商品失败", + "data": { //字段错误的具体信息会在data中给出，key是出错的字段名，value是错误的原因 + "mouldId": "请先添加运费模板" + } + } +} +``` + +## 17.查询订单列表--KongfzOrderList +### 请求信息 +```gotemplate +dll.KongfzOrderList(appId, appSecret, accessToken, orderListJson) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|----|-------------------------------| +| appId | int | 是 | 开放平台分配给应用的AppId | +| appSecret | string | 是 | App密钥 | +| accessToken | string | 否 | 用户登录授权成功后，开放平台颁发给应用的授权信息 | +| orderListJson | string | 是 | 查询订单请求结构体字符串 | +#### orderListJson 请求结构体 +```json +{ + "userType": "String", + "orderStatus": "String", + "pageNum": "Integer", + "pageSize": "Integer", + "isDelete": "Integer", + "startDate": "String", + "endDate": "String", + "startTime": "String", + "endTime": "String", + "startUpdateTime": "String", + "endUpdateTime": "String" +} +``` +### 响应示例 +```json +{ + "requestId": "VNI5AWvIW5cRLR9l", + "requestMethod": "kongfz.order.list", + "successResponse": { + "total": 5, //总条数 + "pages": 1, //总页数 + "size": 5, //当前页条数 + "pageSize": 20, //每页最大条数 + "pageNum": 1, //当前页码 + "list": [ //订单列表 + { + "orderId": 25259131, //订单编号 + "createdTime": "2014/06/25 14:36:59", //订单生成时间 + "shopId": 22281, //店铺编号 + "shopName": "俞含的书摊", //店铺名称 + "shopkeeperId": 2522488, //卖家用户编号 + "shipmentNum": "", //快递单号 + "shippingCom": "", //快递公司编码 + "shippingComName": "", //快递公司名称 + "shippingId": "express", //配送方式 + "shippingName": "快递", //配送方式名称 + "shippingFee": "8.00", //快递费 + "goodsAmount": "15.00", //订单商品金额 + "favorableMoney": "0.00", //商品优惠金额 + "orderAmount": "23.00", //订单金额 + "userId": 3723679, //买家用户编号 + "nickname": "买家昵称", //买家昵称 + "receiver": "姓名, 18812345678, 山东省青岛市市北区顺兴路152号404户, 266021", //收件人信息 + "receiverInfo": { + "area": "24006002000", + "zipCode": "266021", + "provName": "山东省", + "address": "顺兴路152号404户", + "cityName": "青岛市", + "areaName": "市北区", + "receiverName": "姓名", + "mobile": "18812345678", + "phoneNum": "" + }, + "sellerFlagType": 1, //备注类型,孔网是按颜色区分,0是未打标记,1:红,2:黄,3:绿,4:蓝,5:紫 + "sellerRemarkText": "测试备注信息", + "orderStatus": "SellerCancelledBeforeConfirm", //订单状态 + "orderStatusName": "卖家已取消", //订单状态名称 + "promotionId": 0, //活动编号 + "itemsCount": 1, //订单商品数 + "items": [ //订单商品 + { + "itemId": 250140272, //商品编号 + "itemSn": "740", //商品货号 + "number": 1, //购买商品数 + "itemName": "玉娇龙 下", //商品名称 + "img": "", //商品图片 + "isCancel": false, //商品是否被取消 + "orderId": 25259131, //订单编号 + "price": "15.00", //商品价格 + "favorableAmount": "0.00", //商品总优惠金额,即:setFavAmount+couponFavAmount,如果购买了多件,为多件商品的总优惠 + "setFavAmount": "0.00", //卖家设置优惠金额,卖家设置的订单优惠均摊到商品上的优惠,如果购买了多件,为多件商品的总优惠 + "couponFavAmount": "0.00", //优惠券优惠金额,订单使用了优惠券时,均摊到商品上的优惠金额,如果购买了多件,为多件商品的总优惠 + "realAmount": "15.00", //商品实付金额, 如果购买了多件，为多件商品的实付总金额 + "cancelMan": "unkown", //取消商品者，unkown：未知，seller：卖家，buyer：买家 + "quality": "", //商品品相 + "isbn": "" //图书ISBN + } + ] + }, + ] + }, + "errorResponse": null +} +``` + +## 18.整合添加商品和获取孔网图片--OutAddGoodsAndFile +### 请求信息 +```gotemplate +dll.OutAddGoodsAndFile(token, proxy, filePath, formData) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|----|---------------------------| +| token | string | 是 | 孔网token | +| proxy | string | 是 | 代理服务器IP | +| filePath | string | 否 | 图片路径 | +| formData | string | 是 | 商品JSON字符串 | +### 成功响应示例 +```json +{ + "data":9395776720, + "errType":"", + "message":"", + "status":1 +} +``` + +## 查询单个订单--KongfzOrderGet +### 请求信息 +```gotemplate +dll.KongfzOrderGet(appId, appSecret, accessToken, userType,orderId) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|----|---------------------------| +| appId | int | 是 | 开放平台分配给应用的AppId | +| appSecret | string | 是 | App密钥 | +| accessToken | string | 否 | 用户登录授权成功后，开放平台颁发给应用的授权信息 | +| userType | string | 是 | 用户类型。取值 seller: 卖家；buyer: 买家 | +| orderId | string | 是 | 订单编号 | +### 成功响应示例 +```json +{ + "requestId": "GH13BxApG5mRPRl0", + "requestMethod": "kongfz.order.get", + "successResponse": { + "orderId": 73412393, //订单编号 + "createdTime": "2019-05-13 09:31:26", //订单生成时间 + "shopId": 24671, //店铺编号 + "shopName": "网上测试书店（不售书）", //店铺名称 + "shopkeeperId": 3361935, //卖家用户编号 + "shipmentNum": "sa04465846222", //快递单号 + "shippingCom": "registeredPrint", //快递公司编码 + "shippingComName": "挂号印刷品", //快递公司名称 + "shippingId": "registerPost", //配送方式 + "shippingName": "挂号印刷品", //配送方式名称 + "shippingFee": "10.00", //运费 + "goodsAmount": "1.00", //商品金额 + "favorableMoney": "0.00", //订单商品优惠金额 + "orderAmount": "11.00", //订单金额 + "userId": 3361935, //用户编号 + "nickname": "网络测试书店", //用户昵称 + "receiver": "姓名, 18812345678, 山东省青岛市市北区顺兴路152号404户, 266021", //收件人信息 + "receiverInfo": { + "area": "24006002000", + "zipCode": "266021", + "provName": "山东省", + "address": "顺兴路152号404户", + "cityName": "青岛市", + "areaName": "市北区", + "receiverName": "姓名", + "mobile": "18812345678", + "phoneNum": "" + }, + "sellerFlagType": 1, //备注类型,孔网是按颜色区分,0是未打标记,1:红,2:黄,3:绿,4:蓝,5:紫 + "sellerRemarkText": "测试备注信息", + "orderStatus": "Shipped-Returning", //订单状态 + "orderStatusName": "申请退货中", //订单状态名称 + "promotionId": 0, //活动编号 + "itemsCount": 1, //订单商品数 + "items": [ //订单商品 + { + "itemId": 1265078240, //商品编号 + "itemSn": "", //商品货号 + "number": 1, //购买商品数 + "itemName": "测试勿拍19051003", //商品名称 + "img": "//www.kfzimg.com/G07/M00/10/0E/q4YBAFzVOgGAUQvkAAFgEf3-3BU117_s.jpg",//商品图片 + "isCancel": false, //商品是否被取消 + "orderId": 73412393, //订单编号 + "price": "1.00", //商品价格 + "favorableAmount": "0.00", //商品总优惠金额,即:setFavAmount+couponFavAmount,如果购买了多件,为多件商品的总优惠 + "setFavAmount": "0.00", //卖家设置优惠金额,卖家设置的订单优惠均摊到商品上的优惠,如果购买了多件,为多件商品的总优惠 + "couponFavAmount": "0.00", //优惠券优惠金额,订单使用了优惠券时,均摊到商品上的优惠金额,如果购买了多件,为多件商品的总优惠 + "realAmount": "1.00", //商品实付金额, 如果购买了多件，为多件商品的实付总金额 + "cancelMan": "unkown", //取消商品者，unkown：未知，seller：卖家，buyer：买家 + "quality": "六五品", //商品品相 + "isbn": "" //图书ISBN + }, + ], + "express": [ //物流信息 + { + "context": "已签收,他人代收：中通快递收,投递员:齐东事:18226889971," + "time": "2019-05-20 11:47:38", + "status": "签收" //状态字段不是都存在，一般只存在于最新的一条记录中 + }, + ... + ], + "records": [ //订单操作记录 + { + "nickname": "网络测试书店", + "remark": "买家创建退货协议", + "time": "2019-05-19 16:38:59", + "userType": "buyer" + }, + ... + ] + }, + "errorResponse": null +} +``` + + ## 12.初始化--Initialize(可以不调用) ### 请求信息 ```gotemplate diff --git a/md/newcsv.md b/md/newcsv.md new file mode 100644 index 0000000..336ba48 --- /dev/null +++ b/md/newcsv.md @@ -0,0 +1,307 @@ +# csv.dll 使用教程 +## 1.创建DLL工具实例 +### 加载DLL文件 +```gotemplate +// CSVDLL CSV文件工具DLL结构 +type csvDLL struct { + dll *syscall.DLL + openCSVFile *syscall.Proc // 打开CSV文件 + writeHeader *syscall.Proc // 写入表头 + writeRows *syscall.Proc // 写入/覆盖行数据 + updateRow *syscall.Proc // 修改指定行数据 + getRow *syscall.Proc // 获取指定行数据 + closeHandles *syscall.Proc // 关闭指定句柄 + closeAllHandles *syscall.Proc // 关闭所有句柄 + freeCString *syscall.Proc // 释放C字符串 +} + +// 初始化csvDLL +func InitCSVDLL() (*csvDLL, error) { + dllPath := filepath.Join("dll", "csv.dll") + if _, err := os.Stat(dllPath); os.IsNotExist(err) { + return nil, fmt.Errorf("csv DLL 不存在: %s", dllPath) + } + if dll, err := syscall.LoadDLL(dllPath); err != nil { + return nil, fmt.Errorf("加载csv DLL 失败: %s", err) + } else { + return &csvDLL{ + dll: dll, + openCSVFile: dll.MustFindProc("OpenCSVFile"), + writeHeader: dll.MustFindProc("WriteHeader"), + writeRows: dll.MustFindProc("WriteRows"), + updateRow: dll.MustFindProc("UpdateRow"), + getRow: dll.MustFindProc("GetRow"), + closeHandles: dll.MustFindProc("CloseHandles"), + closeAllHandles: dll.MustFindProc("CloseAllHandles"), + freeCString: dll.MustFindProc("FreeCString"), + }, nil + } +} + +dll, err := InitCSVDLL() +``` + +### 获取C字符串 +```gotemplate +// cStr 获取C字符串 +func (m *csvDLL) cStr(p uintptr) string { + if p == 0 { + return "" + } + b := []byte{} + for i := uintptr(0); ; i++ { + c := *(*byte)(unsafe.Pointer(p + i)) + if c == 0 { + break + } + b = append(b, c) + } + s := string(b) + if m.freeCString != nil { + m.freeCString.Call(p) + } + return s +} +``` + +## 2. 使用dll函数示例 +```gotemplate +// 打开CSV文件 +func (m *csvDLL) OpenCSVFile(filename string, delimiter byte, hasHeader bool) (string, error) { + proc, err := m.dll.FindProc("OpenCSVFile") + if err != nil { + return "", fmt.Errorf("找不到函数 OpenCSVFile: %v", err) + } + + filenamePtr, _ := syscall.BytePtrFromString(filename) + hasHeaderInt := C.int(0) + if hasHeader { + hasHeaderInt = 1 + } + + resultPtr, _, _ := proc.Call( + uintptr(unsafe.Pointer(filenamePtr)), + uintptr(delimiter), + uintptr(hasHeaderInt), + ) + + result := m.cStr(resultPtr) + return result, nil +} +``` + +# 接口详情 +## 打开CSV文件--OpenCSVFile +### 请求信息 +```gotemplate +dll.OpenCSVFile(filename, delimiter, hasHeader) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|--|----------------| +| filename | string | 是 | 文件名称(带路径) | +| delimiter | string | 是 | 分隔符(如 ',') | +| hasHeader | string | 是 | 是否有表头（0=是） | +### 响应示例 +```json +{ + "success": true, + "data": { + "handleID": 123456789 + } +} +``` +### 错误响应示例 +```json +{ + "success": false, + "message": "文件不存在: /path/to/file.csv" +} +``` + +## 写入表头--WriteHeader +### 请求信息 +```gotemplate +dll.WriteHeader(handleID, header) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|--|--------------| +| handleID | int | 是 | CSV文件句柄ID | +| header | string | 是 | 表头JSON数组字符串 | +### header参数示例 +```json +["列1", "列2", "列3", "列4"] +``` +### 响应示例 +```json +{ + "success": true +} +``` +### 错误响应示例 +```json +{ + "success": false, + "message": "header JSON解析失败: invalid character '[' looking for beginning of value" +} +``` + +## 写入/覆盖行数据--WriteRows +### 请求信息 +```gotemplate +dll.WriteRows(handleID, rowsData) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|--|---------------| +| handleID | int | 是 | CSV文件句柄ID | +| rowsData | string | 是 | 行数据JSON数组字符串 | +### rowsData参数示例 +```json +[ + ["数据1-1", "数据1-2", "数据1-3", "数据1-4"], + ["数据2-1", "数据2-2", "数据2-3", "数据2-4"], + ["数据3-1", "数据3-2", "数据3-3", "数据3-4"] +] +``` +### 响应示例 +```json +{ + "success": true, + "data": { + "totalRows": 15 + } +} +``` +### 错误响应示例 +```json +{ + "success": false, + "message": "rowsData JSON解析失败: invalid character '[' looking for beginning of value" +} +``` + +## 修改指定行数据--UpdateRow +### 请求信息 +```gotemplate +dll.UpdateRow(handleID, rowNumber, rowData) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|--|---------------| +| handleID | int | 是 | CSV文件句柄ID | +| rowNumber | int | 是 | 要修改的行号（从0开始） | +| rowData | string | 是 | 新行数据JSON数组字符串 | +### rowsData参数示例 +```json +["新数据1", "新数据2", "新数据3", "新数据4"] +``` +### 响应示例 +```json +{ + "success": true, + "message": "成功修改第2行数据" +} +``` +### 错误响应示例 +```json +{ + "success": false, + "message": "行号超出范围: 10, 总行数: 5" +} +``` + +## 获取指定行数据--GetRow +### 请求信息 +```gotemplate +dll.GetRow(handleID, rowNumber) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|--|---------------| +| handleID | int | 是 | CSV文件句柄ID | +| rowNumber | int | 是 | 要获取的行号（从0开始） | +### 响应示例 +```json +{ + "success": true, + "data": ["数据1", "数据2", "数据3", "数据4"] +} +``` +### 错误响应示例 +```json +{ + "success": false, + "message": "行号超出范围: 10, 总行数: 5" +} +``` + +## 合并两个csv文件--MergeCSVFilesSimple +### 请求信息 +```gotemplate +dll.MergeCSVFilesSimple(srcHandleID, dstHandleID, appendMode) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|--|--------------------------| +| srcHandleID | int64 | 是 | 源文件句柄ID | +| dstHandleID | int64 | 是 | 目标文件句柄ID（合并后结果） | +| appendMode | int | 是 | 0:true=追加模式，1:false=覆盖模式 | +### 响应示例 +### 响应示例 +```json +{ + "success": true, + "data": { + "totalRows": 15 + } +} +``` +### 错误响应示例 +```json +{ + "success": false, + "message": "rowsData JSON解析失败: invalid character '[' looking for beginning of value" +} +``` + +## 关闭指定句柄--CloseHandles +### 请求信息 +```gotemplate +dll.CloseHandles(handleID) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|--|---------------| +| handleID | int | 是 | CSV文件句柄ID | +### 响应示例 +```text +关闭句柄 123456789 成功！ +``` +### 错误响应示例 +```text +句柄不存在: 123456789 +``` + +## 关闭所有句柄--CloseAllHandles +### 请求信息 +```gotemplate +dll.CloseAllHandles() +``` +### 请求参数 +无 +### 响应示例 +```text +关闭句柄成功！ +``` + +## 释放C字符串内存--FreeCString +### 请求信息 +```gotemplate +dll.FreeCString(str) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--------|--|-----------| +| str | string | 是 | 需要释放的字符串 | \ No newline at end of file diff --git a/md/pdd.md b/md/pdd.md index 82c1e4a..8810905 100644 --- a/md/pdd.md +++ b/md/pdd.md @@ -209,8 +209,7 @@ orderSn, orderState, waybillNo) ## 4. 拼多多订单同步--PddOrderSynchronization ### 请求信息 ```gotemplate -dll.PddOrderSynchronization(clientId, clientSecret, accessToken, logisticsCompany, logisticsId, -orderSn, orderState, waybillNo) +dll.PddOrderSynchronization(clientId, clientSecret, accessToken, logisticsCompany, logisticsOnlineSendJson) ``` ### 请求参数 | 参数名 | 类型 | 必填 | 说明 | @@ -219,10 +218,7 @@ orderSn, orderState, waybillNo) | clientSecret | string | 是 | 拼多多开放平台ClientSecret | | accessToken | string | 是 | 授权令牌 | | logisticsCompany | string | 是 | 物流公司名称 | -| logisticsId | string | 是 | 物流公司ID | -| orderSn | string | 是 | 拼多多订单号 | -| orderState | string | 是 | 订单状态 | -| waybillNo | string | 是 | 运单号 | +| logisticsOnlineSendJson | string | 是 | 拼多多订单同步json字符串 | ### 响应示例 ```json { @@ -492,4 +488,14 @@ dll.PddOpenDecryptMaskBatch(clientId, clientSecret, accessToken, reqJson) "request_id": "15440104776643887" } } -``` \ No newline at end of file +``` + +## 12.释放C字符串内存--FreeCString +### 请求信息 +```gotemplate +dll.FreeCString(str) +``` +### 请求参数 +| 参数名 | 类型 | 必填 | 说明 | +|--|--|--|----------| +| str | string | 是 | 需要释放的字符串 | diff --git a/pdd/dll/pdd.dll b/pdd/dll/pdd.dll index 62d652e..cb352d9 100644 Binary files a/pdd/dll/pdd.dll and b/pdd/dll/pdd.dll differ diff --git a/pdd/dll/pdd.h b/pdd/dll/pdd.h index 1a876b6..95f58f6 100644 --- a/pdd/dll/pdd.h +++ b/pdd/dll/pdd.h @@ -102,7 +102,7 @@ extern __declspec(dllexport) char* PddErpOrderSync(char* clientId, char* clientS // PddOrderSynchronization 拼多多订单同步 // -extern __declspec(dllexport) char* PddOrderSynchronization(char* clientId, char* clientSecret, char* accessToken, char* logisticsCompany, char* logisticsId, char* orderSn, char* orderState, char* waybillNo); +extern __declspec(dllexport) char* PddOrderSynchronization(char* clientId, char* clientSecret, char* accessToken, char* logisticsCompany, char* logisticsOnlineSendJson); // PddGoodsImgUpload 商品图片上传接口 // @@ -116,7 +116,11 @@ extern __declspec(dllexport) char* PddGoodsAdd(char* clientId, char* clientSecre // extern __declspec(dllexport) char* SelfPddGoodsAdd(char* clientId, char* clientSecret, char* accessToken, char* filePath, char* goodsAddJson); -// 释放C字符串内存 +// PddOpenDecryptMaskBatch 批量数据解密脱敏接口 +// +extern __declspec(dllexport) char* PddOpenDecryptMaskBatch(char* clientId, char* clientSecret, char* accessToken, char* reqJson); + +// FreeCString 释放C字符串内存 // extern __declspec(dllexport) void FreeCString(char* str); diff --git a/pdd/pdd.go b/pdd/pdd.go index 71a210f..a2ea928 100644 --- a/pdd/pdd.go +++ b/pdd/pdd.go @@ -35,7 +35,12 @@ type ErrorWrapper struct { ErrorResponse ErrorResponse `json:"error_response"` // 错误响应 } -// generateSign 生成签名 +/* + * 生成签名 + * param params[map[string]interface{}] 生成签名需要的map + * param clientSecret[string] 客户端密钥 + * return 签名 + */ func generateSign(params map[string]interface{}, clientSecret string) string { // 获取所有键并排序 keys := make([]string, 0, len(params)) @@ -72,7 +77,16 @@ func generateSign(params map[string]interface{}, clientSecret string) string { return strings.ToUpper(hex.EncodeToString(hasher.Sum(nil))) } -// 类目预测 +/* + * 类目预测 + * param clientId[string] 客户端ID + * param clientSecret[string] 客户端密钥 + * param accessToken[string] 访问Token + * param outerCatId[string] 外部类目ID + * param outerCatName[string] 外部类目名称 + * param outerGoodsName[string] 外部商品名称 + * return 类目预测响应结构体，错误信息 + */ func pddGoodsOuterCatMappingGet(clientId, clientSecret, accessToken, outerCatId, outerCatName, outerGoodsName string) (string, error) { // API地址 @@ -134,7 +148,12 @@ func pddGoodsOuterCatMappingGet(clientId, clientSecret, accessToken, return string(responseJSON), nil } -// 快递公司查看 +/* + * 快递公司查看 + * param clientId[string] 客户端ID + * param clientSecret[string] 客户端密钥 + * return 快递公司查看响应结构体，错误信息 + */ func pddLogisticsCompaniesGet(clientId, clientSecret string) (string, error) { url := fmt.Sprint("https://gw-api.pinduoduo.com/api/router") timestamp := fmt.Sprintf("%d", time.Now().Unix()) @@ -195,7 +214,17 @@ func pddLogisticsCompaniesGet(clientId, clientSecret string) (string, error) { return string(responseJSON), nil } -// erp打单信息同步 +/* + * erp打单信息同步 + * param clientId[string] 客户端ID + * param clientSecret[string] 客户端密钥 + * param accessToken[string] 访问Token + * param logisticsId[string] 物流公司编码 + * param orderSn[string] 订单号 + * param orderState[string] 订单状态：1-已打单 + * param waybillNo[string] 运单号 + * return 快递公司查看响应结构体，错误信息 + */ func pddErpOrderSync(clientId, clientSecret, accessToken, logisticsId, orderSn, orderState, waybillNo string) (string, error) { url := fmt.Sprint("https://gw-api.pinduoduo.com/api/router") @@ -266,6 +295,78 @@ func pddErpOrderSync(clientId, clientSecret, accessToken, logisticsId, return string(responseJSON), nil } +/* + * 订单发货通知接口 + * param clientId[string] 客户端ID + * param clientSecret[string] 客户端密钥 + * param accessToken[string] 访问Token + * param logisticsOnlineSendJson[string] 订单发货通知json字符串 + * return 订单发货通知接口响应结构体，错误信息 + */ +func pddLogisticsOnlineSend(clientId, clientSecret, accessToken string, logisticsOnlineSendJson string) (string, error) { + url := fmt.Sprint("https://gw-api.pinduoduo.com/api/router") + timestamp := fmt.Sprintf("%d", time.Now().Unix()) + // 生成签名 + params := map[string]interface{}{ + "type": "pdd.logistics.online.send", + "data_type": "JSON", + "client_id": clientId, + "access_token": accessToken, + "timestamp": timestamp, + } + // 将JSON参数合并到params中 + toParams, err := addStructToParams(logisticsOnlineSendJson, params) + if err != nil { + return "", err + } + + sign := generateSign(toParams, clientSecret) + + if sign == "" { + return "", fmt.Errorf("生成 sign 签名错误！") + } + params["sign"] = sign + + request := gorequest.New() + resp, body, errs := request.Get(url). + Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"). + Set("Accept", "application/json, text/plain, */*"). + Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"). + Timeout(30 * time.Second). + Send(params). + End() + if len(errs) > 0 { + return "", fmt.Errorf("请求失败: %v", errs) + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("HTTP状态码异常: %d, 响应: %s", resp.StatusCode, body) + } + + // 解析响应 + var response map[string]interface{} + if err := json.Unmarshal([]byte(body), &response); err != nil { + return "", fmt.Errorf("解析响应失败: %v, 响应内容: %s", err, body) + } + + // 异常处理 + if response["error_response"] != nil { + var errorWrapper ErrorWrapper + // 解析响应 + if err := json.Unmarshal([]byte(body), &errorWrapper); err != nil { + return "", fmt.Errorf("解析 errorWrapper 失败: %v, 响应内容: %s", err, body) + } + return "", fmt.Errorf("请求失败: %v, 错误码: %d", errorWrapper.ErrorResponse.ErrorMsg, errorWrapper.ErrorResponse.ErrorCode) + } + + // 转换成json字符串 + responseJSON, err := json.Marshal(response) + if err != nil { + return "", fmt.Errorf("JSON序列化失败: %v", err) + } + return string(responseJSON), nil +} + // LogisticsCompany 物流公司信息结构体 type LogisticsCompany struct { Available int `json:"available"` // 是否可用 @@ -284,9 +385,23 @@ type LogisticsResponse struct { LogisticsCompaniesGetResponse LogisticsCompaniesGetResponse `json:"logistics_companies_get_response"` // 物流公司响应 } -// 拼多多订单同步（组合接口：先查物流公司，再同步订单） -func pddOrderSynchronization(clientId, clientSecret, accessToken, logisticsCompany, logisticsId, - orderSn, orderState, waybillNo string) (string, error) { +/* + * 拼多多订单同步（组合接口：先查物流公司，再同步订单） + * param clientId[string] 客户端ID + * param clientSecret[string] 客户端密钥 + * param accessToken[string] 访问Token + * param logisticsCompany[string] 外部类目ID + * param logisticsOnlineSendJson[string] 拼多多订单同步json字符串 + * return 类目预测响应结构体，错误信息 + */ +func pddOrderSynchronization(clientId, clientSecret, accessToken, logisticsCompany, logisticsOnlineSendJson string) (string, error) { + + var logisticsOnlineSendData map[string]interface{} + err := json.Unmarshal([]byte(logisticsOnlineSendJson), &logisticsOnlineSendData) + if err != nil { + return "", fmt.Errorf("logisticsOnlineSendJson JSON解析失败: %v", err) + } + // 1. 获取物流公司列表 logisticsCompaniesGet, err := pddLogisticsCompaniesGet(clientId, clientSecret) if err != nil { @@ -300,20 +415,31 @@ func pddOrderSynchronization(clientId, clientSecret, accessToken, logisticsCompa // 3. 根据物流公司名称查找对应的物流公司ID for _, lc := range response.LogisticsCompaniesGetResponse.LogisticsCompanies { if lc.LogisticsCompany == logisticsCompany { - logisticsId = fmt.Sprintf("%d", lc.ID) + logisticsOnlineSendData["logistics_id"] = fmt.Sprintf("%d", lc.ID) break } } - // 4. 执行ERP订单同步 - erpOrderSync, err := pddErpOrderSync(clientId, clientSecret, accessToken, - logisticsId, orderSn, orderState, waybillNo) + fmt.Println(logisticsOnlineSendData["logistics_id"]) + + logisticsOnlineSendStr, err := json.Marshal(logisticsOnlineSendData) + if err != nil { + return "", fmt.Errorf("logisticsOnlineSendData 序列化失败: %v", err) + } + logisticsOnlineSend, err := pddLogisticsOnlineSend(clientId, clientSecret, accessToken, string(logisticsOnlineSendStr)) if err != nil { return "", err } - return erpOrderSync, nil + return logisticsOnlineSend, nil } -// 商品图片上传接口 +/* + * 商品图片上传接口 + * param clientId[string] 客户端ID + * param clientSecret[string] 客户端密钥 + * param accessToken[string] 访问Token + * param filePath[string] 商品图片文件流 + * return 商品图片上传响应结构体，错误信息 + */ func pddGoodsImgUpload(clientId, clientSecret, accessToken string, filePath string) (string, error) { url := fmt.Sprint("https://gw-upload.pinduoduo.com/api/upload") // 上传专用地址 @@ -562,7 +688,14 @@ type SkuProperty struct { Vid int64 `json:"vid"` // 属性值id } -// 商品新增接口 +/* + * 商品新增接口 + * param clientId[string] 客户端ID + * param clientSecret[string] 客户端密钥 + * param accessToken[string] 访问Token + * param goodsAddJson[string] 新增商品信息json字符串 + * return 商品新增接口响应结构体，错误信息 + */ func pddGoodsAdd(clientId, clientSecret, accessToken string, goodsAddJson string) (string, error) { url := fmt.Sprint("http://gw-api.pinduoduo.com/api/router") @@ -633,7 +766,14 @@ func pddGoodsAdd(clientId, clientSecret, accessToken string, goodsAddJson string return string(responseJSON), nil } -// 联合拼多多图片上传的商品新增（组合接口） +/* + * 联合拼多多图片上传的商品新增（组合接口） + * param clientId[string] 客户端ID + * param clientSecret[string] 客户端密钥 + * param accessToken[string] 访问Token + * param goodsAddJson[string] 新增商品信息json字符串 + * return 商品新增接口响应结构体，错误信息 + */ func selfPddGoodsAdd(clientId, clientSecret, accessToken string, filePath string, goodsAddJson string) (string, error) { // 1. 上传商品图片 upload, err := pddGoodsImgUpload(clientId, clientSecret, accessToken, filePath) @@ -660,7 +800,14 @@ func selfPddGoodsAdd(clientId, clientSecret, accessToken string, filePath string return add, nil } -// 批量数据解密脱敏接口 +/* + * 批量数据解密脱敏接口 + * param clientId[string] 客户端ID + * param clientSecret[string] 客户端密钥 + * param accessToken[string] 访问Token + * param reqJson[string] 批量数据解密脱敏接口json字符串 + * return 批量数据解密脱敏接口响应结构体，错误信息 + */ func pddOpenDecryptMaskBatch(clientId, clientSecret, accessToken string, reqJson string) (string, error) { url := fmt.Sprint("http://gw-api.pinduoduo.com/api/router") // 当前时间戳 @@ -796,17 +943,13 @@ func PddErpOrderSync(clientId, clientSecret, accessToken, logisticsId, // PddOrderSynchronization 拼多多订单同步 // //export PddOrderSynchronization -func PddOrderSynchronization(clientId, clientSecret, accessToken, logisticsCompany, logisticsId, - orderSn, orderState, waybillNo *C.char) *C.char { +func PddOrderSynchronization(clientId, clientSecret, accessToken, logisticsCompany, logisticsOnlineSendJson *C.char) *C.char { goClientId := C.GoString(clientId) goClientSecret := C.GoString(clientSecret) goAccessToken := C.GoString(accessToken) goLogisticsCompany := C.GoString(logisticsCompany) - goLogisticsId := C.GoString(logisticsId) - goOrderSn := C.GoString(orderSn) - goOrderState := C.GoString(orderState) - goWaybillNo := C.GoString(waybillNo) - info, err := pddOrderSynchronization(goClientId, goClientSecret, goAccessToken, goLogisticsCompany, goLogisticsId, goOrderSn, goOrderState, goWaybillNo) + goLogisticsOnlineSendJson := C.GoString(logisticsOnlineSendJson) + info, err := pddOrderSynchronization(goClientId, goClientSecret, goAccessToken, goLogisticsCompany, goLogisticsOnlineSendJson) if err != nil { return C.CString(err.Error()) } @@ -882,5 +1025,5 @@ func FreeCString(str *C.char) { } // main函数 -func main() { -} +//func main() { +//} diff --git a/pdd/pddDll.go b/pdd/pddDll.go index e2adadd..d08a340 100644 --- a/pdd/pddDll.go +++ b/pdd/pddDll.go @@ -135,9 +135,9 @@ func (m *pddDLL) PddErpOrderSync(clientId, clientSecret, accessToken, logisticsI } func main() { - clientId := "203c5a7ba8bd4b8488d5e26f93052642" - clientSecret := "892ffaa86e12b7a3d8d2942b669d9aa520ad8179" - accessToken := "bd96218bb2a146779701506dc1e5e5c478692539" + //clientId := "203c5a7ba8bd4b8488d5e26f93052642" + //clientSecret := "892ffaa86e12b7a3d8d2942b669d9aa520ad8179" + //accessToken := "bd96218bb2a146779701506dc1e5e5c478692539" //outerCatId := "15543" //outerCatName := "书籍/杂志/报纸" //outerGoodsName := "书籍医家金鉴 妇产科学卷" @@ -210,7 +210,7 @@ func main() { // 脱敏 - jsonStr := `[{"data_tag":"251229-272441044622514","encrypted_data":"~AgAAAAPlscEH0psOJAEXpTdsLOWvDJ9bB7IEjIoqNfiDhhJR9NHOxsdZ+PEFluSSCngCikoDU+CP/sSXZJ92ic7+PdNlJNLA7g/6VUMDWF6RvjW9IeRN+lKNarsjWDQR~0~"}]` + //jsonStr := `[{"data_tag":"251229-272441044622514","encrypted_data":"~AgAAAAPlscEH0psOJAEXpTdsLOWvDJ9bB7IEjIoqNfiDhhJR9NHOxsdZ+PEFluSSCngCikoDU+CP/sSXZJ92ic7+PdNlJNLA7g/6VUMDWF6RvjW9IeRN+lKNarsjWDQR~0~"}]` //var records []DataList //err := json.Unmarshal([]byte(jsonStr), &records) @@ -218,12 +218,25 @@ func main() { // log.Fatal("解析JSON失败:", err) //} - batch, err := pddOpenDecryptMaskBatch(clientId, clientSecret, accessToken, jsonStr) - if err != nil { - fmt.Println(err) - } - fmt.Println(batch) - + //batch, err := pddOpenDecryptMaskBatch(clientId, clientSecret, accessToken, jsonStr) + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println(batch) + //clientId := "203c5a7ba8bd4b8488d5e26f93052642" + //clientSecret := "892ffaa86e12b7a3d8d2942b669d9aa520ad8179" + //accessToken := "5b1e9506827049a7a9335302e917d2b896a3d6c7" + //logisticsCompany := "韵达快递" + //logisticsId := "" + //orderSn := "260107-652497405582514" + //orderState := "1" + //waybillNo := "312944253800986" + // + //synchronization, err := pddOrderSynchronization(clientId, clientSecret, accessToken, logisticsCompany, logisticsId, orderSn, orderState, waybillNo) + //if err != nil { + // fmt.Println(err) + //} + //fmt.Println(string(synchronization)) } type DataList struct { diff --git a/proxy/proxy.go b/proxy/proxy.go index 2210e67..d0fe0aa 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -50,7 +50,7 @@ type ProxyManager struct { proxyType string `json:"proxy_type"` // 代理类型 CALF_ELEPHANT_PROXY/TAIL_PROXY } -// 代理健康状态结构体，用于跟踪代理的健康状况 +// ProxyHealth 代理健康状态结构体，用于跟踪代理的健康状况 type ProxyHealth struct { SuccessCount int // 成功次数 FailCount int // 失败次数 @@ -65,7 +65,14 @@ func init() { globalRand = rand.New(rand.NewSource(time.Now().UnixNano())) } -// 获取代理URL，代理类型管理器，根据代理类型构建不同的代理URL +/* + * 获取代理URL，代理类型管理器，根据代理类型构建不同的代理URL + * param proxyType[string] 代理类型 + * param username[string] 代理用户名 + * param password[string] 代理密码 + * param machineCode[string] 机器码 + * return 代理服务器IP,错误信息 + */ func proxyTypeManager(proxyType, username, password, machineCode string) (string, error) { switch proxyType { case CalfElephantProxyType: @@ -80,7 +87,12 @@ func proxyTypeManager(proxyType, username, password, machineCode string) (string } } -// 构建小象代理URL +/* + * 构建小象代理URL + * param username[string] 代理用户名 + * param password[string] 代理密码 + * return 代理服务器IP,错误信息 + */ func buildCalfElephantProxyURL(username, password string) (string, error) { // 随机选择一个代理服务器 server := randomServer() @@ -103,7 +115,13 @@ func buildCalfElephantProxyURL(username, password string) (string, error) { return proxyURL, nil } -// 尝试下一个小象代理服务器 +/* + * 尝试下一个小象代理服务器 + * param username[string] 代理用户名 + * param password[string] 代理密码 + * param failedServer[string] 代理服务器 + * return 代理服务器IP,错误信息 + */ func tryNextCalfElephantProxy(username, password, failedServer string) (string, error) { // 创建服务器副本并排除失败的服务器 availableServers := make([]string, 0) @@ -137,7 +155,11 @@ func tryNextCalfElephantProxy(username, password, failedServer string) (string, return "", fmt.Errorf("所有可用的小象代理服务器都检测失败") } -// 构建内置代理URL +/* + * 构建内置代理URL + * param machineCode[string] 机器码 + * return 代理服务器IP,错误信息 + */ func buildTailProxyURL(machineCode string) (string, error) { // 获取代理列表 proxies, err := getProxies(machineCode) @@ -163,7 +185,11 @@ func buildTailProxyURL(machineCode string) (string, error) { return findWorkingTailProxy(proxies) } -// 过滤健康代理，返回当前健康的代理列表 +/* + * 过滤健康代理，返回当前健康的代理列表 + * param proxies[[]string] 内置代理服务器数组 + * return 内置代理服务器数组,错误信息 + */ func filterHealthyProxies(proxies []string) []string { // 获取读锁，允许多个goroutine同时读取 proxyHealthMutex.RLock() @@ -181,7 +207,11 @@ func filterHealthyProxies(proxies []string) []string { return healthy } -// 查找可用的尾巴代理 +/* + * 查找可用的尾巴代理 + * param proxies[[]string] 内置代理服务器数组 + * return 可用的代理服务器IP,错误信息 + */ func findWorkingTailProxy(proxies []string) (string, error) { // 打乱代理顺序 shuffledProxies := shuffleServers(proxies) @@ -238,7 +268,11 @@ func findWorkingTailProxy(proxies []string) (string, error) { return "", fmt.Errorf("所有尾巴代理都不可用") } -// 检测代理健康状态，通过访问测试网站验证代理可用性 +/* + * 检测代理健康状态，通过访问测试网站验证代理可用性 + * param proxyURL[string] 代理服务器 + * return 错误信息 + */ func checkProxyHealth(proxyURL string) error { start := time.Now() // 记录开始时间，用于计算响应时间 // 创建HTTP请求，设置代理和超时 @@ -263,7 +297,12 @@ func checkProxyHealth(proxyURL string) error { return nil } -// 更新代理健康状态 +/* + * 更新代理健康状态 + * param proxyURL[string] 代理服务器 + * param success[bool] 是否成功 + * param responseTime[time.Duration] 响应时间 + */ func updateProxyHealth(proxyURL string, success bool, responseTime time.Duration) { // 获取写锁，确保更新操作的互斥性 proxyHealthMutex.Lock() @@ -295,7 +334,11 @@ func updateProxyHealth(proxyURL string, success bool, responseTime time.Duration } } -// 获取代理主机名 +/* + * 获取代理主机名 + * param proxyURL[string] 代理服务器 + * return 代理服务器IP + */ func getProxyHost(proxyURL string) string { // 去除协议前缀 if strings.HasPrefix(proxyURL, "http://") { @@ -338,7 +381,11 @@ func shuffleServers(servers []string) []string { return shuffled } -// 检查卡密是否过期 +/* + * 检查卡密是否过期 + * param tailCardSecret[string] 卡密 + * return 是否过期，错误信息 + */ func checkTailCardSecretExpired(tailCardSecret string) (bool, error) { // 获取机器码信息 code, err := getMachineCode(tailCardSecret) @@ -372,7 +419,11 @@ type getMachineCodeResp struct { } `json:"data"` } -// 查询机器码 +/* + * 查询机器码 + * param tailCardSecret[string] 卡密 + * return 响应结构体，错误信息 + */ func getMachineCode(tailCardSecret string) (*getMachineCodeResp, error) { url := "http://114.66.2.223:7842/api/proxies/ip_show" // 构建请求数据 @@ -408,7 +459,12 @@ func getMachineCode(tailCardSecret string) (*getMachineCodeResp, error) { } } -// 充值卡密 +/* + * 充值卡密 + * param tailCardSecret[string] 卡密 + * param machineCode[string] 机器码 + * return 机器码，错误信息 + */ func rechargeCard(tailCardSecret, machineCode string) (string, error) { url := "http://114.66.2.223:7842/api/proxies/ip_recharge" // 构建请求数据 @@ -443,7 +499,11 @@ func rechargeCard(tailCardSecret, machineCode string) (string, error) { return resp.Data.MachineCode, nil } -// 获取代理服务器列表 +/* + * 获取代理服务器列表 + * param machineCode[string] 机器码 + * return 代理服务器组，错误信息 + */ func getProxies(machineCode string) ([]string, error) { log.Printf("[INFO] 开始获取代理列表: %s", machineCode) // 生成签名 diff --git a/so/erp.so b/so/erp.so new file mode 100644 index 0000000..b09b30b Binary files /dev/null and b/so/erp.so differ diff --git a/so/kongfz.h b/so/kongfz.h deleted file mode 100644 index ce93dd7..0000000 --- a/so/kongfz.h +++ /dev/null @@ -1,146 +0,0 @@ -/* Code generated by cmd/cgo; DO NOT EDIT. */ - -/* package command-line-arguments */ - - -#line 1 "cgo-builtin-export-prolog" - -#include - -#ifndef GO_CGO_EXPORT_PROLOGUE_H -#define GO_CGO_EXPORT_PROLOGUE_H - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef struct { const char *p; ptrdiff_t n; } _GoString_; -extern size_t _GoStringLen(_GoString_ s); -extern const char *_GoStringPtr(_GoString_ s); -#endif - -#endif - -/* Start of preamble from import "C" comments. */ - - -#line 3 "main.go" - - #include - - // proxyConfig.dll 函数声明 - extern char* ProxyTypeManager(char* proxyType, char* username, char* password, char* machineCode); - extern void FreeCString(char* str); - -#line 1 "cgo-generated-wrapper" - - -/* End of preamble from import "C" comments. */ - - -/* Start of boilerplate cgo prologue. */ -#line 1 "cgo-gcc-export-header-prolog" - -#ifndef GO_CGO_PROLOGUE_H -#define GO_CGO_PROLOGUE_H - -typedef signed char GoInt8; -typedef unsigned char GoUint8; -typedef short GoInt16; -typedef unsigned short GoUint16; -typedef int GoInt32; -typedef unsigned int GoUint32; -typedef long long GoInt64; -typedef unsigned long long GoUint64; -typedef GoInt64 GoInt; -typedef GoUint64 GoUint; -typedef size_t GoUintptr; -typedef float GoFloat32; -typedef double GoFloat64; -#ifdef _MSC_VER -#if !defined(__cplusplus) || _MSVC_LANG <= 201402L -#include -typedef _Fcomplex GoComplex64; -typedef _Dcomplex GoComplex128; -#else -#include -typedef std::complex GoComplex64; -typedef std::complex GoComplex128; -#endif -#else -typedef float _Complex GoComplex64; -typedef double _Complex GoComplex128; -#endif - -/* - static assertion to make sure the file is being used on architecture - at least with matching size of GoInt. -*/ -typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; - -#ifndef GO_CGO_GOSTRING_TYPEDEF -typedef _GoString_ GoString; -#endif -typedef void *GoMap; -typedef void *GoChan; -typedef struct { void *t; void *v; } GoInterface; -typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; - -#endif - -/* End of boilerplate cgo prologue. */ - -#ifdef __cplusplus -extern "C" { -#endif - - -// 登录(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutLogin(char* username, char* password); - -// 获取用户信息(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetUserMsg(char* token); - -// 获取商品模版(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetGoodsTplMsg(char* token, char* itemId, char* proxy); - -// 获取商品列表-已登的店铺(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetGoodsListMsgFromSelfShop(char* token, char* proxy, char* itemSn, char* priceMin, char* priceMax, int startCreateTime, int endCreateTime, char* requestType, int isItemSnEqual, int page, int size); - -// 删除商品-已登的店铺(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutDelGoodsFromSelfShop(char* token, char* proxy, char* itemId); - -// 新增商品(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutAddGoods(char* token, char* proxy, char* formData); - -// 获取图片URL(官图和拍图)带有店铺过滤 -// -extern __declspec(dllexport) char* OutGetImageFilterShopId(char* token, char* isbn, int shopId, char* proxy, int isLiveImage, int isReturnMsg); - -// 获取商品图片(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetImageByIsbn(char* token, char* isbn, char* proxy, int isLiveImage, int isReturnMsg); - -// 获取商品列表通过店铺ID(带有Out的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetGoodsListMsgByShopId(int shopId, char* proxy, int retPrice, int isImage, char* sortType, char* sort, float priceMin, float priceMax, int pageNum, int returnNum); - -// 获取商品信息通过商品详情链接(带有0ut的都非官方标准接口) -// -extern __declspec(dllexport) char* OutGetGoodsMsgByDetailUrl(char* detailUrl, char* proxy); - -// 获取销量榜商品列表(带有Out的都非官放标准接口) -// -extern __declspec(dllexport) char* OutGetTopGoodsListMsg(int catId, char* proxy); -extern __declspec(dllexport) char* Initialize(char* configJSON); - -// 导出函数：释放C字符串内存 -// -extern __declspec(dllexport) void FreeCString(char* str); - -#ifdef __cplusplus -} -#endif diff --git a/so/kongfz.so b/so/kongfz.so index 4d66ead..aef1ad3 100644 Binary files a/so/kongfz.so and b/so/kongfz.so differ diff --git a/dll/proxy.dll b/so/pdd.so similarity index 53% rename from dll/proxy.dll rename to so/pdd.so index c8e9413..88e1a7c 100644 Binary files a/dll/proxy.dll and b/so/pdd.so differ diff --git a/dll/kongfz.dll b/so/xy.so similarity index 62% rename from dll/kongfz.dll rename to so/xy.so index ae9f169..4cc98ef 100644 Binary files a/dll/kongfz.dll and b/so/xy.so differ diff --git a/zjdydll.go b/zjdydll.go deleted file mode 100644 index 849e037..0000000 --- a/zjdydll.go +++ /dev/null @@ -1,1237 +0,0 @@ -package main - -import "C" -import ( - "encoding/json" - "fmt" - "log" - "net/http" - "os" - "path/filepath" - "syscall" - "unsafe" -) - -// 配置结构 -type Configs struct { - App struct { - MaxRetryTimes int `json:"max_retry_times"` - RateLimitDelay int `json:"rate_limit_delay"` - Size int `json:"size"` - DefaultUserAgent string `json:"default_user_agent"` - } `json:"app"` - - API struct { - LoginURL string `json:"login_url"` - BookSearchURL string `json:"book_search_url"` - ProductSearchURL string `json:"product_search_url"` - } `json:"api"` - - Proxy struct { - Servers string `json:"servers"` - Username string `json:"username"` - Password string `json:"password"` - TailMachineCode string `json:"tail_machine_code"` - TailCardKey string `json:"tail_card_key"` - ProxyFilePath string `json:"proxy_file_path"` - } `json:"proxy"` -} - -// API响应结构 -type APIResponses struct { - Success bool `json:"success"` - Message string `json:"message,omitempty"` - GoodsNum string `json:"goods_num,omitempty"` - PNum string `json:"pnum,omitempty"` - Data interface{} `json:"data,omitempty"` - Error string `json:"error,omitempty"` -} - -// 加载DLL的函数 -type kongfzDLL struct { - dll *syscall.DLL - outLogin *syscall.Proc // 孔网登录 - outGetUserMsg *syscall.Proc // 获取孔网用户信息 - outGetGoodsTplMsg *syscall.Proc // 获取商品模版-已登的店铺 - outGetGoodsListMsgFromSelfShop *syscall.Proc // 获取商品列表-已登的店铺 - outAddGoods *syscall.Proc // 新增商品-已登的店铺 - outDelGoodsFromSelfShop *syscall.Proc // 删除商品-已登的店铺 - outGetImageFilterShopId *syscall.Proc // 获取孔网商品图片(官图和拍图)-带有店铺过滤 - outGetImageByIsbn *syscall.Proc // 获取孔网商品图片和信息(官图和拍图) - outGetGoodsListMsgByShopId *syscall.Proc // 获取商品列表通过店铺ID - outGetGoodsMsgByDetailUrl *syscall.Proc // 获取商品信息通过商品详情链接 - outGetTopGoodsListMsg *syscall.Proc // 获取销量榜商品列表 - freeCString *syscall.Proc // 释放C字符串内存 -} - -// 初始化kongfzDLL -func InitKongfzDLL() (*kongfzDLL, error) { - dllPath := filepath.Join("dll", "kongfz.dll") - if _, err := os.Stat(dllPath); os.IsNotExist(err) { - return nil, fmt.Errorf("kongfz DLL 不存在: %s", dllPath) - } - if dll, err := syscall.LoadDLL(dllPath); err != nil { - return nil, fmt.Errorf("加载 kongfz DLL 失败: %v", err) - } else { - return &kongfzDLL{ - dll: dll, - outLogin: dll.MustFindProc("OutLogin"), - outGetUserMsg: dll.MustFindProc("OutGetUserMsg"), - outGetGoodsTplMsg: dll.MustFindProc("OutGetGoodsTplMsg"), - outGetGoodsListMsgFromSelfShop: dll.MustFindProc("OutGetGoodsListMsgFromSelfShop"), - outAddGoods: dll.MustFindProc("OutAddGoods"), - outDelGoodsFromSelfShop: dll.MustFindProc("OutDelGoodsFromSelfShop"), - outGetImageFilterShopId: dll.MustFindProc("OutGetImageFilterShopId"), - outGetImageByIsbn: dll.MustFindProc("OutGetImageByIsbn"), - outGetGoodsListMsgByShopId: dll.MustFindProc("OutGetGoodsListMsgByShopId"), - outGetGoodsMsgByDetailUrl: dll.MustFindProc("OutGetGoodsMsgByDetailUrl"), - outGetTopGoodsListMsg: dll.MustFindProc("OutGetTopGoodsListMsg"), - freeCString: dll.MustFindProc("FreeCString"), - }, nil - } -} - -// cStr 获取C字符串 -func (m *kongfzDLL) cStr(p uintptr) string { - if p == 0 { - return "" - } - b := []byte{} - for i := uintptr(0); ; i++ { - c := *(*byte)(unsafe.Pointer(p + i)) - if c == 0 { - break - } - b = append(b, c) - } - s := string(b) - if m.freeCString != nil { - m.freeCString.Call(p) - } - return s -} - -type DLLManager struct { - dll *syscall.DLL -} - -func NewDLLManager(dllPath string) (*DLLManager, error) { - dll, err := syscall.LoadDLL(dllPath) - if err != nil { - return nil, fmt.Errorf("加载DLL失败: %v", err) - } - - return &DLLManager{dll: dll}, nil -} - -func (m *DLLManager) Close() { - if m.dll != nil { - m.dll.Release() - } -} - -// 单参数函数调用 -func (m *DLLManager) callFunction(funcName string, arg string) (string, error) { - proc, err := m.dll.FindProc(funcName) - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", funcName, err) - } - - argPtr, _ := syscall.BytePtrFromString(arg) - r1, _, err := proc.Call(uintptr(unsafe.Pointer(argPtr))) - if err != nil && err.Error() != "The operation completed successfully." { - return "", fmt.Errorf("调用函数 %s 失败: %v", funcName, err) - } - - result := (*byte)(unsafe.Pointer(r1)) - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i)))) - } - - // 释放内存 - freeProc, _ := m.dll.FindProc("FreeString") - if freeProc != nil { - freeProc.Call(r1) - } - - return string(resultBytes), nil -} - -// 二参数函数调用 -func (m *DLLManager) callFunctionTwoArgs(funcName string, args ...string) (string, error) { - if len(args) != 2 { - return "", fmt.Errorf("函数 %s 需要2个参数，但提供了 %d 个", funcName, len(args)) - } - - proc, err := m.dll.FindProc(funcName) - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", funcName, err) - } - - // 准备参数指针 - argPtrs := make([]uintptr, 2) - for i, arg := range args { - argPtr, _ := syscall.BytePtrFromString(arg) - argPtrs[i] = uintptr(unsafe.Pointer(argPtr)) - } - - r1, _, err := proc.Call( - argPtrs[0], // proxyType - argPtrs[1], // username - ) - if err != nil && err.Error() != "The operation completed successfully." { - return "", fmt.Errorf("调用函数 %s 失败: %v", funcName, err) - } - - result := (*byte)(unsafe.Pointer(r1)) - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i)))) - } - - // 释放内存 - freeProc, _ := m.dll.FindProc("FreeString") - if freeProc != nil { - freeProc.Call(r1) - } - - return string(resultBytes), nil -} - -// 五参数函数调用 - 用于 GetKFZGTImageURL 和 GetKFZSPTImageURL -func (m *DLLManager) callFunctionFiveArgs(funcName string, args ...string) (string, error) { - if len(args) != 5 { - return "", fmt.Errorf("函数 %s 需要5个参数，但提供了 %d 个", funcName, len(args)) - } - - proc, err := m.dll.FindProc(funcName) - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", funcName, err) - } - - // 准备参数指针 - argPtrs := make([]uintptr, 5) - for i, arg := range args { - argPtr, _ := syscall.BytePtrFromString(arg) - argPtrs[i] = uintptr(unsafe.Pointer(argPtr)) - } - - r1, _, err := proc.Call( - argPtrs[0], // proxyType - argPtrs[1], // username - argPtrs[2], // password - argPtrs[3], // machineCode - argPtrs[4], // isbn - ) - if err != nil && err.Error() != "The operation completed successfully." { - return "", fmt.Errorf("调用函数 %s 失败: %v", funcName, err) - } - - result := (*byte)(unsafe.Pointer(r1)) - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i)))) - } - - // 释放内存 - freeProc, _ := m.dll.FindProc("FreeString") - if freeProc != nil { - freeProc.Call(r1) - } - - return string(resultBytes), nil -} - -// 六参数函数调用 -func (m *DLLManager) callFunctionSixArgs(funcName string, args ...string) (string, error) { - if len(args) != 6 { - return "", fmt.Errorf("函数 %s 需要6个参数，但提供了 %d 个", funcName, len(args)) - } - - proc, err := m.dll.FindProc(funcName) - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", funcName, err) - } - - // 准备参数指针 - argPtrs := make([]uintptr, 6) - for i, arg := range args { - argPtr, _ := syscall.BytePtrFromString(arg) - argPtrs[i] = uintptr(unsafe.Pointer(argPtr)) - } - - r1, _, err := proc.Call( - argPtrs[0], // fetchMode - argPtrs[1], // proxyType - argPtrs[2], // username - argPtrs[3], // password - argPtrs[4], // machineCode - argPtrs[5], // isbn - ) - if err != nil && err.Error() != "The operation completed successfully." { - return "", fmt.Errorf("调用函数 %s 失败: %v", funcName, err) - } - - result := (*byte)(unsafe.Pointer(r1)) - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i)))) - } - - // 释放内存 - freeProc, _ := m.dll.FindProc("FreeString") - if freeProc != nil { - freeProc.Call(r1) - } - - return string(resultBytes), nil -} - -// 十二参数函数调用 - 专门用于 GetKFZShopBookInfo -func (m *DLLManager) callFunctionTwelveArgs(funcName string, args ...string) (string, error) { - if len(args) != 12 { - return "", fmt.Errorf("函数 %s 需要12个参数，但提供了 %d 个", funcName, len(args)) - } - - proc, err := m.dll.FindProc(funcName) - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", funcName, err) - } - - // 准备参数指针 - argPtrs := make([]uintptr, 12) - for i, arg := range args { - argPtr, _ := syscall.BytePtrFromString(arg) - argPtrs[i] = uintptr(unsafe.Pointer(argPtr)) - } - - r1, _, err := proc.Call( - argPtrs[0], // proxyType - argPtrs[1], // username - argPtrs[2], // password - argPtrs[3], // machineCode - argPtrs[4], // shopId - argPtrs[5], // isImage - argPtrs[6], // bookNum - argPtrs[7], // pageNum - argPtrs[8], // sortType - argPtrs[9], // sort - argPtrs[10], // priceDown - argPtrs[11], // priceUp - ) - if err != nil && err.Error() != "The operation completed successfully." { - return "", fmt.Errorf("调用函数 %s 失败: %v", funcName, err) - } - - result := (*byte)(unsafe.Pointer(r1)) - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i)))) - } - - // 释放内存 - freeProc, _ := m.dll.FindProc("FreeString") - if freeProc != nil { - freeProc.Call(r1) - } - - return string(resultBytes), nil -} - -// 十三参数函数调用 - 专门用于 GetKFZShopBookInfo -func (m *DLLManager) callFunctionThirteenArgs(funcName string, args ...string) (string, error) { - if len(args) != 13 { - return "", fmt.Errorf("函数 %s 需要13个参数，但提供了 %d 个", funcName, len(args)) - } - - proc, err := m.dll.FindProc(funcName) - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", funcName, err) - } - - // 准备参数指针 - argPtrs := make([]uintptr, 13) - for i, arg := range args { - argPtr, _ := syscall.BytePtrFromString(arg) - argPtrs[i] = uintptr(unsafe.Pointer(argPtr)) - } - - r1, _, err := proc.Call( - argPtrs[0], // fetchMode - argPtrs[1], // proxyType - argPtrs[2], // username - argPtrs[3], // password - argPtrs[4], // machineCode - argPtrs[5], // shopId - argPtrs[6], // isImage - argPtrs[7], // bookNum - argPtrs[8], // pageNum - argPtrs[9], // sortType - argPtrs[10], // sort - argPtrs[11], // priceDown - argPtrs[12], // priceUp - ) - if err != nil && err.Error() != "The operation completed successfully." { - return "", fmt.Errorf("调用函数 %s 失败: %v", funcName, err) - } - - result := (*byte)(unsafe.Pointer(r1)) - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i)))) - } - - // 释放内存 - freeProc, _ := m.dll.FindProc("FreeString") - if freeProc != nil { - freeProc.Call(r1) - } - - return string(resultBytes), nil -} - -func (m *DLLManager) Initialize(configJSON string) (string, error) { - return m.callFunction("Initialize", configJSON) -} - -// 修改后的 GetKFZGTImageURL - 需要5个参数 -func (m *DLLManager) GetKFZGTImageURL(proxyType, username, password, machineCode, isbn string) (string, error) { - return m.callFunctionFiveArgs("GetKFZGTImageURL", - proxyType, username, password, machineCode, isbn) -} - -// 修改后的 GetKFZSPTImageURL - 需要5个参数 -func (m *DLLManager) GetKFZSPTImageURL(proxyType, username, password, machineCode, isbn string) (string, error) { - return m.callFunctionFiveArgs("GetKFZSPTImageURL", - proxyType, username, password, machineCode, isbn) -} - -// 修改后的 GetKFZShopBookInfo - 需要12个参数 -func (m *DLLManager) GetKFZShopBookInfo(fetchMode, proxyType, username, password, machineCode, shopId, isImage, bookNum, pageNum, sortType, sort, priceDown, priceUp string) (string, error) { - return m.callFunctionThirteenArgs("GetKFZShopBookInfo", - fetchMode, proxyType, username, password, machineCode, shopId, isImage, bookNum, pageNum, sortType, sort, priceDown, priceUp) -} - -// 获取 GetUrlBookDetails - 需要6个参数 -func (m *DLLManager) GetUrlBookDetails(fetchMode, proxyType, username, password, machineCode, url string) (string, error) { - return m.callFunctionSixArgs("GetUrlBookDetails", fetchMode, proxyType, username, password, machineCode, url) -} - -// 获取 OutLogin -func (m *DLLManager) OutLogin(username, password string) (string, error) { - // 获取函数 - proc, err := m.dll.FindProc("OutLogin") - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", "OutLogin", err) - } - // 转换参数 - usernamePtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(username))) - passwordPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(password))) - // 调用函数 - ret, _, err := proc.Call(usernamePtr, passwordPtr) - if ret == 0 { - return "", fmt.Errorf("DLL调用失败: %v", err) - } - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i)))) - } - - // 释放内存（假设DLL提供了FreeMemory函数） - findProc, err := m.dll.FindProc("FreeString") - if findProc != nil { - findProc.Call(ret) - } - return string(resultBytes), nil -} - -// 获取 OutGetUserMsg -func (m *DLLManager) OutGetUserMsg(token string) (string, error) { - // 获取函数 - proc, err := m.dll.FindProc("OutGetUserMsg") - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", "OutGetUserMsg", err) - } - // 转换参数 - tokenPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(token))) - // 调用函数 - ret, _, err := proc.Call(tokenPtr) - if ret == 0 { - return "", fmt.Errorf("DLL调用失败: %v", err) - } - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i)))) - } - - // 释放内存（假设DLL提供了FreeMemory函数） - findProc, err := m.dll.FindProc("FreeString") - if findProc != nil { - findProc.Call(ret) - } - return string(resultBytes), nil -} - -// 获取 OutGetGoodsTplMsg -func (m *DLLManager) OutGetGoodsTplMsg(token, itemId, proxy string) (string, error) { - // 获取函数 - proc, err := m.dll.FindProc("OutGetGoodsTplMsg") - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", "OutGetGoodsTplMsg", err) - } - // 转换参数 - tokenPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(token))) - itemIdPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(itemId))) - proxyPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(proxy))) - // 调用函数 - ret, _, err := proc.Call(tokenPtr, itemIdPtr, proxyPtr) - if ret == 0 { - return "", fmt.Errorf("DLL调用失败: %v", err) - } - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i)))) - } - - // 释放内存（假设DLL提供了FreeMemory函数） - findProc, err := m.dll.FindProc("FreeString") - if findProc != nil { - findProc.Call(ret) - } - return string(resultBytes), nil -} - -// 获取 OutGetGoodsListMsgFromSelfShop -func (m *DLLManager) OutGetGoodsListMsgFromSelfShop(token string, proxy string, itemSn string, priceMin string, priceMax string, startCreateTime string, - endCreateTime string, requestType string, isItemSnEqual int, page int, size int) (string, error) { - // 获取函数 - proc, err := m.dll.FindProc("OutGetGoodsListMsgFromSelfShop") - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", "OutGetGoodsListMsgFromSelfShop", err) - } - // 转换参数 - tokenPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(token))) - proxyPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(proxy))) - itemSnPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(itemSn))) - priceMinPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(priceMin))) - priceMaxPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(priceMax))) - startCreateTimePtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(startCreateTime))) - endCreateTimePtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(endCreateTime))) - requestTypePtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(requestType))) - isItemSnEqualPtr := uintptr(isItemSnEqual) - pagePtr := uintptr(page) - sizePtr := uintptr(size) - // 调用函数 - ret, _, err := proc.Call(tokenPtr, proxyPtr, itemSnPtr, priceMinPtr, priceMaxPtr, startCreateTimePtr, endCreateTimePtr, requestTypePtr, isItemSnEqualPtr, pagePtr, sizePtr) - if ret == 0 { - return "", fmt.Errorf("DLL调用失败: %v", err) - } - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i)))) - } - - // 释放内存（假设DLL提供了FreeMemory函数） - findProc, err := m.dll.FindProc("FreeString") - if findProc != nil { - findProc.Call(ret) - } - return string(resultBytes), nil -} - -// 获取 OutGetImageByIsbn -func (m *DLLManager) OutGetImageByIsbn(token string, isbn string, proxy string, isLiveImage bool, isReturnMsg bool) (string, error) { - // 获取函数 - proc, err := m.dll.FindProc("OutGetImageByIsbn") - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", "OutGetImageByIsbn", err) - } - // 转换参数 - tokenPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(token))) - isbnPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(isbn))) - var proxyPtr uintptr - if proxy != "" { - proxyPtr = uintptr(unsafe.Pointer(syscall.StringBytePtr(proxy))) - } else { - proxyPtr = 0 - } - var isLiveImageInt int - if isLiveImage { - isLiveImageInt = 0 - } else { - isLiveImageInt = 1 - } - isLiveImagePtr := uintptr(isLiveImageInt) - var isReturnMsgInt int - if isReturnMsg { - isLiveImageInt = 0 - } else { - isLiveImageInt = 1 - } - isReturnMsgPtr := uintptr(isReturnMsgInt) - // 调用函数 - ret, _, err := proc.Call(tokenPtr, isbnPtr, proxyPtr, isLiveImagePtr, isReturnMsgPtr) - if ret == 0 { - return "", fmt.Errorf("DLL调用失败: %v", err) - } - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i)))) - } - - // 释放内存（假设DLL提供了FreeMemory函数） - findProc, err := m.dll.FindProc("FreeString") - if findProc != nil { - findProc.Call(ret) - } - return string(resultBytes), nil -} - -// 获取 OutGetGoodsMsgByDetailUrl -func (m *DLLManager) OutGetGoodsMsgByDetailUrl(url, proxy string) (string, error) { - // 获取函数 - proc, err := m.dll.FindProc("OutGetGoodsMsgByDetailUrl") - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", "OutGetGoodsMsgByDetailUrl", err) - } - // 转换参数 - urlPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(url))) - var proxyPtr uintptr - if proxy != "" { - proxyPtr = uintptr(unsafe.Pointer(syscall.StringBytePtr(proxy))) - } else { - proxyPtr = 0 - } - // 调用函数 - ret, _, err := proc.Call(urlPtr, proxyPtr) - if ret == 0 { - return "", fmt.Errorf("DLL调用失败: %v", err) - } - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i)))) - } - - // 释放内存（假设DLL提供了FreeMemory函数） - findProc, err := m.dll.FindProc("FreeString") - if findProc != nil { - findProc.Call(ret) - } - return string(resultBytes), nil -} - -// 获取 OutGetGoodsListMsgByShopId -func (m *DLLManager) OutGetGoodsListMsgByShopId(shopId int, proxy string, isImage bool, - sortType string, sort string, priceMin float32, priceMax float32, - pageNum, returnNum int) (string, error) { - // 获取函数 - proc, err := m.dll.FindProc("OutGetGoodsListMsgByShopId") - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", "OutGetGoodsListMsgByShopId", err) - } - // 转换参数 - shopIdPtr := uintptr(shopId) - var isImageInt int - if isImage { - isImageInt = 0 - } else { - isImageInt = 1 - } - isImagePtr := uintptr(isImageInt) - sortTypePtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(sortType))) - sortPtr := uintptr(unsafe.Pointer(syscall.StringBytePtr(sort))) - priceMinPtr := uintptr(*(*uint32)(unsafe.Pointer(&priceMin))) - priceMaxPtr := uintptr(*(*uint32)(unsafe.Pointer(&priceMax))) - pageNumPtr := uintptr(pageNum) - returnNumPtr := uintptr(returnNum) - var proxyPtr uintptr - if proxy != "" { - proxyPtr = uintptr(unsafe.Pointer(syscall.StringBytePtr(proxy))) - } else { - proxyPtr = 0 - } - // 调用函数 - ret, _, err := proc.Call(shopIdPtr, proxyPtr, isImagePtr, sortTypePtr, sortPtr, priceMinPtr, priceMaxPtr, pageNumPtr, returnNumPtr) - if ret == 0 { - return "", fmt.Errorf("DLL调用失败: %v", err) - } - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i)))) - } - - // 释放内存（假设DLL提供了FreeMemory函数） - findProc, err := m.dll.FindProc("FreeString") - if findProc != nil { - findProc.Call(ret) - } - return string(resultBytes), nil -} - -// 获取 OutGetTopGoodsListMsg -func (m *DLLManager) OutGetTopGoodsListMsg(catId int, proxy string) (string, error) { - // 获取函数 - proc, err := m.dll.FindProc("OutGetTopGoodsListMsg") - if err != nil { - return "", fmt.Errorf("找不到函数 %s: %v", "OutGetTopGoodsListMsg", err) - } - // 转换参数 - catIdPtr := uintptr(catId) - var proxyPtr uintptr - if proxy != "" { - proxyPtr = uintptr(unsafe.Pointer(syscall.StringBytePtr(proxy))) - } else { - proxyPtr = 0 - } - // 调用函数 - ret, _, err := proc.Call(catIdPtr, proxyPtr) - if ret == 0 { - return "", fmt.Errorf("DLL调用失败: %v", err) - } - - // 将C字符串转换为Go字符串 - var resultBytes []byte - for i := 0; ; i++ { - if *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i))) == 0 { - break - } - resultBytes = append(resultBytes, *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(ret)) + uintptr(i)))) - } - - // 释放内存（假设DLL提供了FreeMemory函数） - findProc, err := m.dll.FindProc("FreeString") - if findProc != nil { - findProc.Call(ret) - } - return string(resultBytes), nil -} - -// 创建默认配置 -func createDefaultConfig() Config { - var configs Config - - // App配置 - configs.App.MaxRetryTimes = 3 - configs.App.RateLimitDelay = 1000 - configs.App.Size = 10 - configs.App.DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - - // API配置 - configs.API.LoginURL = "https://login.kongfz.com/Pc/Login/account" - configs.API.BookSearchURL = "https://search.kongfz.com/pc-gw/search-web/client/pc/bookLib/keyword/list" - configs.API.ProductSearchURL = "https://search.kongfz.com/pc-gw/search-web/client/pc/product/keyword/list" - - // 代理配置（需要根据实际情况修改） - configs.Proxy.Servers = "http-dynamic.xiaoxiangdaili.com,http-dynamic-S02.xiaoxiangdaili.com,http-dynamic-S03.xiaoxiangdaili.com,http-dynamic-S04.xiaoxiangdaili.com" - configs.Proxy.Username = "1297757178467602432" - configs.Proxy.Password = "QgQBvP7f" - configs.Proxy.TailMachineCode = "b7bf22a237ec692f13fcc2c43ee63252" - configs.Proxy.TailCardKey = "DL_20_YK_1920acb2129844c2aabade3896560a9b" - configs.Proxy.ProxyFilePath = "dll/proxyConfig.dll" - - configs.Database.Username = "newAdmin" - configs.Database.Password = "bYPp8SbBe5F7nz2i" - configs.Database.Host = "146.56.227.42:3306" - configs.Database.Name = "newadmin" - - return configs -} - -// 获取当前可执行文件所在目录 -func getExecutableDir() string { - exePath, err := os.Executable() - if err != nil { - return "." - } - return filepath.Dir(exePath) -} - -// 格式化输出JSON响应 -func printFormattedResponse(operation string, result string, err error) int { - var apiResp APIResponses - fmt.Printf("\n%s:\n", operation) - fmt.Println("=" + string(make([]byte, 50)) + "=") - - if err != nil { - fmt.Printf("❌ 错误: %v\n", err) - } - var count int - if err := json.Unmarshal([]byte(result), &apiResp); err == nil { - if apiResp.Success { - fmt.Printf("✅ 成功: %s\n", apiResp.Message) - if apiResp.Data != nil { - fmt.Println("📊 数据:") - jsonData, _ := json.MarshalIndent(apiResp.Data, "", " ") - fmt.Println(string(jsonData)) - // 方法1: 判断 Data 的类型并统计条数 - switch data := apiResp.Data.(type) { - case []interface{}: - // 如果是切片/数组 - count = len(data) - fmt.Printf("📈 数据条数: %d\n", count) - case []BookInfo: - // 如果是 BookInfo 切片 - count = len(data) - fmt.Printf("📚 图书数量: %d\n", count) - case map[string]interface{}: - // 如果是映射 - count = len(data) - fmt.Printf("🗂️ 数据字段数: %d\n", count) - case []ProductInfo: - // 如果是 ProductInfo 切片 - count = len(data) - fmt.Printf("🛒 商品数量: %d\n", count) - default: - fmt.Printf("ℹ️ 数据类型: %T\n", apiResp.Data) - } - } - if apiResp.GoodsNum != "" { - fmt.Printf("📦 商品数量: %s\n", apiResp.GoodsNum) - } - if apiResp.PNum != "" { - fmt.Printf("📄 页码: %s\n", apiResp.PNum) - } - } else { - fmt.Printf("❌ 失败: %s\n", apiResp.Error) - } - } else { - fmt.Printf("原始响应: %s\n", result) - // 如果是代理地址，可以进一步处理 - if len(result) > 0 && (result[0:4] == "http" || result[0:3] == "socks") { - fmt.Printf("✅ 代理服务器地址获取成功\n") - fmt.Printf("🔗 代理地址: %s\n", result) - } - } - return count -} - -// 专门处理代理配置响应的函数 -func handleProxyResponse(result string, err error) { - fmt.Printf("\n代理配置测试:\n") - fmt.Println("=" + string(make([]byte, 50)) + "=") - - if err != nil { - fmt.Printf("❌ 错误: %v\n", err) - return - } - - // 代理配置通常返回代理服务器地址，不是JSON格式 - fmt.Printf("📡 代理服务器地址: %s\n", result) - fmt.Printf("✅ 代理配置获取成功\n") - - // 如果需要，可以在这里进一步处理代理地址 - // 例如：验证代理格式、提取IP和端口等 - if result != "" { - fmt.Printf("🔗 可用代理: %s\n", result) - } -} - -// ProxyTypeManager 调用代理类型管理器 -func (m *DLLManager) ProxyTypeManager(proxyType, username, password, machineCode string) (string, error) { - proc, err := m.dll.FindProc("ProxyTypeManager") - if err != nil { - return "", fmt.Errorf("找不到函数 ProxyTypeManager: %v", err) - } - - // 准备参数 - proxyTypePtr, _ := syscall.BytePtrFromString(proxyType) - usernamePtr, _ := syscall.BytePtrFromString(username) - passwordPtr, _ := syscall.BytePtrFromString(password) - machineCodePtr, _ := syscall.BytePtrFromString(machineCode) - - // 调用函数 - r1, _, err := proc.Call( - uintptr(unsafe.Pointer(proxyTypePtr)), - uintptr(unsafe.Pointer(usernamePtr)), - uintptr(unsafe.Pointer(passwordPtr)), - uintptr(unsafe.Pointer(machineCodePtr)), - ) - - if err != nil && err.Error() != "The operation completed successfully." { - return "", fmt.Errorf("调用 ProxyTypeManager 失败: %v", err) - } - - // 转换结果 - result := (*byte)(unsafe.Pointer(r1)) - var resultBytes []byte - for i := 0; ; i++ { - bytePtr := (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(result)) + uintptr(i))) - if *bytePtr == 0 { - break - } - resultBytes = append(resultBytes, *bytePtr) - } - - // 释放内存 - freeProc, _ := m.dll.FindProc("FreeString") - if freeProc != nil { - freeProc.Call(r1) - } - - return string(resultBytes), nil -} - -// FreeCString 释放C字符串内存 -func (m *DLLManager) FreeCString(strPtr uintptr) error { - proc, err := m.dll.FindProc("FreeCString") - if err != nil { - return fmt.Errorf("找不到函数 FreeCString: %v", err) - } - - _, _, err = proc.Call(strPtr) - if err != nil && err.Error() != "The operation completed successfully." { - return fmt.Errorf("调用 FreeCString 失败: %v", err) - } - - return nil -} - -func handleGetKFZShopBookInfo(w http.ResponseWriter, r *http.Request) { - // 加载DLL - manager, err := NewDLLManager("dll/kongfz.dll") - if err != nil { - fmt.Printf("初始化DLL管理器失败: %v", err) - } - //defer manager.Close() - - log.Println("✅ DLL加载成功") - - // 使用默认配置初始化 - config := createDefaultConfig() - initializeConfig(config) - configJSON, err := json.Marshal(config) - if err != nil { - fmt.Printf("序列化配置失败: %v", err) - } - - result, err := manager.Initialize(string(configJSON)) - if err != nil { - fmt.Printf("初始化失败: %v", err) - } - - var initResp APIResponses - if err := json.Unmarshal([]byte(result), &initResp); err != nil { - fmt.Printf("解析初始化响应失败: %v", err) - } - - if !initResp.Success { - fmt.Printf("初始化失败: %s", initResp.Message) - } - - log.Println("✅ DLL初始化成功") - // 设置响应头 - w.Header().Set("Content-Type", "application/json; charset=utf-8") - - // 只支持GET请求 - if r.Method != http.MethodGet { - sendErrorResponse(w, http.StatusMethodNotAllowed, "只支持GET方法") - return - } - - //// 获取代理信息 - //typeManager, err := manager.ProxyTypeManager("CALF_ELEPHANT_PROXY", "1297757178467602432", "QgQBvP7f", "") - //if err != nil { - // fmt.Printf(err.Error()) - //} - //fmt.Println(typeManager) - - // - //user, err := manager.OutLogin("18904056801", "Long6166@@") - //if err != nil { - // fmt.Printf(err.Error()) - //} - //fmt.Println(user) - // - //var data APIResponse - //if err := json.Unmarshal([]byte(user), &data); err != nil { - // log.Printf("[ERROR] 解析第 %d 页失败: %v", data, err) - //} - //var token string - //// 从 Data 中提取 token - //if dataMap, ok := data.Data.(map[string]interface{}); ok { - // if tk, exists := dataMap["token"]; exists { - // fmt.Println("Token:", tk) - // token = tk.(string) - // // Token: d322c80960ab6722922912dd2ce219d4b4099d54 - // } else { - // fmt.Println("Token 不存在") - // } - //} else { - // fmt.Println("Data 格式不正确") - //} - //fmt.Println(token) - - //msg, err := manager.OutGetUserMsg(token) - //if err != nil { - // fmt.Printf(err.Error()) - //} - //fmt.Println(msg) - // - //tplMsg, err := manager.OutGetGoodsTplMsg(token, "9142583516", "") - //if err != nil { - // fmt.Printf(err.Error()) - //} - //fmt.Println("tplMsg: ", tplMsg) - // - //shop, err := manager.OutGetGoodsListMsgFromSelfShop("a0c75646dfefc23a9c387865d43197283192df0c", "", - // "", "", "", "", "", "allUnSold", 0, 1, 100) - //if err != nil { - // fmt.Printf(err.Error()) - //} - //fmt.Println(shop) - - // 获取孔网图片 - fmt.Println("获取孔网图片") - isbnss, err := manager.OutGetImageByIsbn("81cc0e948433343309dcf3bfe100a802803934e5", "9787020106752", "", true, false) - if err != nil { - fmt.Printf(err.Error()) - } - fmt.Println(isbnss) - - var api struct { - Success bool `json:"success"` - Data *BookInfo `json:"data"` - } - err = json.Unmarshal([]byte(isbnss), &api) - if err != nil { - fmt.Printf(err.Error()) - } - fmt.Println(api.Data.BookPic) - //// 获取cookie - //cookie, err := loginCookie() - //if err != nil { - // fmt.Printf(err.Error()) - //} - //fmt.Println(cookie) - - //account, err := getRandomAccount() - //if err != nil { - // fmt.Printf(err.Error()) - //} - //fmt.Println(account) - - //var isbns = []string{ - // "9787500601593", "9787506331746", "9787020106752", "9787807089353", "9787536692930", "9787530221532", "9787544270878", "9787208061644", "9787506365437", "9787513708371", "9787513922135", "9787536693968", "9787541151736", "9787544213561", "9787544754781", "9787549204304", "9787108006639", "9787531764700", "9787544253994", "9787540456023", "9787540456429", "9787544267618", "9787544277723", "9787550008496", "9787806070680", "9787806801529", "9787807530244", "9787020008728", "9787020139590", "9787204055401", "9787500680239", "9787505724778", "9787506365680", "9787530221525", "9787536097261", - //} - // - //for _, isbn := range isbns { - // // 获取孔网图片 - // isbnss, err := outGetImageByIsbn("", isbn, "", 0, 0) - // if err != nil { - // fmt.Printf(err.Error()) - // } - // fmt.Println(isbnss) - //} - - //books, num, pNum, err := outGetGoodsListMsgByShopId(3092, - // typeManager, true, "", - // "", 0, 14, 1, 5) - - //msg, err := manager.OutGetTopGoodsListMsg(0, "") - //if err != nil { - // - //} - //fmt.Println(msg) - //fetchMode := r.URL.Query().Get("fetchMode") - //proxyType := r.URL.Query().Get("proxyType") - //username := r.URL.Query().Get("username") - //password := r.URL.Query().Get("password") - //machineCode := r.URL.Query().Get("machineCode") - //url := r.URL.Query().Get("url") - //shopId := r.URL.Query().Get("shopId") //https://shop.kongfz.com/793900/all/0_100_0_0_1_sort_desc_0_0/ - //isImage := r.URL.Query().Get("isImage") - //bookNum := r.URL.Query().Get("bookNum") - //pageNum := r.URL.Query().Get("pageNum") - //sortType := r.URL.Query().Get("sortType") - //sort := r.URL.Query().Get("sort") - //priceDown := r.URL.Query().Get("priceDown") - //priceUp := r.URL.Query().Get("priceUp") - - //details, err := manager.OutGetGoodsMsgByDetailUrl("https://book.kongfz.com/151391/6531924071/", "") - //if err != nil { - // - //} - //fmt.Println(details) - - //id, err := manager.OutGetGoodsListMsgByShopId(1181761, - // "", true, "", - // "", 0, 0, 1, 5) - //if err != nil { - // fmt.Println(err) - //} - //fmt.Println(id) - //var pa ShopBookResult - //if err := json.Unmarshal([]byte(id), &pa); err != nil { - // log.Printf("[ERROR] 解析第 %d 页失败: %v", pa, err) - //} - //for _, urls := range pa.Data.Data { - // ss, err := proxyTypeManager("CALF_ELEPHANT_PROXY", "1297757178467602432", "QgQBvP7f", "") - // if err != nil { - // fmt.Printf(err.Error()) - // } - // fmt.Println(ss) - // url, err := manager.OutGetGoodsMsgByDetailUrl(urls.DetailURL, ss) - // if err != nil { - // fmt.Println(err) - // } - // fmt.Println(url) - //} - - //books, num, pNum, err := outGetGoodsListMsgByShopId(788247, - // typeManager, 1, 0, "", - // "", 0, 0, 1, 5) - // - //fmt.Println(books) - //fmt.Println(num) - //fmt.Println(pNum) - //fmt.Println(err) - //for _, book := range books { - // ss, err := proxyTypeManager("TAIL_PROXY", "", "", "07f4d0fbcff99966c2b37b0c1fb7f01c") - // if err != nil { - // fmt.Printf(err.Error()) - // } - // fmt.Println(ss) - // url, err := outGetGoodsMsgByDetailUrl(book.DetailUrl, ss) - // if err != nil { - // fmt.Println(err) - // } - // fmt.Println(url) - //} - -} - -func manegiyoutGetGoodsListMsgFromSelfShop() { - -} - -// 发送错误响应 -func sendErrorResponse(w http.ResponseWriter, statusCode int, message string) { - response := APIResp{ - Success: false, - Message: message, - } - w.WriteHeader(statusCode) - json.NewEncoder(w).Encode(response) -} - -type ShopBookResult struct { - Success bool `json:"success"` - Message string `json:"message"` - Data struct { - GoodsNum string `json:"goods_num"` - Pnum string `json:"pnum"` - Data []struct { - BookName string `json:"book_name"` - Author string `json:"author"` - Publisher string `json:"publisher"` - ISBN string `json:"isbn"` - PublishTime int64 `json:"publication_time"` - Edition string `json:"edition"` - PrintTime string `json:"print_time"` - FixPrice string `json:"fix_price"` - BindingLayout string `json:"binding_layout"` - Format string `json:"format"` - Paper string `json:"paper"` - Pages string `json:"pages"` - Wordage string `json:"wordage"` - Languages string `json:"languages"` - Era string `json:"era"` - EngravingMethod string `json:"engraving_method"` - Dimensions string `json:"dimensions"` - VolumeNumber string `json:"volume_number"` - BookPic string `json:"book_pic"` - BookPicS string `json:"book_pic_s"` - SellingPrice string `json:"selling_price"` - Condition string `json:"condition"` - ExpressDeliveryFee string `json:"express_delivery_fee"` - Editor string `json:"editor"` - Category string `json:"category"` - BuyCount string `json:"buy_count"` - SellCount string `json:"sell_count"` - Content string `json:"content"` - Mid int `json:"mid"` - ItemID int64 `json:"item_id"` - ShopID int `json:"shop_id"` - DetailURL string `json:"detail_url"` - } `json:"data"` - } `json:"data"` -} - -func main() { - http.HandleFunc("/api/kfzShopBookInfo", handleGetKFZShopBookInfo) - port := "8989" - server := &http.Server{ - Addr: ":" + port, - Handler: nil, - } - // 启动服务器 - if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - fmt.Printf("服务器启动失败: %v\n", err) - } -}