package main import ( "database/sql" "fmt" "log" "strings" "time" _ "github.com/go-sql-driver/mysql" ) const ( MaxOpenConns = 20 MaxIdleConns = 10 ConnMaxLifetime = 10 * time.Minute ConnMaxIdleTime = 5 * time.Minute ) // 雪花算法ID生成器 type Snowflake struct { sequence int64 lastTimestamp int64 } var snowflake = &Snowflake{} // 生成19位雪花算法ID func (s *Snowflake) GenerateID() int64 { const ( epoch int64 = 1704067200000 // 2024-01-01 00:00:00 machineID int64 = 1 sequenceMask int64 = 4095 timestampShift = 22 machineIDShift = 12 ) now := time.Now().UnixNano() / 1e6 timestamp := now - epoch if timestamp == s.lastTimestamp { s.sequence = (s.sequence + 1) & sequenceMask if s.sequence == 0 { for timestamp <= s.lastTimestamp { time.Sleep(100 * time.Microsecond) now = time.Now().UnixNano() / 1e6 timestamp = now - epoch } } } else { s.sequence = 0 } s.lastTimestamp = timestamp id := (timestamp << timestampShift) | (machineID << machineIDShift) | s.sequence return id } func main() { fmt.Println("========================================") fmt.Println(" ISBN数据迁移工具") fmt.Println("========================================") fmt.Println() // 连接源数据库 (Master) fmt.Println("正在连接源数据库 (Master)...") dsnMaster := "zhishu:XsRR4K3ATizyc5BK@tcp(146.56.227.42:3306)/zhishu?charset=utf8mb4&parseTime=True&loc=Local&timeout=5s" dbMaster, err := sql.Open("mysql", dsnMaster) if err != nil { log.Fatalf("❌ 打开源数据库连接失败: %v", err) } defer dbMaster.Close() dbMaster.SetMaxOpenConns(MaxOpenConns) dbMaster.SetMaxIdleConns(MaxIdleConns) dbMaster.SetConnMaxLifetime(ConnMaxLifetime) dbMaster.SetConnMaxIdleTime(ConnMaxIdleTime) err = dbMaster.Ping() if err != nil { log.Fatalf("❌ 连接源数据库失败: %v", err) } fmt.Println("✅ 源数据库连接成功") // 连接目标数据库 (Slave) fmt.Println("正在连接目标数据库 (Slave)...") dsnSlave := "zhishu_slave:7DpixiEdCs5p3PEr@tcp(36.212.12.247:3306)/zhishu_slave?charset=utf8mb4&parseTime=True&loc=Local&timeout=5s" dbSlave, err := sql.Open("mysql", dsnSlave) if err != nil { log.Fatalf("❌ 打开目标数据库连接失败: %v", err) } defer dbSlave.Close() dbSlave.SetMaxOpenConns(MaxOpenConns) dbSlave.SetMaxIdleConns(MaxIdleConns) dbSlave.SetConnMaxLifetime(ConnMaxLifetime) dbSlave.SetConnMaxIdleTime(ConnMaxIdleTime) err = dbSlave.Ping() if err != nil { log.Fatalf("❌ 连接目标数据库失败: %v", err) } fmt.Println("✅ 目标数据库连接成功\n") // 从源数据库查询数据 fmt.Println("========================================") fmt.Println(" 步骤1: 从源数据库读取数据") fmt.Println("========================================") query := `SELECT isbn, create_time FROM shop_goods_rejection WHERE rejection_reason = '您发布的商品涉嫌违规,请您检查相关商品并立即整改' ORDER BY create_time ASC` startTime := time.Now() rows, err := dbMaster.Query(query) if err != nil { log.Fatalf("❌ 查询源数据失败: %v", err) } defer rows.Close() type Record struct { ISBN string CreateTime time.Time } var records []Record for rows.Next() { var r Record var isbn sql.NullString var createTime sql.NullTime if err := rows.Scan(&isbn, &createTime); err != nil { log.Printf("读取数据失败: %v", err) continue } if isbn.Valid && isbn.String != "" { r.ISBN = isbn.String if createTime.Valid { r.CreateTime = createTime.Time } records = append(records, r) } } elapsed := time.Since(startTime) fmt.Printf("查询完成,耗时: %v\n", elapsed) fmt.Printf("共读取到 %d 条记录\n\n", len(records)) if len(records) == 0 { fmt.Println("没有需要迁移的数据") return } // 查询目标表中已有的ISBN(去重) fmt.Println("========================================") fmt.Println(" 步骤2: 检查目标表已有数据") fmt.Println("========================================") existingISBNs := make(map[string]bool) existingRows, err := dbSlave.Query("SELECT add_txt FROM t_filter_set WHERE limitation_type = '0'") if err != nil { log.Printf("⚠️ 查询已有数据失败(表可能为空): %v", err) } else { for existingRows.Next() { var isbn string if err := existingRows.Scan(&isbn); err == nil { existingISBNs[isbn] = true } } existingRows.Close() } fmt.Printf("目标表中已有 %d 条ISBN记录\n\n", len(existingISBNs)) // 过滤掉已存在的ISBN fmt.Println("========================================") fmt.Println(" 步骤3: 过滤重复数据") fmt.Println("========================================") var newRecords []Record duplicateCount := 0 for _, r := range records { if !existingISBNs[r.ISBN] { newRecords = append(newRecords, r) } else { duplicateCount++ } } fmt.Printf("过滤完成\n") fmt.Printf("需要插入的新记录: %d 条\n", len(newRecords)) fmt.Printf("已存在的重复记录: %d 条\n\n", duplicateCount) if len(newRecords) == 0 { fmt.Println("没有新数据需要插入") return } // 批量插入数据 fmt.Println("========================================") fmt.Println(" 步骤4: 批量插入数据") fmt.Println("========================================") batchSize := 100 totalInserted := 0 now := time.Now() for i := 0; i < len(newRecords); i += batchSize { end := i + batchSize if end > len(newRecords) { end = len(newRecords) } batch := newRecords[i:end] // 构建批量插入SQL var valueStrings []string var args []interface{} for _, r := range batch { // 这里不需要单独生成id,后面会重新构建 createTime := now.Format("2006-01-02 15:04:05") valueStrings = append(valueStrings, "(?, '1', '0', '0', ?, 1, ?, 1, ?, '0', '0', '000000', NULL, NULL, '0', NULL)") args = append(args, r.ISBN, createTime, createTime) } // 重新构建包含ID的SQL valueStrings = make([]string, 0) args = make([]interface{}, 0) for _, r := range batch { id := snowflake.GenerateID() createTime := now.Format("2006-01-02 15:04:05") valueStrings = append(valueStrings, "(?, '1', '0', '0', ?, 1, ?, 1, ?, '0', '0', '000000', NULL, NULL, '0', NULL)") args = append(args, id, r.ISBN, createTime, createTime) } insertSQL := fmt.Sprintf("INSERT INTO t_filter_set (id, filter_type, limitation_type, add_way, add_txt, create_by, create_time, update_by, update_time, status, del_flag, tenant_id, create_dept, reason, sort, shop_type) VALUES %s", strings.Join(valueStrings, ", ")) result, err := dbSlave.Exec(insertSQL, args...) if err != nil { log.Printf("❌ 批次 %d-%d 插入失败: %v", i+1, end, err) continue } rowsAffected, _ := result.RowsAffected() totalInserted += int(rowsAffected) fmt.Printf("进度: %d/%d ( %.1f%% )\n", totalInserted, len(newRecords), float64(totalInserted)/float64(len(newRecords))*100) } totalElapsed := time.Since(startTime) fmt.Println("\n========================================") fmt.Println(" 迁移完成") fmt.Println("========================================") fmt.Printf("源数据总数: %d 条\n", len(records)) fmt.Printf("重复过滤: %d 条\n", duplicateCount) fmt.Printf("成功插入: %d 条\n", totalInserted) fmt.Printf("总耗时: %v\n", totalElapsed) fmt.Println("\n✅ 数据迁移完成!") }