diff --git a/Export_module/Export.go b/Export_module/Export.go index bc025b96702596c7ccb99139ff3ecc121aac439f..c4a941f4247efbf97bc405ae6bce650b9c4c84f8 100644 --- a/Export_module/Export.go +++ b/Export_module/Export.go @@ -1,15 +1,16 @@ package Export_module import ( - "bufio" "dba_toolbox/Global" - "encoding/csv" "fmt" - _ "io" + "io" + "math" "os" - "path/filepath" + "strings" + "sync" "time" + "github.com/chzyer/readline" "github.com/gookit/color" ) @@ -21,23 +22,38 @@ var ( ) func Export() { - - var table_name string dirPath, err := getCurrentFilePath() if err != nil { panic(err) } - fmt.Printf("当前文件所在路径:%s\n", dirPath) fmt.Println("欢迎使用DBA_TOOLBOX的数据导出功能") - fmt.Print("请输入要导出的表_格式[库名.表名]_例:test.tb_test> ") - fmt.Scanln(&table_name) - start_Time := time.Now() - timeStr := start_Time.Format("2006-01-02 15:04:05") - fmt.Println("开始时间为:", timeStr) - //数据库保护查询地址 - // address := Get_address() + rl, err := readline.New("请输入要导出的表_格式[库名.表名]_例:test.tb_test> ") + if err != nil { + panic(err) + } + defer rl.Close() + + table_name, err := rl.Readline() + if err != nil { + panic(err) + } + defaultInput := "yes" + r2, err := readline.New("导出CSV文件是否合并 yes or no [default yes]> ") + if err != nil { + panic(err) + } + defer rl.Close() + + input, err := r2.Readline() + if err != nil { + panic(err) + } + + input = strings.TrimSpace(input) + + start_Time := time.Now() var count int query := fmt.Sprintf("SELECT COUNT(*) FROM %s", table_name) @@ -45,39 +61,51 @@ func Export() { if err != nil { panic(err.Error()) } - //切成4线程 - steps := count / 4 - //测试切成6线程 - // steps := count / 6 - sharding := 0 - for i := 0; i < 4; i++ { - - fileName := fmt.Sprintf("%s/%d.csv", dirPath, i) - // fmt.Print(fileName) + if input == "" { + input = defaultInput + } - sharding = steps * i - go CV_EXPORTS(sharding, steps, table_name, fileName) - time.Sleep(time.Second) + if strings.EqualFold(input, "no") { + //切成4线程 + steps := int(math.Ceil(float64(count) / float64(4))) + for i := 0; i < 4; i++ { + fileName := fmt.Sprintf("%s/%d.csv", dirPath, i) + sharding := steps * i + go CV_EXPORTS(sharding, steps, table_name, fileName) + time.Sleep(time.Second) + } + } else { + //切成6线程 + steps := int(math.Ceil(float64(count) / float64(6))) + var wg sync.WaitGroup // 声明一个WaitGroup + for i := 0; i < 6; i++ { + fileName := fmt.Sprintf("%s/%d.csv", dirPath, i) + sharding := steps * i + wg.Add(1) // 为每个协程增加计数 + go func(start, step int, table, fileName string) { + defer wg.Done() // 在协程结束时减少计数 + CV_EXPORTS(start, step, table, fileName) + }(sharding, steps, table_name, fileName) + } + wg.Wait() // 等待所有协程完成 + mergeCSV(table_name) } - // mergeCSV(dirPath) + color.BgGreen.Println("导出表成功:", table_name) + //获取结束时间 end_Time := time.Now() - //转时间格式为字符串 - timeStre := end_Time.Format("2006-01-02 15:04:05") - //输出结束时间 - fmt.Println("结束时间为:", timeStre) //计算两时间相差 elapsed_time := end_Time.Sub(start_Time) //输出总耗时 - fmt.Println("总耗时:", elapsed_time) + fmt.Println("导出总耗时:", elapsed_time) } func CV_EXPORTS(start int, steps int, table string, filename string) { query := fmt.Sprintf("select * into outfile '%s' from %s Limit %d , %d;", filename, table, start, steps) - fmt.Println(query) + // fmt.Println(query) counts, err := Global.DB.Query(query) if err != nil { fmt.Printf("单表导出失败err:%v\n", err) @@ -86,197 +114,49 @@ func CV_EXPORTS(start int, steps int, table string, filename string) { } -// func mergeCSV(dirPath string) { -// files := []string{"1.csv", "2.csv", "3.csv", "4.csv", "5.csv"} - -// outputFile, err := os.OpenFile(filepath.Join(dirPath, "0.csv"), os.O_APPEND|os.O_WRONLY, os.ModeAppend) -// if err != nil { -// panic(err) -// } -// defer outputFile.Close() - -// writer := csv.NewWriter(outputFile) -// defer writer.Flush() - -// for _, file := range files { -// inputFile, err := os.Open(filepath.Join(dirPath, file)) -// if err != nil { -// panic(err) -// } -// defer inputFile.Close() - -// reader := csv.NewReader(inputFile) -// for { -// record, err := reader.Read() -// if err != nil { -// if err == io.EOF { -// break -// } else { -// panic(err) -// } -// } -// if err := writer.Write(record); err != nil { -// panic(err) -// } -// } -// } -// } - -// 第二次版本,并行处理,但是会导致顺序不一样func mergeCSV(dirPath string) { -// files := []string{"1.csv", "2.csv", "3.csv"} - -// outputFile, err := os.OpenFile(filepath.Join(dirPath, "0.csv"), os.O_APPEND|os.O_WRONLY, os.ModeAppend) -// if err != nil { -// panic(err) -// } -// defer outputFile.Close() - -// writer := csv.NewWriter(outputFile) -// defer writer.Flush() - -// recordCh := make(chan []string) - -// var wg sync.WaitGroup -// for _, file := range files { -// wg.Add(1) -// go func(file string) { -// defer wg.Done() - -// inputFile, err := os.Open(filepath.Join(dirPath, file)) -// if err != nil { -// panic(err) -// } -// defer inputFile.Close() - -// reader := csv.NewReader(inputFile) -// for { -// record, err := reader.Read() -// if err != nil { -// if err == io.EOF { -// break -// } else { -// panic(err) -// } -// } -// recordCh <- record -// } -// }(file) -// } - -// go func() { -// wg.Wait() -// close(recordCh) -// }() - -// for record := range recordCh { -// if err := writer.Write(record); err != nil { -// panic(err) -// } -// } -// } - -// 第一次优化版本func mergeCSV(dirPath string) { -// files := []string{"0.csv", "1.csv", "2.csv", "3.csv"} - -// outputFile, err := os.Create("All_csv.csv") -// if err != nil { -// panic(err) -// } -// defer outputFile.Close() - -// writer := csv.NewWriter(outputFile) -// defer writer.Flush() - -// recordCh := make(chan []string) - -// var wg sync.WaitGroup -// for _, file := range files { -// wg.Add(1) -// go func(file string) { -// defer wg.Done() - -// inputFile, err := os.Open(filepath.Join(dirPath, file)) -// if err != nil { -// panic(err) -// } -// defer inputFile.Close() - -// reader := csv.NewReader(inputFile) -// for { -// record, err := reader.Read() -// if err != nil { -// if err == io.EOF { -// break -// } else { -// panic(err) -// } -// } -// recordCh <- record -// } -// }(file) -// } - -// go func() { -// wg.Wait() -// close(recordCh) -// }() - -// for record := range recordCh { -// if err := writer.Write(record); err != nil { -// panic(err) -// } -// } -// } - -// 最初版本 -func Merge_cvs(dirPath string) { - // 定义CSV文件路径 +func getCurrentFilePath() (string, error) { + dirPath, err := os.Getwd() + if err != nil { + fmt.Println("无法获取当前工作目录:", err) + } + // winwos测试用 + // dirPath = "D:/mycode/gitee/dba_toolbox" + return dirPath, nil +} - files := []string{"0.csv", "1.csv", "2.csv", "3.csv"} +func mergeCSV(table_name string) { + files := []string{"1.csv", "2.csv", "3.csv", "4.csv", "5.csv"} - // 创建输出文件 - outputFile, err := os.Create("All_csv.csv") + // 打开0.csv用于追加 + outputFile, err := os.OpenFile("0.csv", os.O_APPEND|os.O_WRONLY, 0644) if err != nil { panic(err) } defer outputFile.Close() - // 创建CSV writer - writer := csv.NewWriter(bufio.NewWriter(outputFile)) - defer writer.Flush() - - // 从多个文件中读取数据并写入输出文件 + // 依次打开1.csv、2.csv、3.csv,并将它们的内容追加到0.csv中 for _, file := range files { - inputFile, err := os.Open(filepath.Join(dirPath, file)) + inputFile, err := os.Open(file) if err != nil { panic(err) } defer inputFile.Close() - // 创建CSV reader - reader := csv.NewReader(bufio.NewReader(inputFile)) - - // 从CSV文件中读取数据 - records, err := reader.ReadAll() + _, err = io.Copy(outputFile, inputFile) if err != nil { panic(err) } + } - // 将数据写入输出文件 - for _, record := range records { - if err := writer.Write(record); err != nil { - panic(err) - } - } + // 重命名0.csv为ALL.csv + if err := os.Rename("0.csv", table_name+".csv"); err != nil { + panic(err) } -} -func getCurrentFilePath() (string, error) { - dirPath, err := os.Getwd() - if err != nil { - fmt.Println("无法获取当前工作目录:", err) + // 删除1.csv、2.csv、3.csv + for _, file := range files { + if err := os.Remove(file); err != nil { + panic(err) + } } - //winwos测试用 - // dirPath = "D:/mycode/gitee/dba_toolbox" - return dirPath, nil } diff --git a/go.mod b/go.mod index c09980a27bacbf32950f705b4b2619076f26f781..31ddaab4f4b689e6eb1ef513f3189addd1d53d4a 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( ) require ( + github.com/chzyer/readline v1.5.1 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect golang.org/x/sys v0.6.0 // indirect ) diff --git a/go.sum b/go.sum index 8c939376afc62e467098fe3d7c064682f296bf10..7c889ac622f121551b4bb6ddc9a4a209f9f75ba4 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= +github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= @@ -7,6 +11,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/main b/main index ff3bce5e919787e809210b414fb2e64f6ae99fab..77b43f2a406ea5499b6b228bfcb3d0ec160e14d3 100644 Binary files a/main and b/main differ diff --git a/main.exe b/main.exe deleted file mode 100644 index 55038a4171fe446af732803432438dbcf5ba2c76..0000000000000000000000000000000000000000 Binary files a/main.exe and /dev/null differ