package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
"strings"
"time"
)
func readCsvYield(fPath string, seq rune, printNum int) chan map[string]interface{} {
/*
fPath: 输入文件
seq: 分隔符
printNum: 打印的行数
*/
// var yield chan map[string]interface{}
yield := make(chan map[string]interface{})
var resDict map[string]interface{}
resDict = make(map[string]interface{})
go func() {
file, err := os.Open(fPath)
if err != nil {
fmt.Println(err)
close(yield)
}
defer file.Close()
reader := csv.NewReader(file)
// 设置分隔符seq
reader.Comma = seq
//reader.Comment = '\xEF\xBB\xBF'
totalRows := -1
var keys []string
for {
row, err := reader.Read()
if err != nil && err != io.EOF {
fmt.Printf("can not read , err is %s", err)
close(yield)
}
if err == io.EOF {
fmt.Println("total:", totalRows)
close(yield)
break
}
// 获取key
if totalRows == -1 {
for _, r := range row {
// utf-8 bom去掉
r = strings.Replace(r, "\uFEFF", "", -1)
keys = append(keys, r)
}
totalRows += 1
continue
}
// 组合成字典
for idx, item := range row {
resDict[keys[idx]] = item
}
totalRows += 1
yield <- resDict
resDict = make(map[string]interface{})
if totalRows%printNum == 0 {
fmt.Println(totalRows)
}
}
}()
return yield
}
func writeCsv(outFile string, data []map[string]interface{}) {
file, err := os.OpenFile(outFile, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
fmt.Println(err)
}
defer file.Close()
// 写入UTF-8 BOM 防止中文乱码
file.WriteString("\xEF\xBB\xBF")
w := csv.NewWriter(file)
var key []string
for idx, item := range data {
var values []string
// 第一行写入 获取key 写入第一行
if idx == 0 {
var tmpV []string
for k, v := range item {
key = append(key, k)
v := Strval(v)
tmpV = append(tmpV, v)
}
w.Write(key)
w.Write(tmpV)
continue
}
for _, k := range key {
v := Strval(item[k])
values = append(values, v)
}
w.Write(values)
w.Flush()
}
}
func writeCsvWithHeader(outFile string, data []map[string]interface{}, headers []string) {
file, err := os.OpenFile(outFile, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
fmt.Println(err)
}
defer file.Close()
var key []string
// 写入UTF-8 BOM 防止中文乱码
file.WriteString("\xEF\xBB\xBF")
w := csv.NewWriter(file)
key = append(key, headers...)
w.Write(key)
for _, item := range data {
var values []string
for _, k := range key {
v := Strval(item[k])
values = append(values, v)
}
w.Write(values)
w.Flush()
}
}
func getHeader(inFile string, seq rune) []string{
var headers []string
file, err := os.Open(inFile)
if err != nil {
fmt.Println("读取文件出错", err)
return headers
}
reader := csv.NewReader(file)
reader.Comma = seq
recon, _ := reader.Read()
headers = append(headers, recon...)
defer file.Close()
return headers
}
func Strval(value interface{}) string {
var key string
if value == nil {
return key
}
switch value.(type) {
case float64:
ft := value.(float64)
key = strconv.FormatFloat(ft, 'f', -1, 64)
case float32:
ft := value.(float32)
key = strconv.FormatFloat(float64(ft), 'f', -1, 64)
case int:
it := value.(int)
key = strconv.Itoa(it)
case uint:
it := value.(uint)
key = strconv.Itoa(int(it))
case int8:
it := value.(int8)
key = strconv.Itoa(int(it))
case uint8:
it := value.(uint8)
key = strconv.Itoa(int(it))
case int16:
it := value.(int16)
key = strconv.Itoa(int(it))
case uint16:
it := value.(uint16)
key = strconv.Itoa(int(it))
case int32:
it := value.(int32)
key = strconv.Itoa(int(it))
case uint32:
it := value.(uint32)
key = strconv.Itoa(int(it))
case int64:
it := value.(int64)
key = strconv.FormatInt(it, 10)
case uint64:
it := value.(uint64)
key = strconv.FormatUint(it, 10)
case string:
key = value.(string)
case []byte:
key = string(value.([]byte))
default:
newValue, _ := json.Marshal(value)
key = string(newValue)
}
return key
}
func main() {
fileName := "D:\\tmp\\20220228\\9982.csv"
startTime := time.Now()
var res []map[string]interface{}
for line := range readCsvYield(fileName, ',', 100000) {
//fmt.Println(line["credit_code"])
//fmt.Println(line["name"])
res = append(res, line)
}
fmt.Println(res)
fmt.Println(len(res))
endTime := time.Now()
fmt.Println(endTime.Sub(startTime))
}