the program code snippet looks like this:
maxPage := *flagMaxPage
if maxPage > 0 {
jobs := make(chan int, maxPage)
results := make(chan []map[string]string, maxPage)
// 10jobsresults
for w := 1; w <= 10; wPP {
go doWork(jobs, results)
}
//
for j := 1; j <= maxPage; jPP {
jobs <- j
}
close(jobs)
//db connect
db, err := mysqlUtil.CreateReadDb()
defer db.Close()
if err != nil {
log.Fatal("connect db err ", err)
fmt.Println("connect db err ", err)
}
//
for j := 1; j <= maxPage; jPP {
data := <-results
//
doDb(db, data)
fmt.Println(len(data))
}
close(results)
fmt.Println("all done...")
}doDb function is as follows:
func doDb(db *sql.DB,data []map[string]string) {
if data != nil {
for _, v := range data {
if v == nil {
continue
}
if v["title"] == "" {
continue
}
if v["url"] == "" {
continue
}
//
sqlStr := `select * from film where title_md5 ="` + Md5(v["title"]) + `"`
rows, err := db.Query(sqlStr)
if err != nil {
fmt.Println("query err1 ", err)
continue
}
existData, err := mysqlUtil.GetRow(rows)
if err != nil {
fmt.Println("query err2 ", err)
continue
}
//,
if len(existData) == 0 {
score := v["score"]
scoreVal, err := strconv.ParseFloat(score, 64)
if err != nil {
fmt.Println("insert data parse score error ", err)
continue
}
f1 := decimal.NewFromFloat(scoreVal)
f2 := decimal.NewFromFloat(100)
aaa := f1.Mul(f2).IntPart()
insertScore := strconv.Itoa(int(aaa))
titleMd5 := Md5(v["title"])
sqlInsert := `insert into film(title, img, uptime, url, score, title_md5) values ("` + v["title"] + `", "` + v["img"] + `", "` + v["uptime"] +
`","` + v["url"] + `","` + insertScore + `","` + titleMd5 + `")`
db.Query(sqlInsert)
}
}
}
}after testing, about 20, err) runs. The second parameter of doDb function can contain up to 15 data at a time. After executing the results of 16 co-runs, a large number of data will be reported in fmt.Println ("query err1", err)).
query err1 dial tcp: lookup xxxx.com: no such hostwhen I package this error, I execute
manuallynslookup xxxx.comparsing is normal. Excuse me, why is this? DoDb this function is an one to deal with result, a total of only 20 records 15, why there is such a problem?