其实通过http请求已经获得了种子的信息了,但是传播存储种子好像是违法的,所以就存储些描述信息吧。
之前 python 跑的太慢了。这个 go 并发不知道写的有没有问题?!
package main import ( "bufio" "bytes" "crypto/sha1" "database/sql" "fmt" _ "github.com/go-sql-driver/MysqL" "io" "log" "net" "net/http" "os" "strconv" "strings" "time" ) import bencode "code.google.com/p/bencode-go" type FileDict struct { Length int64 "length" Path []string "path" Md5sum string "md5sum" } type InfoDict struct { FileDuration []int64 "file-duration" FileMedia []int64 "file-media" // Single file Name string "name" Length int64 "length" Md5sum string "md5sum" // Multiple files Files []FileDict "files" PieceLength int64 "piece length" Pieces string "pieces" Private int64 "private" } type MetaInfo struct { Info InfoDict "info" InfoHash string "info hash" Announce string "announce" AnnounceList [][]string "announce-list" CreationDate int64 "creation date" Comment string "comment" CreatedBy string "created by" Encoding string "encoding" } func (MetaInfo *MetaInfo) ReadTorrentMetaInfoFile(r io.Reader) bool { fileMetaData,er := bencode.Decode(r) if er != nil { return false } MetaInfoMap,ok := fileMetaData.(map[string]interface{}) if !ok { return false } var bytesBuf bytes.Buffer for mapKey,mapVal := range MetaInfoMap { switch mapKey { case "info": if er = bencode.Marshal(&bytesBuf,mapVal); er != nil { return false } infoHash := sha1.New() infoHash.Write(bytesBuf.Bytes()) MetaInfo.InfoHash = string(infoHash.Sum(nil)) if er = bencode.Unmarshal(&bytesBuf,&MetaInfo.Info); er != nil { return false } case "announce-list": if er = bencode.Marshal(&bytesBuf,mapVal); er != nil { return false } if er = bencode.Unmarshal(&bytesBuf,&MetaInfo.AnnounceList); er != nil { return false } case "announce": if aa,ok := mapVal.(string); ok { MetaInfo.Announce = aa } case "creation date": if tt,ok := mapVal.(int64); ok { MetaInfo.CreationDate = tt } case "comment": if cc,ok := mapVal.(string); ok { MetaInfo.Comment = cc } case "created by": if cb,ok := mapVal.(string); ok { MetaInfo.CreatedBy = cb } case "encoding": if ed,ok := mapVal.(string); ok { MetaInfo.Encoding = ed } } } return true } func makeUrl(hashinfo string) string { url := "http://bt.Box.n0808.com/%s/%s/%s.torrent" str := strings.ToUpper(hashinfo) return fmt.Sprintf(url,str[0:2],str[len(str)-2:],str) } func logFile(msg string) { f,err := os.OpenFile("logfile_torrent.txt",os.O_RDWR|os.O_CREATE|os.O_APPEND,0666) if err != nil { return } defer f.Close() log.SetOutput(f) log.Println(msg) } var timeout = time.Duration(2 * time.Second) func dialTimeout(network,addr string) (net.Conn,error) { return net.DialTimeout(network,addr,timeout) } func pullTorrent(url string) (int,error) { req,err := http.NewRequest("GET",url,nil) if err != nil { return 1,err } req.Header.Add("User-Agent","Mozilla/5.0") req.Header.Add("Host","bt.Box.n0808.com") req.Header.Add("Accept","*/*") req.Header.Add("Connection","Keep-Alive") transport := http.Transport{ Dial: dialTimeout,} client := &http.Client{ Transport: &transport,} resp,err := client.Do(req) if err != nil { return 2,err } defer resp.Body.Close() var MetaTorrent MetaInfo ok := MetaTorrent.ReadTorrentMetaInfoFile(resp.Body) if !ok { return 3,nil } name := MetaTorrent.Info.Name hashInfo := fmt.Sprintf("%X",MetaTorrent.InfoHash) created := MetaTorrent.CreationDate var fileLength int64 var fileDownLoadList bytes.Buffer var fileList string for _,fileDict := range MetaTorrent.Info.Files { fileLength += fileDict.Length for _,path := range fileDict.Path { fileDownLoadList.WriteString(path) fileDownLoadList.WriteString("\r\n") } } fileList = fileDownLoadList.String() var fileLengthTotal int64 if fileLength > 0 { fileLengthTotal = fileLength / (1024 * 1024) } if fileLengthTotal > 0 { db,err := sql.Open("MysqL","root:admin@tcp(127.0.0.1:3306)/678BT?charset=utf8&timeout=3s") if err != nil { return 4,err } defer db.Close() stmtIns,err := db.Prepare("INSERT INTO magnet (hashinfo,name,files,length,created,indexd) VALUES(?,?,?)") if err != nil { return 5,err } defer stmtIns.Close() timestamp := time.Now().Unix() _,error := stmtIns.Exec(hashInfo,fileList,fileLengthTotal,timestamp) if error != nil { return 6,error } } return 0,nil } func popChan(chs []chan int) { for _,vv := range chs { tmp := <-vv fmt.Println(tmp) } } func main() { f,err := os.Open("torrent.txt") if err != nil { log.Fatal(err) } defer f.Close() bf := bufio.NewReader(f) no := 0 ch := make([]chan int,128) op := 0 for { if no >= 128 { popChan(ch) no = 0 } ch[no] = make(chan int) line,isPrefix,err := bf.ReadLine() if err == io.EOF { break } if err != nil { logFile(err.Error()) } if isPrefix { logFile("LINE TOO LONG") } torrent := strings.Trim(string(line),"\r\n") torrent1 := strings.Trim(torrent,"\r") torrent2 := strings.Trim(torrent1,"\n") if len(torrent2) > 10 { go func(chx chan int,nox int) { ret,err := pullTorrent(makeUrl(torrent2)) if ret != 0 { logFile(strconv.Itoa(ret)) if err != nil { logFile(err.Error()) } } chx <- nox }(ch[no],no) no++ } op++ if op%1000 == 0 { fmt.Println(no) } } }