在下面的代码中,我使用golang中的bufio将消息写入文件.我的磁盘I / O速度约为1000M / s.奇怪的是,当写入文件的大小小于20G时,写入速度约为每秒800M~900M,略低于I / O速度.但是,当文件大小超过21G时,我发现写入速度大约是每秒200M,远低于I / O速度.我不知道为什么,任何人都可以帮助我？谢谢.
package main
import "fmt"
import (
"os"
"time"
"flag"
"bufio"
)
func main() {
var pRound = flag.Int64("round",3500000,"loop round")
flag.Parse()
var message string
for i := 0; i < 1024; i++ {
message += "1234567890"
}
message += "\n"
f,err := os.OpenFile("server",os.O_CREATE|os.O_APPEND|os.O_RDWR,0666)
if err != nil {
fmt.Println(err)
return
}
w := bufio.NewWriter(f)
var i int64 = 0
round := *pRound
start := time.Now()
for i = 0; i < round; i++ {
w.WriteString(message)
}
w.Flush()
f.Close()
end := time.Now()
nanoseconds := end.Sub(start).Nanoseconds()
speed := 1000000000 * round / nanoseconds
fmt.Printf("round: %v\n",round)
fmt.Printf("Nanoseconds: %v\n",nanoseconds)
fmt.Printf("speed: %v\n",speed)
}
我想我问过一个愚蠢的问题.关于Vorsprung的回应,我在这里展示了我的C代码.我用C语言再次测试它.我用go语言找到了相同的结果.我的测试结果是：
./a.out 10000 seconds: 7 milliseconds: 260910 total: 10485760000 Bytes total: 10000 M total: 9.76562 G speed: 1377.24 M/s ./a.out 20000 seconds: 24 milliseconds: 7249 total: 20971520000 Bytes total: 20000 M total: 19.5312 G speed: 833.082 M/s ./a.out 30000 seconds: 80 milliseconds: 518970 total: 31457280000 Bytes total: 30000 M total: 29.2969 G speed: 372.583 M/s ./a.out 40000 seconds: 134 milliseconds: 615910 total: 41943040000 Bytes total: 40000 M total: 39.0625 G speed: 297.142 M/s
以下是我的C代码：
#include <fcntl.h>
#include <errno.h>
#include <limits.h>
#include <sys/uio.h>
#include <time.h>
#include <sys/time.h>
#include <iostream>
#include <string.h>
#include <stdlib.h>
int main(int argc,char* argv[]) {
if (argc != 2) {
std::cout << "usage: " << argv[0] << " round" << std::endl;
return -1;
}
int round = atoi(argv[1]);
int fd = open("file.data",O_CREAT | O_APPEND | O_RDWR,0666);
if (fd == -1) {
std::cout << "open file error: " << strerror(errno) << std::endl;
return -1;
}
struct iovec vec[IOV_MAX];
int len = 1024;
for (int i = 0; i < IOV_MAX; i++) {
vec[i].iov_base = new char[len];
vec[i].iov_len = len;
char *buf = NULL;
for (int j = 0; j < len - 1; j++) {
buf = (char*)vec[i].iov_base;
buf[j] = j % 10 + '1';
}
buf[len - 1] = '\n';
}
timeval tv1;
gettimeofday(&tv1,NULL);
for (int i = 0; i < round; i++) {
writev(fd,vec,IOV_MAX);
}
close(fd);
timeval tv2;
gettimeofday(&tv2,NULL);
for (int i = 0; i < IOV_MAX; i++) {
char* buf = (char*)vec[i].iov_base;
delete[] buf;
}
std::cout << "seconds: " << tv2.tv_sec - tv1.tv_sec << std::endl;
std::cout << "milliseconds: " << tv2.tv_usec - tv1.tv_usec << std::endl;
int64_t total = int64_t(len) * IOV_MAX * round;
float t = (tv2.tv_sec - tv1.tv_sec) * 1000000.0 + (tv2.tv_usec - tv1.tv_usec);
float speed = 1000000.0 * total / t / 1024 / 1024;
std::cout << "total: " << total << " Bytes" << std::endl;
std::cout << "total: " << total / 1024.0 / 1024.0 << " M" << std::endl;
std::cout << "total: " << total / 1024.0 / 1024.0 / 1024.0 << " G" << std::endl;
std::cout << "speed: " << speed << " M/s" << std::endl;
return 0;
}
现在我的diskio.go测试结果显示在这里.因为我不知道如何在评论中评论易读的结果,我在这里展示.
time ./diskio -size=4 written: 4294967296B 26237051975ns 4.29GB 26.24s 163.70MB/s real 0m26.980s user 0m0.397s sys 0m4.874s time ./diskio -size=8 written: 8589934592B 57803019028ns 8.59GB 57.80s 148.61MB/s real 0m59.192s user 0m0.813s sys 0m9.607s time ./diskio -size=10 written: 10737418240B 68331989999ns 10.74GB 68.33s 157.14MB/s real 1m10.288s user 0m0.946s sys 0m12.024s time ./diskio -size=20 written: 21474836480B 141169506440ns 21.47GB 141.17s 152.12MB/s real 2m25.037s user 0m1.881s sys 0m24.029s time ./diskio -size=30 written: 32212254720B 203807569664ns 32.21GB 203.81s 158.05MB/s real 3m29.345s user 0m2.925s sys 0m33.528s
diskio.go来自https://stackoverflow.com/a/47889346/5616809
我想我得到了答案,测试结果是由磁盘缓冲区原因引起的.我使用hdparm commend测试了我的磁盘速度,我得到了这个：
hdparm -Tt /dev/sde1 /dev/sde1: Timing cached reads: 18166 MB in 2.00 seconds = 9093.93 MB/sec Timing buffered disk reads: 584 MB in 3.01 seconds = 194.18 MB/sec
因此,当文件大小小于约18166M时,我的程序可能正在将字节写入缓冲区.之后程序正在写入磁盘,因此速度较慢.
您的问题无法重现.你的代码有bug.
我们希望磁盘写入时间会受到许多因素的影响：程序,其他程序,操作系统,硬件等.
引导独立的专用计算机,然后运行diskio.go程序.你得到了什么结果？例如,在我的机器上：
$go build diskio.go $time ./diskio -size=32 written: 34359738368B 154333936544ns 34.36GB 154.33s 222.63MB/s real 2m35.323s user 0m6.418s sys 0m41.994s $time ./diskio -size=16 written: 17179869184B 77901269159ns 17.18GB 77.90s 220.53MB/s real 1m18.746s user 0m2.849s sys 0m21.721s $time ./diskio -size=8 written: 8589934592B 38940248134ns 8.59GB 38.94s 220.59MB/s real 0m39.625s user 0m1.719s sys 0m12.493s $time ./diskio -size=1 written: 1073741824B 4738082404ns 1.07GB 4.74s 226.62MB/s real 0m4.851s user 0m0.069s sys 0m0.755s $
正如预期的那样,在程序中花费的时间很少,在操作系统上花费的时间更多,在磁盘上花费了大量的时间.写入速度没有急剧变化.
diskio.go：
package main
import (
"bufio"
"flag"
"fmt"
"os"
"time"
)
func writeFile(fSize int64) error {
fName := `/home/peter/diskio` // test file
defer os.Remove(fName)
f,err := os.Create(fName)
if err != nil {
return err
}
const defaultBufSize = 4096
buf := make([]byte,defaultBufSize)
buf[len(buf)-1] = '\n'
w := bufio.NewWriterSize(f,len(buf))
start := time.Now()
written := int64(0)
for i := int64(0); i < fSize; i += int64(len(buf)) {
nn,err := w.Write(buf)
written += int64(nn)
if err != nil {
return err
}
}
err = w.Flush()
if err != nil {
return err
}
err = f.Sync()
if err != nil {
return err
}
since := time.Since(start)
err = f.Close()
if err != nil {
return err
}
fmt.Printf("written: %dB %dns %.2fGB %.2fs %.2fMB/s\n",written,since,float64(written)/1000000000,float64(since)/float64(time.Second),(float64(written)/1000000)/(float64(since)/float64(time.Second)),)
return nil
}
var size = flag.Int("size",8,"file size in GiB")
func main() {
flag.Parse()
fSize := int64(*size) * (1024 * 1024 * 1024)
err := writeFile(fSize)
if err != nil {
fmt.Fprintln(os.Stderr,fSize,err)
}
}
