Published on

Buffered vs non buffered file IO using go

Authors

In Go, you can use the bufio package to perform buffered I/O on files. Buffered I/O means that data is read from or written to a buffer in memory, rather than reading from or writing to the file directly. This can be more efficient because it reduces the number of system calls and can improve the performance of the program.

Let's compare I/O operation between buffered IO and without buffer.

Write without buffer

Create file /tmp/test.txt then fill it directly using WriteString function with value some text!\n

fileio_test.go

_104
package main
_104
_104
import (
_104
"bufio"
_104
"io"
_104
"os"
_104
"testing"
_104
)
_104
_104
func BenchmarkWriteFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
for i := 0; i < 100000; i++ {
_104
f.WriteString("some text!\n")
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkWriteFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
w := bufio.NewWriter(f)
_104
_104
for i := 0; i < 100000; i++ {
_104
w.WriteString("some text!\n")
_104
}
_104
_104
w.Flush()
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
b := make([]byte, 10)
_104
_104
_, err = f.Read(b)
_104
for err == nil {
_104
_, err = f.Read(b)
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
r := bufio.NewReader(f)
_104
_104
_, err = r.ReadString('\n')
_104
for err == nil {
_104
_, err = r.ReadString('\n')
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBufferedScanner(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
scanner := bufio.NewScanner(f)
_104
_104
for scanner.Scan() {
_104
scanner.Text()
_104
}
_104
_104
if err := scanner.Err(); err != nil {
_104
fmt.Fprintln(os.Stderr, "reading standard input:", err)
_104
}
_104
_104
f.Close()
_104
}
_104
}

Write using buffer

Instead of writing directly to file, use bufio.NewWriter(f) to buffer data then flush it to file.

fileio_test.go

_104
package main
_104
_104
import (
_104
"bufio"
_104
"io"
_104
"os"
_104
"testing"
_104
)
_104
_104
func BenchmarkWriteFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
for i := 0; i < 100000; i++ {
_104
f.WriteString("some text!\n")
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkWriteFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
w := bufio.NewWriter(f)
_104
_104
for i := 0; i < 100000; i++ {
_104
w.WriteString("some text!\n")
_104
}
_104
_104
w.Flush()
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
b := make([]byte, 10)
_104
_104
_, err = f.Read(b)
_104
for err == nil {
_104
_, err = f.Read(b)
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
r := bufio.NewReader(f)
_104
_104
_, err = r.ReadString('\n')
_104
for err == nil {
_104
_, err = r.ReadString('\n')
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBufferedScanner(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
scanner := bufio.NewScanner(f)
_104
_104
for scanner.Scan() {
_104
scanner.Text()
_104
}
_104
_104
if err := scanner.Err(); err != nil {
_104
fmt.Fprintln(os.Stderr, "reading standard input:", err)
_104
}
_104
_104
f.Close()
_104
}
_104
}

Read without buffer

Reading file without buffer is read chunk byte data until got EOF (end of file)

fileio_test.go

_104
package main
_104
_104
import (
_104
"bufio"
_104
"io"
_104
"os"
_104
"testing"
_104
)
_104
_104
func BenchmarkWriteFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
for i := 0; i < 100000; i++ {
_104
f.WriteString("some text!\n")
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkWriteFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
w := bufio.NewWriter(f)
_104
_104
for i := 0; i < 100000; i++ {
_104
w.WriteString("some text!\n")
_104
}
_104
_104
w.Flush()
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
b := make([]byte, 10)
_104
_104
_, err = f.Read(b)
_104
for err == nil {
_104
_, err = f.Read(b)
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
r := bufio.NewReader(f)
_104
_104
_, err = r.ReadString('\n')
_104
for err == nil {
_104
_, err = r.ReadString('\n')
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBufferedScanner(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
scanner := bufio.NewScanner(f)
_104
_104
for scanner.Scan() {
_104
scanner.Text()
_104
}
_104
_104
if err := scanner.Err(); err != nil {
_104
fmt.Fprintln(os.Stderr, "reading standard input:", err)
_104
}
_104
_104
f.Close()
_104
}
_104
}

Read using buffered reader

Read using buffer is similar with writing file using buffer, using r := bufio.NewReader(f) to read all string line by line.

fileio_test.go

_104
package main
_104
_104
import (
_104
"bufio"
_104
"io"
_104
"os"
_104
"testing"
_104
)
_104
_104
func BenchmarkWriteFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
for i := 0; i < 100000; i++ {
_104
f.WriteString("some text!\n")
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkWriteFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
w := bufio.NewWriter(f)
_104
_104
for i := 0; i < 100000; i++ {
_104
w.WriteString("some text!\n")
_104
}
_104
_104
w.Flush()
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
b := make([]byte, 10)
_104
_104
_, err = f.Read(b)
_104
for err == nil {
_104
_, err = f.Read(b)
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
r := bufio.NewReader(f)
_104
_104
_, err = r.ReadString('\n')
_104
for err == nil {
_104
_, err = r.ReadString('\n')
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBufferedScanner(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
scanner := bufio.NewScanner(f)
_104
_104
for scanner.Scan() {
_104
scanner.Text()
_104
}
_104
_104
if err := scanner.Err(); err != nil {
_104
fmt.Fprintln(os.Stderr, "reading standard input:", err)
_104
}
_104
_104
f.Close()
_104
}
_104
}

Read using buffered scanner

Using scanner to read all files.

fileio_test.go

_104
package main
_104
_104
import (
_104
"bufio"
_104
"io"
_104
"os"
_104
"testing"
_104
)
_104
_104
func BenchmarkWriteFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
for i := 0; i < 100000; i++ {
_104
f.WriteString("some text!\n")
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkWriteFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
w := bufio.NewWriter(f)
_104
_104
for i := 0; i < 100000; i++ {
_104
w.WriteString("some text!\n")
_104
}
_104
_104
w.Flush()
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
b := make([]byte, 10)
_104
_104
_, err = f.Read(b)
_104
for err == nil {
_104
_, err = f.Read(b)
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
r := bufio.NewReader(f)
_104
_104
_, err = r.ReadString('\n')
_104
for err == nil {
_104
_, err = r.ReadString('\n')
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBufferedScanner(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
scanner := bufio.NewScanner(f)
_104
_104
for scanner.Scan() {
_104
scanner.Text()
_104
}
_104
_104
if err := scanner.Err(); err != nil {
_104
fmt.Fprintln(os.Stderr, "reading standard input:", err)
_104
}
_104
_104
f.Close()
_104
}
_104
}

Write without buffer

Create file /tmp/test.txt then fill it directly using WriteString function with value some text!\n

Write using buffer

Instead of writing directly to file, use bufio.NewWriter(f) to buffer data then flush it to file.

Read without buffer

Reading file without buffer is read chunk byte data until got EOF (end of file)

Read using buffered reader

Read using buffer is similar with writing file using buffer, using r := bufio.NewReader(f) to read all string line by line.

Read using buffered scanner

Using scanner to read all files.

fileio_test.go
ExpandClose

_104
package main
_104
_104
import (
_104
"bufio"
_104
"io"
_104
"os"
_104
"testing"
_104
)
_104
_104
func BenchmarkWriteFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
for i := 0; i < 100000; i++ {
_104
f.WriteString("some text!\n")
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkWriteFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Create("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
w := bufio.NewWriter(f)
_104
_104
for i := 0; i < 100000; i++ {
_104
w.WriteString("some text!\n")
_104
}
_104
_104
w.Flush()
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFile(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
b := make([]byte, 10)
_104
_104
_, err = f.Read(b)
_104
for err == nil {
_104
_, err = f.Read(b)
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBuffered(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
r := bufio.NewReader(f)
_104
_104
_, err = r.ReadString('\n')
_104
for err == nil {
_104
_, err = r.ReadString('\n')
_104
}
_104
if err != io.EOF {
_104
panic(err)
_104
}
_104
_104
f.Close()
_104
}
_104
}
_104
_104
func BenchmarkReadFileBufferedScanner(b *testing.B) {
_104
for n := 0; n < b.N; n++ {
_104
f, err := os.Open("/tmp/test.txt")
_104
if err != nil {
_104
panic(err)
_104
}
_104
_104
scanner := bufio.NewScanner(f)
_104
_104
for scanner.Scan() {
_104
scanner.Text()
_104
}
_104
_104
if err := scanner.Err(); err != nil {
_104
fmt.Fprintln(os.Stderr, "reading standard input:", err)
_104
}
_104
_104
f.Close()
_104
}
_104
}

The result from benchmark above is here:


_12
❯ go test -bench=File -benchmem
_12
goos: linux
_12
goarch: amd64
_12
pkg: github.com/h4ckm03d/golang-playground/6-practical-benchmark
_12
cpu: AMD Ryzen 7 5800H with Radeon Graphics
_12
BenchmarkWriteFile-4 14 80855082 ns/op 120 B/op 3 allocs/op
_12
BenchmarkWriteFileBuffered-4 438 12895724 ns/op 4216 B/op 4 allocs/op
_12
BenchmarkReadFile-4 44 25724519 ns/op 120 B/op 3 allocs/op
_12
BenchmarkReadFileBuffered-4 391 3051139 ns/op 1604219 B/op 100004 allocs/op
_12
BenchmarkReadFileBufferedScanner-4 864 1340817 ns/op 4216 B/op 4 allocs/op
_12
PASS
_12
ok github.com/h4ckm03d/golang-playground/6-practical-benchmark 11.107s

Based on the benchmark results, it looks like using buffered I/O is generally more efficient than using non-buffered I/O.

In particular, the WriteFileBuffered and ReadFileBufferedScanner benchmarks show significantly better performance than the corresponding non-buffered benchmarks. The ReadFileBuffered benchmark also shows improved performance, but not as significantly as the other two.

It's worth noting that the ReadFileBuffered benchmark allocates significantly more memory than the other benchmarks, which could be a concern if you are working with large files and need to minimize memory usage. In this case, you might want to consider using the ReadFileBufferedScanner benchmark, which has lower memory usage but still shows improved performance over the non-buffered version.