diff --git a/utils.go b/utils.go
index 230771606ca4bede9a8e6c602e177195cf3b387f..1d38cdfd66dd959274069ed2aa4afe7f22c0c4e2 100644
--- a/utils.go
+++ b/utils.go
@@ -247,6 +247,16 @@ func CborReadHeader(br io.Reader) (byte, uint64, error) {
 }
 
 func readByteBuf(r io.Reader, scratch []byte) (byte, error) {
+	// Reading a single byte from these buffers is much faster than copying
+	// into a slice.
+	switch r := r.(type) {
+	case *bytes.Buffer:
+		return r.ReadByte()
+	case *bytes.Reader:
+		return r.ReadByte()
+	case *bufio.Reader:
+		return r.ReadByte()
+	}
 	n, err := r.Read(scratch[:1])
 	if err != nil {
 		return 0, err