combine length + message before writing

This will help secio send fewer packets when negotiating. it _does_ cost us a
copy, but there's not much we can do about that.

msgio.go

@@ -76,13 +76,20 @@ type ReadWriteCloser interface {
 type writer struct {
 	W io.Writer
 
+	pool *pool.BufferPool
 	lock sync.Mutex
 }
 
 // NewWriter wraps an io.Writer with a msgio framed writer. The msgio.Writer
 // will write the length prefix of every message written.
 func NewWriter(w io.Writer) WriteCloser {
-	return &writer{W: w}
+	return NewWriterWithPool(w, pool.GlobalPool)
+}
+
+// NewWriterWithPool is identical to NewWriter but allows the user to pass a
+// custom buffer pool.
+func NewWriterWithPool(w io.Writer, p *pool.BufferPool) WriteCloser {
+	return &writer{W: w, pool: p}
 }
 
 func (s *writer) Write(msg []byte) (int, error) {
@@ -96,10 +103,13 @@ func (s *writer) Write(msg []byte) (int, error) {
 func (s *writer) WriteMsg(msg []byte) (err error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
-	if err := WriteLen(s.W, len(msg)); err != nil {
-		return err
-	}
-	_, err = s.W.Write(msg)
+
+	buf := s.pool.Get(len(msg) + lengthSize)
+	NBO.PutUint32(buf, uint32(len(msg)))
+	copy(buf[lengthSize:], msg)
+	_, err = s.W.Write(buf)
+	s.pool.Put(buf)
+
 	return err
 }
 

varint.go

@@ -12,16 +12,21 @@ import (
 type varintWriter struct {
 	W io.Writer
 
-	lbuf [binary.MaxVarintLen64]byte // for encoding varints
-	lock sync.Mutex                  // for threadsafe writes
+	pool *pool.BufferPool
+	lock sync.Mutex // for threadsafe writes
 }
 
 // NewVarintWriter wraps an io.Writer with a varint msgio framed writer.
 // The msgio.Writer will write the length prefix of every message written
 // as a varint, using https://golang.org/pkg/encoding/binary/#PutUvarint
 func NewVarintWriter(w io.Writer) WriteCloser {
+	return NewVarintWriterWithPool(w, pool.GlobalPool)
+}
+
+func NewVarintWriterWithPool(w io.Writer, p *pool.BufferPool) WriteCloser {
 	return &varintWriter{
-		W: w,
+		pool: p,
+		W:    w,
 	}
 }
 
@@ -37,12 +42,12 @@ func (s *varintWriter) WriteMsg(msg []byte) error {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
-	length := uint64(len(msg))
-	n := binary.PutUvarint(s.lbuf[:], length)
-	if _, err := s.W.Write(s.lbuf[:n]); err != nil {
-		return err
-	}
-	_, err := s.W.Write(msg)
+	buf := s.pool.Get(len(msg) + binary.MaxVarintLen64)
+	n := binary.PutUvarint(buf, uint64(len(msg)))
+	n += copy(buf[n:], msg)
+	_, err := s.W.Write(buf[:n])
+	s.pool.Put(buf)
+
 	return err
 }