Finished store

go.mod

@@ -5,5 +5,6 @@ go 1.16
 require (
 	github.com/golang/protobuf v1.4.1
 	github.com/gorilla/mux v1.8.0
+	github.com/stretchr/testify v1.7.0
 	google.golang.org/protobuf v1.25.0
 )

go.sum

@@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -23,7 +25,12 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -63,5 +70,9 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/log/store.go

@@ -0,0 +1,116 @@
+package log
+
+import (
+	"bufio"
+	"encoding/binary"
+	"os"
+	"sync"
+)
+
+var (
+	// enc is the endianess used to store records.
+	enc = binary.BigEndian
+)
+
+const (
+	// lenWidth determines how many bytes will be used to store the length of the record.
+	lenWidth = 8
+)
+
+type store struct {
+	// type embedding of an os file.
+	*os.File
+
+	mu   sync.Mutex
+	buf  *bufio.Writer
+	size uint64
+}
+
+// newStore returns a ready to use store ginven a file descriptor.
+func newStore(f *os.File) (*store, error) {
+	// Get information for the given file descriptor.
+	fi, err := os.Stat(f.Name())
+	if err != nil {
+		// Return nil becuase it's the zero value for a pointer.
+		return nil, err
+	}
+
+	// This is useful when working with pre-existing files, which could be the case when restarting.
+	size := uint64(fi.Size())
+	return &store{
+		File: f,
+		size: size,
+		buf:  bufio.NewWriter(f),
+	}, nil
+}
+
+// Append writes the provided bytes as a record to the end of the store.
+// Returns the size fo the record and the position of the record within the store.
+func (s *store) Append(p []byte) (n uint64, pos uint64, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	pos = s.size
+	// Write the size of the record before the actual record.
+	if err := binary.Write(s.buf, enc, uint64(len(p))); err != nil {
+		return 0, 0, err
+	}
+
+	w, err := s.buf.Write(p)
+	// I don't think this is the way of doing it, because Write could return an
+	// error even though it wrote some bytes to the file.
+	if err != nil {
+		return 0, 0, err
+	}
+	w += lenWidth
+	s.size += uint64(w)
+	return uint64(w), pos, nil
+}
+
+// Read retrieves the record at position pos from the store.
+func (s *store) Read(pos uint64) ([]byte, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Commit any buffered data to the file.
+	if err := s.buf.Flush(); err != nil {
+		return nil, err
+	}
+
+	// Read the size of the record at pos.
+	size := make([]byte, lenWidth)
+	// Could remove `File` because of type embedding but leaving could be better for clarity?
+	if _, err := s.File.ReadAt(size, int64(pos)); err != nil {
+		return nil, err
+	}
+
+	// Read the actual record data given its offset and size.
+	b := make([]byte, enc.Uint64(size))
+	// Could remove `File` because of type embedding but leaving could be better for clarity?
+	if _, err := s.File.ReadAt(b, int64(pos+lenWidth)); err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func (s *store) ReadAt(p []byte, off int64) (int, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if err := s.buf.Flush(); err != nil {
+		return 0, err
+	}
+
+	// Could remove `File` because of type embedding but leaving could be better for clarity?
+	return s.File.ReadAt(p, off)
+}
+
+func (s *store) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if err := s.buf.Flush(); err != nil {
+		return err
+	}
+	return s.File.Close()
+}

internal/log/store_test.go

@@ -0,0 +1,20 @@
+package log
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	write = []byte("hello world")
+	width = uint64(len(write) + lenWidth)
+)
+
+func testAppend(t *testing.T, s *store) {
+	t.Helper()
+	for i := uint64(1); i < 4; i++ {
+		n, pos, err := s.Append(write)
+		require.NoError(t, err)
+	}
+}