Finished store
This commit is contained in:
parent
a09c98c243
commit
6762591ea2
4 changed files with 148 additions and 0 deletions
1
go.mod
1
go.mod
|
|
@ -5,5 +5,6 @@ go 1.16
|
|||
require (
|
||||
github.com/golang/protobuf v1.4.1
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/stretchr/testify v1.7.0
|
||||
google.golang.org/protobuf v1.25.0
|
||||
)
|
||||
|
|
|
|||
11
go.sum
11
go.sum
|
|
@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
|
|||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
|
|
@ -23,7 +25,12 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
|
|||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
|
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
|
@ -63,5 +70,9 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
|
|||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
116
internal/log/store.go
Normal file
116
internal/log/store.go
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
package log
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var (
|
||||
// enc is the endianess used to store records.
|
||||
enc = binary.BigEndian
|
||||
)
|
||||
|
||||
const (
|
||||
// lenWidth determines how many bytes will be used to store the length of the record.
|
||||
lenWidth = 8
|
||||
)
|
||||
|
||||
type store struct {
|
||||
// type embedding of an os file.
|
||||
*os.File
|
||||
|
||||
mu sync.Mutex
|
||||
buf *bufio.Writer
|
||||
size uint64
|
||||
}
|
||||
|
||||
// newStore returns a ready to use store ginven a file descriptor.
|
||||
func newStore(f *os.File) (*store, error) {
|
||||
// Get information for the given file descriptor.
|
||||
fi, err := os.Stat(f.Name())
|
||||
if err != nil {
|
||||
// Return nil becuase it's the zero value for a pointer.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// This is useful when working with pre-existing files, which could be the case when restarting.
|
||||
size := uint64(fi.Size())
|
||||
return &store{
|
||||
File: f,
|
||||
size: size,
|
||||
buf: bufio.NewWriter(f),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Append writes the provided bytes as a record to the end of the store.
|
||||
// Returns the size fo the record and the position of the record within the store.
|
||||
func (s *store) Append(p []byte) (n uint64, pos uint64, err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
pos = s.size
|
||||
// Write the size of the record before the actual record.
|
||||
if err := binary.Write(s.buf, enc, uint64(len(p))); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
w, err := s.buf.Write(p)
|
||||
// I don't think this is the way of doing it, because Write could return an
|
||||
// error even though it wrote some bytes to the file.
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
w += lenWidth
|
||||
s.size += uint64(w)
|
||||
return uint64(w), pos, nil
|
||||
}
|
||||
|
||||
// Read retrieves the record at position pos from the store.
|
||||
func (s *store) Read(pos uint64) ([]byte, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
// Commit any buffered data to the file.
|
||||
if err := s.buf.Flush(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Read the size of the record at pos.
|
||||
size := make([]byte, lenWidth)
|
||||
// Could remove `File` because of type embedding but leaving could be better for clarity?
|
||||
if _, err := s.File.ReadAt(size, int64(pos)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Read the actual record data given its offset and size.
|
||||
b := make([]byte, enc.Uint64(size))
|
||||
// Could remove `File` because of type embedding but leaving could be better for clarity?
|
||||
if _, err := s.File.ReadAt(b, int64(pos+lenWidth)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (s *store) ReadAt(p []byte, off int64) (int, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if err := s.buf.Flush(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Could remove `File` because of type embedding but leaving could be better for clarity?
|
||||
return s.File.ReadAt(p, off)
|
||||
}
|
||||
|
||||
func (s *store) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if err := s.buf.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.File.Close()
|
||||
}
|
||||
20
internal/log/store_test.go
Normal file
20
internal/log/store_test.go
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
package log
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
write = []byte("hello world")
|
||||
width = uint64(len(write) + lenWidth)
|
||||
)
|
||||
|
||||
func testAppend(t *testing.T, s *store) {
|
||||
t.Helper()
|
||||
for i := uint64(1); i < 4; i++ {
|
||||
n, pos, err := s.Append(write)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue