From b31e0c17af7d537ad7e2f0bea80553c25a6dcbeb Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Sun, 8 Mar 2026 03:11:09 -0400 Subject: [PATCH 01/34] Starting with the skiplist implementation --- internals/file/skiplist.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 internals/file/skiplist.go diff --git a/internals/file/skiplist.go b/internals/file/skiplist.go new file mode 100644 index 0000000..3767b18 --- /dev/null +++ b/internals/file/skiplist.go @@ -0,0 +1,25 @@ +package file + +import "math/rand/v2" + +var MAX_LEVEL int = 32; + +type Node struct { + key string + levels []*Node +} + +type Skiplist struct { + head *Node + +} + +func randLevel() int { + var level int = 1; + + for(rand.IntN(2) == 1 && level < MAX_LEVEL){ + level++; + } + + return level; +} From 90500f733b9ef8620bfe1214812dab87c00fe5dc Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:07:31 -0400 Subject: [PATCH 02/34] Made the search and insert methods for the skip list --- internals/file/skiplist.go | 59 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/internals/file/skiplist.go b/internals/file/skiplist.go index 3767b18..e1246e5 100644 --- a/internals/file/skiplist.go +++ b/internals/file/skiplist.go @@ -2,16 +2,71 @@ package file import "math/rand/v2" -var MAX_LEVEL int = 32; +const MAX_LEVEL = 32; type Node struct { key string + value []byte levels []*Node } type Skiplist struct { head *Node - +} + +func CreateSkiplist() *Skiplist { + head := &Node{ + key: "", + value: nil, + levels: make([]*Node, MAX_LEVEL), + }; + return &Skiplist{head: head}; +} + +func (list *Skiplist) Search(key string) []byte { + curr := list.head; + var next *Node; + + for i := len(list.head.levels)-1; i >= 0; i-- { + + for next = curr.levels[i]; next != nil && key > next.key; next = curr.levels[i] { + curr = next; + } + if next != nil && next.key == key { + return next.value; + } + } + + return nil; +} + +func (list *Skiplist) Insert(key string, value []byte) { + var update [MAX_LEVEL]*Node; + curr := list.head; + + for i:= MAX_LEVEL-1; i >= 0; i-- { + for curr.levels[i] != nil && key > curr.levels[i].key { + curr = curr.levels[i]; + } + + update[i] = curr; + } + + newNode := &Node{ + key: key, + value: value, + levels: make([]*Node, randLevel()), + } + + for i := 0; i < len(newNode.levels); i++ { + newNode.levels[i] = update[i].levels[i]; + update[i].levels[i] = newNode; + } +} + +func (list *Skiplist) Delete(key string) []byte { + + return nil; } func randLevel() int { From c619d8620ac81b6f47339087fb6d5f07be495a5d Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Sat, 14 Mar 2026 22:08:41 -0400 Subject: [PATCH 03/34] Moved skiptable to it's own folder --- internals/{file => memtable}/skiplist.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internals/{file => memtable}/skiplist.go (100%) diff --git a/internals/file/skiplist.go b/internals/memtable/skiplist.go similarity index 100% rename from internals/file/skiplist.go rename to internals/memtable/skiplist.go From c4e98ee6f4d965adfa1a568077ef38be882d843b Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Sat, 14 Mar 2026 22:09:09 -0400 Subject: [PATCH 04/34] Moved skiptable to it's own folder --- internals/memtable/skiplist.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internals/memtable/skiplist.go b/internals/memtable/skiplist.go index e1246e5..cabedeb 100644 --- a/internals/memtable/skiplist.go +++ b/internals/memtable/skiplist.go @@ -1,4 +1,4 @@ -package file +package memtable import "math/rand/v2" From 0333ad99c3ae680ac337e522de154f2ce440bd69 Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Tue, 24 Mar 2026 00:55:38 -0400 Subject: [PATCH 05/34] Almost done with the memtable, next thing we need to worry about is the file i/o changes. --- internals/memtable/memtable.go | 60 ++++++++++++++++++++++++++++++ internals/memtable/skiplist.go | 66 +++++++++++++++++++++++++++++---- internals/record/record.go | 2 +- internals/record/record_test.go | 2 +- main.go | 3 +- 5 files changed, 122 insertions(+), 11 deletions(-) create mode 100644 internals/memtable/memtable.go diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go new file mode 100644 index 0000000..7e03dd7 --- /dev/null +++ b/internals/memtable/memtable.go @@ -0,0 +1,60 @@ +package memtable + +import ( + "github.com/Vince-maple-byte/KeyData/internals/record" +) + +const MAX_SIZE = 3200 + +type Memtable struct{ + list *Skiplist + size int +} + +func CreateMemtable() *Memtable { + return &Memtable{ + list: CreateSkiplist(), + size: 0, + } +} + +func (memtable *Memtable) Write(key, value string) (bool, error) { + record, err := record.CreateRecord(key, value, "PUT"); + + if err != nil { + return false, err; + } + + memtable.list.Insert(key, record); + memtable.size += 1; + + if(memtable.size == MAX_SIZE) { + //We are going to call a writeToFile function here + memtable.list.EmptyList(); + memtable.size = 0; + + } + + return true, nil; +} + +//TODO: Fix this so that it is added into the skiplist as a record with an empty value and a tombstone of 1 +func (memtable *Memtable) Delete(key string) (bool, error) { + _, err := memtable.list.Delete(key); + + if err != nil { + return false, err; + } + + return true, nil; +} + +func (memtable *Memtable) Get(key string) ([] byte, error) { + record, err := memtable.list.Search(key); + + if err != nil { + return nil,err; + } + + return record, nil; +} diff --git a/internals/memtable/skiplist.go b/internals/memtable/skiplist.go index cabedeb..c99153a 100644 --- a/internals/memtable/skiplist.go +++ b/internals/memtable/skiplist.go @@ -1,6 +1,9 @@ package memtable -import "math/rand/v2" +import ( + "errors" + "math/rand/v2" +) const MAX_LEVEL = 32; @@ -23,8 +26,8 @@ func CreateSkiplist() *Skiplist { return &Skiplist{head: head}; } -func (list *Skiplist) Search(key string) []byte { - curr := list.head; +func (list *Skiplist) Search(key string) ([]byte, error) { + curr := list.head; var next *Node; for i := len(list.head.levels)-1; i >= 0; i-- { @@ -33,11 +36,11 @@ func (list *Skiplist) Search(key string) []byte { curr = next; } if next != nil && next.key == key { - return next.value; + return next.value, nil; } } - return nil; + return nil, errors.New("Element doesn't exist"); } func (list *Skiplist) Insert(key string, value []byte) { @@ -51,6 +54,13 @@ func (list *Skiplist) Insert(key string, value []byte) { update[i] = curr; } + + //This code block checks if the key is already inside of the skiplist + next := curr.levels[0]; + if next != nil && next.key == key { + next.value = value; + return; + } newNode := &Node{ key: key, @@ -64,9 +74,51 @@ func (list *Skiplist) Insert(key string, value []byte) { } } -func (list *Skiplist) Delete(key string) []byte { +func (list *Skiplist) Delete(key string) ([]byte,error) { + deleteNode, err := list.searchNode(key); + + if err != nil { + return nil, err; + } + + update := make([]*Node, len(deleteNode.levels)); + curr := list.head; + val := deleteNode.value; + + for i:= len(update)-1; i>=0; i-- { + for curr.levels[i] != nil && key > curr.levels[i].key { + curr = curr.levels[i] + } + + update[i] = curr; + } + + for i:= 0; i < len(update); i++ { + update[i].levels[i] = deleteNode.levels[i]; + } + + return val, nil; +} + +func (list Skiplist) searchNode(key string) (*Node, error) { + curr := list.head; + var next *Node; + + for i := len(list.head.levels)-1; i >= 0; i-- { + + for next = curr.levels[i]; next != nil && key > next.key; next = curr.levels[i] { + curr = next; + } + if next != nil && next.key == key { + return next, nil; + } + } + + return nil, errors.New("Element doesn't exist"); +} - return nil; +func (list *Skiplist) EmptyList() { + list = CreateSkiplist(); } func randLevel() int { diff --git a/internals/record/record.go b/internals/record/record.go index b1eef14..ec97bc0 100644 --- a/internals/record/record.go +++ b/internals/record/record.go @@ -1,4 +1,4 @@ -package records +package record import ( "encoding/binary" diff --git a/internals/record/record_test.go b/internals/record/record_test.go index 87e8591..34f8ad5 100644 --- a/internals/record/record_test.go +++ b/internals/record/record_test.go @@ -1,4 +1,4 @@ -package records_test +package record_test import ( "encoding/binary" diff --git a/main.go b/main.go index 99d8d48..d31aa06 100644 --- a/main.go +++ b/main.go @@ -13,8 +13,7 @@ import ( /* TODO: - -Make the Write method +Refactor the file folder to handle file operations for both in memory data store (skiplist) and the file i/o in case we have a file miss. So get rid of the map, and the everything else. How the structure of the timestamp would look like Timestamp | CRC32 error checksum| Tombstone (It's one byte long; basically 0 and 1 to determine whether this is a deleted key or not) | Key Size | Payload(Value) Size | Key Value | Payload From a308b79121efe77e3183641fd05e1d2633ed6db3 Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Thu, 26 Mar 2026 01:44:40 -0400 Subject: [PATCH 06/34] Got started with the saving the contents of the memtable into a file --- internals/file/filesave.go | 41 ++++++++++++++++++++++++++++++++++ internals/memtable/memtable.go | 17 +++----------- 2 files changed, 44 insertions(+), 14 deletions(-) create mode 100644 internals/file/filesave.go diff --git a/internals/file/filesave.go b/internals/file/filesave.go new file mode 100644 index 0000000..bbefb0b --- /dev/null +++ b/internals/file/filesave.go @@ -0,0 +1,41 @@ +package file + +import ( + "errors" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/Vince-maple-byte/KeyData/internals/record" +) + +const COMPACTION_SIZE = 4; + +//TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. +func WriteToFile(list Skiplist) (bool, error) { + filepath := "./internals/data"; + filename := "" + files, err := os.ReadDir(filepath); + + if err != nil { + return false, err; + } + + if len(files) < 1 { + filename = "kd_1.sst" + } else { + index,err := strconv.Atoi(strings.Split(files[len(files)-1].Name(), "_")[1]); + + if err != nil { + return false, err; + } + filename = "kd_"+strconv.Itoa(index+1)+".sst" + } + + + + record +} + + diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index 7e03dd7..511b1aa 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -18,8 +18,8 @@ func CreateMemtable() *Memtable { } } -func (memtable *Memtable) Write(key, value string) (bool, error) { - record, err := record.CreateRecord(key, value, "PUT"); +func (memtable *Memtable) Write(key, value, operation string) (bool, error) { + record, err := record.CreateRecord(key, value, operation); if err != nil { return false, err; @@ -38,18 +38,7 @@ func (memtable *Memtable) Write(key, value string) (bool, error) { return true, nil; } -//TODO: Fix this so that it is added into the skiplist as a record with an empty value and a tombstone of 1 -func (memtable *Memtable) Delete(key string) (bool, error) { - _, err := memtable.list.Delete(key); - - if err != nil { - return false, err; - } - - return true, nil; -} - -func (memtable *Memtable) Get(key string) ([] byte, error) { +func (memtable *Memtable) Get(key string) ([]byte, error) { record, err := memtable.list.Search(key); if err != nil { From 99fc61210498c7dd0451ce972888f38799670398 Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Thu, 26 Mar 2026 19:47:17 -0400 Subject: [PATCH 07/34] Added a method in skiplist to allow for us to save the entire memtable as a 2d slice --- internals/file/filesave.go | 10 ++++++++-- internals/memtable/skiplist.go | 22 ++++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/internals/file/filesave.go b/internals/file/filesave.go index bbefb0b..1455474 100644 --- a/internals/file/filesave.go +++ b/internals/file/filesave.go @@ -6,14 +6,14 @@ import ( "path/filepath" "strconv" "strings" - + "github.com/Vince-maple-byte/KeyData/internals/memtable" "github.com/Vince-maple-byte/KeyData/internals/record" ) const COMPACTION_SIZE = 4; //TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. -func WriteToFile(list Skiplist) (bool, error) { +func WriteToFile(list memtable.Skiplist) (bool, error) { filepath := "./internals/data"; filename := "" files, err := os.ReadDir(filepath); @@ -33,8 +33,14 @@ func WriteToFile(list Skiplist) (bool, error) { filename = "kd_"+strconv.Itoa(index+1)+".sst" } + file, err := os.Create(filename); + + if err != nil { + return false, err; + } + // file.Write() record } diff --git a/internals/memtable/skiplist.go b/internals/memtable/skiplist.go index c99153a..5bfd312 100644 --- a/internals/memtable/skiplist.go +++ b/internals/memtable/skiplist.go @@ -15,6 +15,7 @@ type Node struct { type Skiplist struct { head *Node + size int } func CreateSkiplist() *Skiplist { @@ -23,7 +24,10 @@ func CreateSkiplist() *Skiplist { value: nil, levels: make([]*Node, MAX_LEVEL), }; - return &Skiplist{head: head}; + return &Skiplist{ + head: head + size: 0; + }; } func (list *Skiplist) Search(key string) ([]byte, error) { @@ -72,6 +76,8 @@ func (list *Skiplist) Insert(key string, value []byte) { newNode.levels[i] = update[i].levels[i]; update[i].levels[i] = newNode; } + + list.size++; } func (list *Skiplist) Delete(key string) ([]byte,error) { @@ -93,7 +99,7 @@ func (list *Skiplist) Delete(key string) ([]byte,error) { update[i] = curr; } - for i:= 0; i < len(update); i++ { + for i := range(len(update)) { update[i].levels[i] = deleteNode.levels[i]; } @@ -121,6 +127,18 @@ func (list *Skiplist) EmptyList() { list = CreateSkiplist(); } +func (list *Skiplist) EntireList() ([][]byte) { + curr := list.head.levels[0] + res := make([][]byte); + + for _ := range(list.size) { + res = append(res, curr.value); + curr = curr.levels[0]; + } + + return res; +} + func randLevel() int { var level int = 1; From c7ccec2c30284ff7b27c04b25a3184ec643c11fe Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Thu, 26 Mar 2026 20:43:07 -0400 Subject: [PATCH 08/34] Made a function in record to easily access each element in a record --- internals/file/filesave.go | 5 ++++- internals/record/record.go | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/internals/file/filesave.go b/internals/file/filesave.go index 1455474..019778c 100644 --- a/internals/file/filesave.go +++ b/internals/file/filesave.go @@ -39,9 +39,12 @@ func WriteToFile(list memtable.Skiplist) (bool, error) { return false, err; } + contents := list.EntireList(); + + //Create the indexing block for the file + // file.Write() - record } diff --git a/internals/record/record.go b/internals/record/record.go index ec97bc0..a230154 100644 --- a/internals/record/record.go +++ b/internals/record/record.go @@ -90,3 +90,20 @@ func CreateRecord(key, payload, operation string) ([]byte, error) { return b, nil } + +func GetContents(content []byte) (timestamp time.Time,checksum uint32,tombstone uint8,keySize uint32,payloadSize uint32,key string, payload string) { + timeByte := content[:8] + timestamp = time.Unix(0, int64(binary.BigEndian.Uint64(timeByte))) + + checksum = binary.BigEndian.Uint32(content[8:12]); + + tombstone = uint8(content[12]); + + keySize = binary.BigEndian.Uint32(content[13:17]) + payloadSize = binary.BigEndian.Uint32(content[17:21]); + + key = string(content[21:keySize+21]); + payload = string(content[keySize+21 : (keySize+21)+payloadSize]) + + return; +} From f943011d47721d7c3b84161c54930de4eb309e70 Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Sun, 19 Apr 2026 23:09:01 -0400 Subject: [PATCH 09/34] Able to finish the write operations for the file. Next up handling the read operations. --- internals/file/filesave.go | 76 ++++++++++++++++++++++++++++++++-- internals/memtable/skiplist.go | 13 +++--- internals/record/record.go | 8 ++-- 3 files changed, 84 insertions(+), 13 deletions(-) diff --git a/internals/file/filesave.go b/internals/file/filesave.go index 019778c..93beab4 100644 --- a/internals/file/filesave.go +++ b/internals/file/filesave.go @@ -1,11 +1,11 @@ package file import ( - "errors" + "encoding/binary" "os" - "path/filepath" "strconv" "strings" + "github.com/Vince-maple-byte/KeyData/internals/memtable" "github.com/Vince-maple-byte/KeyData/internals/record" ) @@ -13,6 +13,14 @@ import ( const COMPACTION_SIZE = 4; //TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. +//How the file is going to look like +//File block, index block, footer +//The file block is going to contain all of the contents of the file aka the records +//The index block is going to contain the byte offset of the location of certain records in the file block +//The index block is going to be sorted and when doing file reading we will first start out in the end of the index block +//until we find a key that's less than or equal to the key saved +//Footer block tells us where the file block starts and the index block starts. This will always be a set +//byte size of 128 bytes long for the info of file block is 64 bytes and the info for index block is 64 bytes long func WriteToFile(list memtable.Skiplist) (bool, error) { filepath := "./internals/data"; filename := "" @@ -41,10 +49,72 @@ func WriteToFile(list memtable.Skiplist) (bool, error) { contents := list.EntireList(); + //Create file block + fileblock := fileblock(contents); + //Create the indexing block for the file + indexblock := indexblock(contents); + + //Create the footer block for the file + footerblock := footerblock(fileblock); + // file.Write() + write := append(fileblock, indexblock...) + write = append(write, footerblock...) - // file.Write() + _,err = file.Write(write) + + if err != nil { + return false, err; + } + + err = file.Chmod(0444); + + if err != nil { + return false, err; + } + + err = file.Close(); + + if err != nil { + return false, err; + } + + + return true, nil } +func fileblock(contents [][]byte) []byte { + block := make([]byte, 0); + + for _,v := range(contents) { + block = append(block, v...); + } + + return block; +} + +func indexblock(contents [][]byte) []byte { + block := make([]byte, 0); + currByteoffset := uint64(0); + + for i := 0; i < len(contents); i++ { + _,_,_,_,_,key,_ := record.GetContents(contents[i]); + if i % COMPACTION_SIZE == 0 { + block = append(block, []byte(key)...) + block = append(block, binary.BigEndian.AppendUint64(make([]byte, 0), currByteoffset)...) + } + currByteoffset += uint64(i); + } + return block; +} + +func footerblock(fileblock []byte) []byte { + block := make([]byte, 0); + + block = append(block, binary.BigEndian.AppendUint64(make([]byte, 0), uint64(0))...) + block = append(block, binary.BigEndian.AppendUint64(make([]byte, 0), uint64(len(fileblock)))...) + + return block +} diff --git a/internals/memtable/skiplist.go b/internals/memtable/skiplist.go index 5bfd312..b604957 100644 --- a/internals/memtable/skiplist.go +++ b/internals/memtable/skiplist.go @@ -25,8 +25,8 @@ func CreateSkiplist() *Skiplist { levels: make([]*Node, MAX_LEVEL), }; return &Skiplist{ - head: head - size: 0; + head: head, + size: 0, }; } @@ -102,7 +102,8 @@ func (list *Skiplist) Delete(key string) ([]byte,error) { for i := range(len(update)) { update[i].levels[i] = deleteNode.levels[i]; } - + + list.size--; return val, nil; } @@ -128,10 +129,10 @@ func (list *Skiplist) EmptyList() { } func (list *Skiplist) EntireList() ([][]byte) { - curr := list.head.levels[0] - res := make([][]byte); + curr := list.head.levels[0]; + res := make([][]byte, list.size); - for _ := range(list.size) { + for range(list.size) { res = append(res, curr.value); curr = curr.levels[0]; } diff --git a/internals/record/record.go b/internals/record/record.go index a230154..953651b 100644 --- a/internals/record/record.go +++ b/internals/record/record.go @@ -92,18 +92,18 @@ func CreateRecord(key, payload, operation string) ([]byte, error) { } func GetContents(content []byte) (timestamp time.Time,checksum uint32,tombstone uint8,keySize uint32,payloadSize uint32,key string, payload string) { - timeByte := content[:8] - timestamp = time.Unix(0, int64(binary.BigEndian.Uint64(timeByte))) + timeByte := content[:8]; + timestamp = time.Unix(0, int64(binary.BigEndian.Uint64(timeByte))); checksum = binary.BigEndian.Uint32(content[8:12]); tombstone = uint8(content[12]); - keySize = binary.BigEndian.Uint32(content[13:17]) + keySize = binary.BigEndian.Uint32(content[13:17]); payloadSize = binary.BigEndian.Uint32(content[17:21]); key = string(content[21:keySize+21]); - payload = string(content[keySize+21 : (keySize+21)+payloadSize]) + payload = string(content[keySize+21 : (keySize+21)+payloadSize]); return; } From 5cd46c3fcad640e4ba6b1a2739049fb0a8561d26 Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Mon, 20 Apr 2026 23:52:29 -0400 Subject: [PATCH 10/34] Getting started with the file i/o for reads. --- internals/file/filesave.go | 57 ++++++++++++++++++++++++++++++++-- internals/memtable/memtable.go | 25 +++++++++++++-- 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/internals/file/filesave.go b/internals/file/filesave.go index 93beab4..e3af9ad 100644 --- a/internals/file/filesave.go +++ b/internals/file/filesave.go @@ -2,6 +2,7 @@ package file import ( "encoding/binary" + "errors" "os" "strconv" "strings" @@ -12,7 +13,60 @@ import ( const COMPACTION_SIZE = 4; -//TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. +func ReadFromStorage() ([]byte, error) { + filepath := "./internals/data"; + + files, err := os.ReadDir(filepath); + + for i := len(files) - 1; i >= 0; i-- { + var record []byte; + record, err := readfile(files[i].Name()); + + if err == nil { + return record, nil; + } + } + + return nil, errors.New("Key/Value pair could not be found") +} + +func readfile(filename string) ([]byte, error) { + file, err := os.Open(filename); + + if err != nil { + return nil, err; + } + + fileInfo, err := file.Stat(); + + if err != nil { + return nil, err; + } + + size := fileInfo.Size(); + + _, err = file.Seek(size - int64(128), 0); + + if err != nil { + return nil, err; + } + + footer := make([]byte, 128); + + file.Read(footer); + + fileOffset := binary.BigEndian.Uint64(footer[:64]); + + indexOffset := binary.BigEndian.Uint64(footer[64:]); + + // TODO: finish with this method. Start with the index block and do a binary search for each key in the block + // until you find a key that is equal to greater than that block while being less than the next key. If the + // key is not equal than we just do the binary search again but this time in the block of the key that is represented + // in the key in our file block. If it is still not there then the key/value pair doesn't exist. +} + + +//TODO:Make unit tests for write method for the file i/o; use the diagram that I made as a guide. //How the file is going to look like //File block, index block, footer //The file block is going to contain all of the contents of the file aka the records @@ -57,7 +111,6 @@ func WriteToFile(list memtable.Skiplist) (bool, error) { //Create the footer block for the file footerblock := footerblock(fileblock); - // file.Write() write := append(fileblock, indexblock...) write = append(write, footerblock...) diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index 511b1aa..4190c50 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -1,6 +1,8 @@ -package memtable +package memtable import ( + "github.com/Vince-maple-byte/KeyData/internals/file" + "github.com/Vince-maple-byte/KeyData/internals/memtable" "github.com/Vince-maple-byte/KeyData/internals/record" ) @@ -28,8 +30,13 @@ func (memtable *Memtable) Write(key, value, operation string) (bool, error) { memtable.list.Insert(key, record); memtable.size += 1; - if(memtable.size == MAX_SIZE) { - //We are going to call a writeToFile function here + if(memtable.size >= MAX_SIZE) { + _, errF := file.WriteToFile(&memtable.list) + + if errF != nil { + return false, errF; + } + memtable.list.EmptyList(); memtable.size = 0; @@ -38,6 +45,18 @@ func (memtable *Memtable) Write(key, value, operation string) (bool, error) { return true, nil; } +func (memtable *Memtable) Read(key string) ([]byte, error) { + record, err := memtable.list.Search(key); + + if err != nil { + //Call the method to perform the file i/o operation for reads + + //If we still return an error we would return a nil/empty byte slice and a error + } + + return record, nil; +} + func (memtable *Memtable) Get(key string) ([]byte, error) { record, err := memtable.list.Search(key); From daa47c2ebe8ca217228a3a957012c4a1e5b8067a Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Thu, 14 May 2026 19:27:01 -0400 Subject: [PATCH 11/34] Refactored the code since there where some errors with importing --- internals/file/file.go | 266 --------------------- internals/file/file_test.go | 343 ---------------------------- internals/file/filesave.go | 173 -------------- internals/memtable/memtable.go | 6 +- internals/memtable/skiplist_test.go | 1 + internals/sstable/writer.go | 133 +++++++++++ main.go | 102 +-------- 7 files changed, 142 insertions(+), 882 deletions(-) delete mode 100644 internals/file/file.go delete mode 100644 internals/file/file_test.go delete mode 100644 internals/file/filesave.go create mode 100644 internals/memtable/skiplist_test.go create mode 100644 internals/sstable/writer.go diff --git a/internals/file/file.go b/internals/file/file.go deleted file mode 100644 index 86feb02..0000000 --- a/internals/file/file.go +++ /dev/null @@ -1,266 +0,0 @@ -package file - -import ( - "encoding/binary" - "errors" - "fmt" - "os" - "time" - - records "github.com/Vince-maple-byte/KeyData/internals/record" -) - -type File struct { - File *os.File - Size int64 - Index map[string]int64 -} - -/*This is how we are opening up the file, reading, and writing to the file -We are using os.OpenFile that returns an os.File pointer that allows us to use methods such as -file.Seek(offset int64, whence int) which sets the byte offset in our program to point to where in the file each record is located, - and whence just specifies if we should start in the beginning of the file, the current offset, or the end of the file - -file.Read(b []byte) which would read from the file starting at the current byte offset the os.File is pointing to, -and then adding n bytes where n is the underlining size of the byte slice (remember a slice is just a dynamic array in go); - -file.OpenFile() allows us to open the file and return the *os.File, with this we can run the methods mentioned above, - and pass in some arguments such as os.O_APPEND to specify what we can do in the file; - the prem argument is only valid if the OpenFile is creating the file for the first time - and specifies the special bits(setuid, sticky bit), the owner permission, group permissions, and the global permissions in this format - 0664 - Where for the last three digits - 4 = read permission - 2 = write permission - 1 = execute permission - 0 = no permission - -*/ - -// If the file doesn't exist we will just return an empty File object and an network error code since we don't want the -// program to stop running for an invalid file path. This would help us in the future when we accept network calls. -func OpenFile(fileName string) (File, error) { - //Just created fileEmpty in here so that I can reuse it in other areas where an error can happen - fileEmpty := File{ - File: nil, - Size: 0, - Index: nil, - } - file, err := os.OpenFile(fileName, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0644) - - if err != nil { - return fileEmpty, err - } - - fileInfo, err := file.Stat() - - if err != nil { - return fileEmpty, err - } - - size := fileInfo.Size() - - f := File{ - File: file, - Size: size, - Index: make(map[string]int64), - } - - if size > 0 { - fileContents, errf := f.ReadFile(0, int(f.Size)) - - if errf != nil { - return fileEmpty, errf - } - - f.Index = f.PopulateMap(fileContents) - } - - return f, err -} - -// Were are basically going to go through the entire file record by record decoding the each record -// We are also going to checking the checksum for each record -// And if we encounter an incorrect checksum we will stop from there -// Potential solution: We can make a new file writing the values from the beginning until the records were valid and then just update the file struct to be only pointing to the valid log and deleting the old log -// NOTE: The value part of the map is for the byte offset to locate the record -func (f *File) PopulateMap(fileContents []byte) map[string]int64 { - m := make(map[string]int64) - index := 0 - - for index < len(fileContents) { - - keySizeSection := index + 13 - - if keySizeSection >= len(fileContents) { - break - } - - keySize := binary.BigEndian.Uint32(fileContents[keySizeSection : index+17]) - - payloadSizeSection := index + 17 - - if payloadSizeSection >= len(fileContents) { - break - } - - payloadSize := binary.BigEndian.Uint32(fileContents[payloadSizeSection : index+21]) - - payloadByte := fileContents[int(keySize)+(index+21) : int(keySize)+(index+21)+int(payloadSize)] - - checksum := binary.BigEndian.Uint32(fileContents[index+8 : index+12]) - - var validChecksum bool = records.ChecksumChecker(payloadByte, checksum) - - if validChecksum { - key := string(fileContents[index+21 : int(keySize)+(index+21)]) - - m[key] = int64(index) - - //The length of the record will always be 21 + keySize + payloadSize - index += int(21 + keySize + payloadSize) - } else { - f.trimFile(index) - break - } - - } - - return m -} - -// This method gets executed when a record in the database is found out to be invalid through checksum. -// When this is the case we just take all of the values before the invalid one and save it to a new file. -func (f *File) trimFile(endPoint int) { - newFileContents, err := f.ReadFile(0, endPoint) - - if err != nil { - panic("Something went wrong with trying to read the file") - } - - fileName := f.File.Name() - - err = os.Remove(fileName) - - if err != nil { - panic("Unable to delete the file " + fileName) - } - - f.File, err = os.OpenFile(fileName, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0644) - - if err != nil { - panic("Unable to open/create file " + fileName) - } - - size, err := f.File.Write(newFileContents) - - if err != nil { - panic("Was not able to write into " + fileName) - } - - f.Size = int64(size) - -} - -// If we get -1 that means that the file was not successfully written. -func (f *File) PutContents(key, payload, operation string) (amountAdded int, err error) { - var recordBytes []byte - recordBytes, err = records.CreateRecord(key, payload, operation) - - if err != nil { - return -1, err - } - - amountAdded, err = f.File.Write(recordBytes) - - if err != nil { - return -1, err - } - - f.UpdateMap(recordBytes) - - f.Size += int64(amountAdded) - - return amountAdded, nil -} - -// In the case where the key is deleted, we can just return the offset of where the deleted record exists just like a regular read -// and from there we can just read from the tombstone whether it was deleted or not -func (f *File) UpdateMap(contents []byte) { - var offset int64 = f.Size - - keySize := binary.BigEndian.Uint32(contents[13:17]) - - key := string(contents[21 : keySize+21]) - - f.Index[key] = offset -} - -// We can now read from the file from the starting byte offset to the ending byte -func (f *File) ReadFile(startingPoint, endingPoint int) ([]byte, error) { - if startingPoint < 0 { - panic("Incorrect starting point") - } - - file := f.File - - //We would change the number for seek to be the specific byte offset in the map from the file struct - _, err := file.Seek(int64(startingPoint), 0) - if err != nil { - fmt.Println(err) - return nil, err - } - - b := make([]byte, endingPoint) - - _, error := file.Read(b) - - if error != nil { - fmt.Println(error) - return nil, error - } - - //fmt.Println(string(f)); - return b, nil - -} - -func (f *File) GetContents(key string) (deleted bool, payload string, timestamp time.Time, err error) { - keyOffset := f.Index[key] - - content, e := f.ReadFile(int(keyOffset), int(f.Size)) - - if e != nil { - err = e - return - } - - timeByte := content[:8] - tombstone := content[12] - keySize := binary.BigEndian.Uint32(content[13:17]) - payloadSize := binary.BigEndian.Uint32(content[17:21]) - timestamp = time.Unix(0, int64(binary.BigEndian.Uint64(timeByte))) - - if tombstone == 1 { - deleted = true - payload = "" - return - } else { - deleted = false - } - - if err != nil { - return - } - - keySaved := string(content[21 : keySize+21]) - - if key != keySaved { - err = errors.New("Invalid key given") - return - } - - payload = string(content[keySize+21 : (keySize+21)+payloadSize]) - - return -} diff --git a/internals/file/file_test.go b/internals/file/file_test.go deleted file mode 100644 index a96f12c..0000000 --- a/internals/file/file_test.go +++ /dev/null @@ -1,343 +0,0 @@ -package file_test - -import ( - "maps" - "testing" - - "github.com/Vince-maple-byte/KeyData/internals/file" - records "github.com/Vince-maple-byte/KeyData/internals/record" -) - -//TODO: Make test cases for the missing methods - -func TestReadFile(t *testing.T) { - //Testing table - tests := []struct { - testName string - fileName string - key string - payload string - operation string - expected int - }{ - { - testName: "TestingInitFile", - fileName: "./init.txt", - key: "a", - payload: "a", - operation: "PUT", - expected: 23, //The length of the byte array - }, - } - - for _, test := range tests { - - t.Run(test.testName, func(t *testing.T) { - //When - f, err := file.OpenFile(test.fileName) - if err != nil { - t.Errorf("%v was not able to be opened", test.fileName) - } - - var byteContent []byte - byteContent, err = f.ReadFile(0, 23) - - if err != nil { - t.Error(err) - } - - //Actual - if len(byteContent) != test.expected { - t.Errorf("For %v, this amount was added %v instead of %v", test.testName, len(byteContent), test.expected) - } - - }) - - } - -} - -func TestPutContents(t *testing.T) { - //Testing table - tests := []struct { - testName string - fileName string - key string - payload string - operation string - expected int - }{ - { - testName: "TestingPut", - fileName: "./init.txt", - key: "a", - payload: "a", - operation: "PUT", - expected: 23, - }, - { - testName: "TestingDelete", - fileName: "./init.txt", - key: "a", - payload: "", - operation: "DELETE", - expected: 22, - }, - { - testName: "TestingDelete", - fileName: "./init.txt", - key: "b", - payload: "b", - operation: "PUT", - expected: 23, - }, - } - - for _, test := range tests { - - t.Run(test.testName, func(t *testing.T) { - //When - f, err := file.OpenFile(test.fileName) - if err != nil { - t.Errorf("%v was not able to be opened", test.fileName) - } - - var amountAdded int - amountAdded, err = f.PutContents(test.key, test.payload, test.operation) - - if err != nil { - t.Error(err) - } - - //Actual - if amountAdded != test.expected { - t.Errorf("For %v, this amount was added %v instead of %v", test.testName, amountAdded, test.expected) - } - - }) - - } -} - -func TestGetContents(t *testing.T) { - //Testing table - tests := []struct { - testName string - fileName string - key string - deleted bool - expectedPayload string - }{ - { - testName: "TestingInitFileA", - fileName: "./init.txt", - key: "a", - deleted: true, - expectedPayload: "", - }, - { - testName: "TestingInitFileB", - fileName: "./init.txt", - key: "b", - deleted: false, - expectedPayload: "b", - }, - } - - for _, test := range tests { - - t.Run(test.testName, func(t *testing.T) { - //When - f, err := file.OpenFile(test.fileName) - if err != nil { - t.Errorf("%v was not able to be opened", test.fileName) - } - - delete, payload, _, errf := f.GetContents(test.key) - - if errf != nil { - t.Error(errf) - } - - //Actual - if delete != test.deleted || string(payload) != test.expectedPayload { - t.Errorf("For %v, this record %v returns an invalid value %v and %v", test.testName, test.key, delete, string(payload)) - } - - }) - - } -} - -func TestOpenFile(t *testing.T) { - //Testing table - tests := []struct { - testName string - fileName string - expectedSize int64 - expectedErr bool - }{ - { - testName: "IfFileDoesExists", - fileName: "./init.txt", - expectedSize: 0, - expectedErr: false, - }, - { - testName: "IfFileDoesNotExists", - fileName: "./internals/file/fake.txt", - expectedSize: 0, - expectedErr: true, - }, - } - - for _, test := range tests { - - t.Run(test.testName, func(t *testing.T) { - //When - f, err := file.OpenFile(test.fileName) - - //Actual - if test.expectedErr && err == nil { - t.Errorf("Expected error, got nil") - } - - if !test.expectedErr && err != nil { - t.Errorf("Unexpected error: %v", err) - } - - if f.Size != test.expectedSize { - t.Errorf( - "incorrect file size: got %d, want %d", - f.Size, - test.expectedSize, - ) - } - - if !test.expectedErr && f.File == nil { - t.Errorf("expected file handle to be non-nil") - } - - }) - - } -} - -func TestPopulateMap(t *testing.T) { - record1, _ := records.CreateRecord("foo", "1", "PUT") - recordF, _ := records.CreateRecord("foo", "4", "PUT") - record2, _ := records.CreateRecord("bar", "1", "PUT") - record3, _ := records.CreateRecord("n", "2", "PUT") - record4, _ := records.CreateRecord("b", "3", "PUT") - combinedUnique := append(append([]byte{}, record1...), record2...) - combinedUnique = append(combinedUnique, record3...) - combinedUnique = append(combinedUnique, record4...) - combinedSame := append(append([]byte{}, record1...), recordF...) - tests := []struct { - testName string - ByteSlice []byte - expectedMap map[string]int64 - }{ - { - testName: "IfByteSliceIsEmpty", - ByteSlice: make([]byte, 0), - expectedMap: make(map[string]int64), - }, - { - testName: "IfByteSliceHasValues", - ByteSlice: record1, - expectedMap: map[string]int64{"foo": 0}, - }, - { - testName: "IfByteSliceHasMultipleValues", - ByteSlice: combinedUnique, - expectedMap: map[string]int64{"foo": 0, "bar": 25, "b": 73, "n": 50}, - }, - { - testName: "IfByteSliceHasMultiplePutsOftheSameKey", - ByteSlice: combinedSame, - expectedMap: map[string]int64{"foo": 25}, - }, - } - - for _, test := range tests { - //When - f := file.File{} - m := f.PopulateMap(test.ByteSlice) - - if !maps.Equal(test.expectedMap, m) { - t.Errorf("Map mismatch between\nexpected %v\nand\nactual %v", test.expectedMap, m) - } - } -} - -func TestUpdateMap(t *testing.T) { - record1, _ := records.CreateRecord("foo", "1", "PUT") - record2, _ := records.CreateRecord("foo", "2", "PUT") - record3, _ := records.CreateRecord("bar", "3", "PUT") - emptyMap := make(map[string]int64) - fileEmpty := file.File{ - File: nil, - Size: 0, - Index: emptyMap, - } - - fileNonEmpty := file.File{ - File: nil, - Size: 25, - Index: map[string]int64{"foo": 0}, - } - - tests := []struct { - testName string - ByteSlice []byte - fileStruct file.File - expectedFileStruct file.File - }{ - { - testName: "IfMapWasEmpty", - ByteSlice: record1, - fileStruct: fileEmpty, - expectedFileStruct: file.File{ - File: nil, - Size: 0, //Keeping track of the size doesn't matter since the write function will handle that - Index: map[string]int64{"foo": 0}, - }, - }, - { - testName: "IfMapIsNotEmptyAndWeAreAddingAnExistingKey", - ByteSlice: record2, - fileStruct: fileNonEmpty, - expectedFileStruct: file.File{ - File: nil, - Size: 0, //Keeping track of the size doesn't matter since the write function will handle that - Index: map[string]int64{"foo": 25}, - }, - }, - { - testName: "IfMapIsNotEmptyAndWeAreAddingANewKey", - ByteSlice: record3, - fileStruct: file.File{ - File: nil, - Size: 25, - Index: map[string]int64{"foo": 0}, - }, - expectedFileStruct: file.File{ - File: nil, - Size: 0, //Keeping track of the size doesn't matter since the write function will handle that - Index: map[string]int64{"foo": 0, "bar": 25}, - }, - }, - } - - for _, test := range tests { - //When - test.fileStruct.UpdateMap(test.ByteSlice) - - //Then - if !maps.Equal(test.fileStruct.Index, test.expectedFileStruct.Index) { - t.Errorf("Map mismatch between expected %v and actual %v for test %v", - test.fileStruct.Index, test.expectedFileStruct.Index, test.testName) - } - } -} diff --git a/internals/file/filesave.go b/internals/file/filesave.go deleted file mode 100644 index e3af9ad..0000000 --- a/internals/file/filesave.go +++ /dev/null @@ -1,173 +0,0 @@ -package file - -import ( - "encoding/binary" - "errors" - "os" - "strconv" - "strings" - - "github.com/Vince-maple-byte/KeyData/internals/memtable" - "github.com/Vince-maple-byte/KeyData/internals/record" -) - -const COMPACTION_SIZE = 4; - -func ReadFromStorage() ([]byte, error) { - filepath := "./internals/data"; - - files, err := os.ReadDir(filepath); - - for i := len(files) - 1; i >= 0; i-- { - var record []byte; - record, err := readfile(files[i].Name()); - - if err == nil { - return record, nil; - } - } - - return nil, errors.New("Key/Value pair could not be found") -} - -func readfile(filename string) ([]byte, error) { - file, err := os.Open(filename); - - if err != nil { - return nil, err; - } - - fileInfo, err := file.Stat(); - - if err != nil { - return nil, err; - } - - size := fileInfo.Size(); - - _, err = file.Seek(size - int64(128), 0); - - if err != nil { - return nil, err; - } - - footer := make([]byte, 128); - - file.Read(footer); - - fileOffset := binary.BigEndian.Uint64(footer[:64]); - - indexOffset := binary.BigEndian.Uint64(footer[64:]); - - // TODO: finish with this method. Start with the index block and do a binary search for each key in the block - // until you find a key that is equal to greater than that block while being less than the next key. If the - // key is not equal than we just do the binary search again but this time in the block of the key that is represented - // in the key in our file block. If it is still not there then the key/value pair doesn't exist. -} - - -//TODO:Make unit tests for write method for the file i/o; use the diagram that I made as a guide. -//How the file is going to look like -//File block, index block, footer -//The file block is going to contain all of the contents of the file aka the records -//The index block is going to contain the byte offset of the location of certain records in the file block -//The index block is going to be sorted and when doing file reading we will first start out in the end of the index block -//until we find a key that's less than or equal to the key saved -//Footer block tells us where the file block starts and the index block starts. This will always be a set -//byte size of 128 bytes long for the info of file block is 64 bytes and the info for index block is 64 bytes long -func WriteToFile(list memtable.Skiplist) (bool, error) { - filepath := "./internals/data"; - filename := "" - files, err := os.ReadDir(filepath); - - if err != nil { - return false, err; - } - - if len(files) < 1 { - filename = "kd_1.sst" - } else { - index,err := strconv.Atoi(strings.Split(files[len(files)-1].Name(), "_")[1]); - - if err != nil { - return false, err; - } - filename = "kd_"+strconv.Itoa(index+1)+".sst" - } - - file, err := os.Create(filename); - - if err != nil { - return false, err; - } - - contents := list.EntireList(); - - //Create file block - fileblock := fileblock(contents); - - //Create the indexing block for the file - indexblock := indexblock(contents); - - //Create the footer block for the file - footerblock := footerblock(fileblock); - - write := append(fileblock, indexblock...) - write = append(write, footerblock...) - - _,err = file.Write(write) - - if err != nil { - return false, err; - } - - err = file.Chmod(0444); - - if err != nil { - return false, err; - } - - err = file.Close(); - - if err != nil { - return false, err; - } - - - return true, nil -} - -func fileblock(contents [][]byte) []byte { - block := make([]byte, 0); - - for _,v := range(contents) { - block = append(block, v...); - } - - return block; -} - -func indexblock(contents [][]byte) []byte { - block := make([]byte, 0); - currByteoffset := uint64(0); - - for i := 0; i < len(contents); i++ { - _,_,_,_,_,key,_ := record.GetContents(contents[i]); - if i % COMPACTION_SIZE == 0 { - block = append(block, []byte(key)...) - block = append(block, binary.BigEndian.AppendUint64(make([]byte, 0), currByteoffset)...) - } - currByteoffset += uint64(i); - } - return block; -} - -func footerblock(fileblock []byte) []byte { - block := make([]byte, 0); - - block = append(block, binary.BigEndian.AppendUint64(make([]byte, 0), uint64(0))...) - block = append(block, binary.BigEndian.AppendUint64(make([]byte, 0), uint64(len(fileblock)))...) - - return block -} - diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index 4190c50..7984e2e 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -1,9 +1,8 @@ package memtable import ( - "github.com/Vince-maple-byte/KeyData/internals/file" - "github.com/Vince-maple-byte/KeyData/internals/memtable" "github.com/Vince-maple-byte/KeyData/internals/record" + "github.com/Vince-maple-byte/KeyData/internals/sstable" ) const MAX_SIZE = 3200 @@ -31,7 +30,8 @@ func (memtable *Memtable) Write(key, value, operation string) (bool, error) { memtable.size += 1; if(memtable.size >= MAX_SIZE) { - _, errF := file.WriteToFile(&memtable.list) + content := memtable.list.EntireList(); + _, errF := sstable.WriteToFile(content) if errF != nil { return false, errF; diff --git a/internals/memtable/skiplist_test.go b/internals/memtable/skiplist_test.go new file mode 100644 index 0000000..2497c51 --- /dev/null +++ b/internals/memtable/skiplist_test.go @@ -0,0 +1 @@ +package memtable_test diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go new file mode 100644 index 0000000..b9f3320 --- /dev/null +++ b/internals/sstable/writer.go @@ -0,0 +1,133 @@ +package sstable + +import ( + "encoding/binary" + "os" + "slices" + "strconv" + "strings" + + "github.com/Vince-maple-byte/KeyData/internals/record" +) + +const ( + COMPACTION_SIZE = 4; + INDEX_BLOCK = 20; +) + +//TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. +func WriteToFile(list [][] byte) (bool, error) { + filepath := "./internals/data"; + filename := "" + files, err := os.ReadDir(filepath); + + if err != nil { + return false, err; + } + + if len(files) < 1 { + filename = "kd_1.sst" + } else { + index,err := strconv.Atoi(strings.Split(files[len(files)-1].Name(), "_")[1]); + + if err != nil { + return false, err; + } + filename = "kd_"+strconv.Itoa(index+1)+".sst" + } + + file, err := os.Create(filename); + + if err != nil { + return false, err; + } + + contents := list; + offset := fileOffset(contents); + index := createIndexBlock(contents, offset); + + + contents = append(contents, index); + + content := slices.Concat(contents...); + + _,err = file.Write(content); + + if err != nil { + return false, err; + } + + file.Sync(); + + compact(files); + + return true, nil; +} + +//Create the indexing block for the file + /* + The indexing block is responsible for allowing us to perform reads much more efficiently. + + How is it done: + The skiplist would be called to write into the file after 3200 entries are filled. + When this is done, we can categorize the file into 20 sections of 160 elements. + + In the indexing block, each entry would represent a specific block. + The entry will contain the key and the byte offset of the lowest key value in the block. + So it would look like this for example + keyA=0,keyB=202,keyC=403, etc. + + Since all of the records in the file are organized in sorted order, we can use the block to traverse through the file much quicker + + For example, if we have a key called N, we would start by comparing the key of the 20th block, + if the key is greater than or equal to the key N we would start from the byte offset of that block, + else we keep on going backwards by 1 block until we find a block where the key is greater than or equal to the block. + + Only issue with this method. In the case where the key is not located in the file, we would still need to do this traversal, + Solution using bloom filters. + + Format: + Size of index block: uint64 + Each entry will follow like this: + KeySize: uint32 bytes long; + Key: n bytes long (string) (n for keysize) + Byte offset: uint64 + */ + +func createIndexBlock(contentList [][]byte, offset []uint64) []byte { + index := make([]byte, 0, INDEX_BLOCK * 16); + + for i:=0; i < len(contentList); i=i+(len(contentList)/INDEX_BLOCK) { + _,_,_,keysize,_,key,_:= record.GetContents(contentList[i]); + + index = binary.BigEndian.AppendUint32(index, uint32(keysize)); + + index = append(index, key...); + + index = binary.BigEndian.AppendUint64(index, offset[i]); + } + + var size uint64 = uint64(len(index)); + index = append(binary.BigEndian.AppendUint64([]byte{}, size), index...); + + return index; +} + +func fileOffset(contentList [][]byte) []uint64 { + var offsetTracker uint64 = 0; + offset := make([]uint64, len(contentList)); + + for i, v := range(contentList) { + recordsize := len(v); + + offset[i] = offsetTracker; + + offsetTracker += uint64(recordsize); + } + + return offset; +} +func compact(files []os.DirEntry) { + + +} diff --git a/main.go b/main.go index d31aa06..e9bfff6 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,9 @@ package main -import ( - "fmt" - - "github.com/Vince-maple-byte/KeyData/internals/file" - //"time" - // "github.com/Vince-maple-byte/KeyData/internals/file" - //"github.com/Vince-maple-byte/KeyData/internals/record" - // "unsafe" -) +//"time" +// "github.com/Vince-maple-byte/KeyData/internals/file" +//"github.com/Vince-maple-byte/KeyData/internals/record" +// "unsafe" /* @@ -39,94 +34,7 @@ From there we can move on to SSTable, compaction, LSM Tables, and finally bloom */ func main() { - var database string - fmt.Println("Welcome to KeyData") - fmt.Print("Enter your database name to get started\n") - - _, err := fmt.Scan(&database) - - for err != nil { - fmt.Println("Please enter a valid database name") - _, err = fmt.Scan(&database) - } - - database += ".log" - - f, errf := file.OpenFile(database) - defer f.File.Close() - - if errf != nil { - panic(errf) - } - - fmt.Println("Database", database, "opened successfully") - willContinue := true - - for willContinue { - fmt.Println("Enter an operation: PUT, DELETE, GET") - - var operation string - _, err := fmt.Scan(&operation) - - if err != nil { - break - } - - if operation == "PUT" { - fmt.Println("Enter a key") - var key string - fmt.Scan(&key) - fmt.Println("Enter the data that you want to save") - var payload string - fmt.Scan(&payload) - - numAdded, _ := f.PutContents(key, payload, "PUT") - - if numAdded > -1 { - fmt.Println("Was able to successfully add the key/value pair into the database") - } else { - fmt.Println("Was not able to successfully add the key/value pair into the database") - } - - } - if operation == "DELETE" { - fmt.Println("Enter a key") - var key string - fmt.Scan(&key) - - numAdded, _ := f.PutContents(key, "", "DELETE") - - if numAdded > -1 { - fmt.Println("Was able to successfully delete the key/value pair into the database") - } else { - fmt.Println("Was not able to successfully delete the key/value pair into the database") - } - - } - if operation == "GET" { - fmt.Println("Enter a key") - var key string - fmt.Scan(&key) - - deleted, payload, timestamp, err := f.GetContents(key) - - if err != nil { - fmt.Println("Was able not able to retrieve the file contents for", key) - } else { - fmt.Printf("For key %v:\nThe payload: %v\nDeleted: %v\nThe timestamp: %v\n", key, payload, deleted, timestamp) - } - - } - operation = "" - - fmt.Println("Do you want to continue?(Y/N)") - var choice string - fmt.Scan(&choice) - - if choice == "N" || choice == "n" { - break - } + - } } From f718f372d33d5f3a26a0866ac2d22c4e9ef06aab Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Thu, 14 May 2026 19:43:27 -0400 Subject: [PATCH 12/34] I was able to add some unit tests for the skip list and fix a potential bug with emptying the skiplist --- internals/memtable/skiplist.go | 15 +- internals/memtable/skiplist_test.go | 271 +++++++++++++++++++++++++++- 2 files changed, 284 insertions(+), 2 deletions(-) diff --git a/internals/memtable/skiplist.go b/internals/memtable/skiplist.go index b604957..0fa72fd 100644 --- a/internals/memtable/skiplist.go +++ b/internals/memtable/skiplist.go @@ -125,7 +125,20 @@ func (list Skiplist) searchNode(key string) (*Node, error) { } func (list *Skiplist) EmptyList() { - list = CreateSkiplist(); + // head := &Node{ + // key: "", + // value: nil, + // levels: make([]*Node, MAX_LEVEL), + // }; + // return &Skiplist{ + // head: head, + // size: 0, + // }; + list.head.key = ""; + list.head.value = nil; + list.head.levels = make([]*Node, MAX_LEVEL); + + list.size = 0; } func (list *Skiplist) EntireList() ([][]byte) { diff --git a/internals/memtable/skiplist_test.go b/internals/memtable/skiplist_test.go index 2497c51..8283f9a 100644 --- a/internals/memtable/skiplist_test.go +++ b/internals/memtable/skiplist_test.go @@ -1 +1,270 @@ -package memtable_test +package memtable + +import ( + "testing" +) + +// --- helpers --- + +func newList(t *testing.T) *Skiplist { + t.Helper() + return CreateSkiplist() +} + +// --- CreateSkiplist --- + +func TestCreateSkiplist_NotNil(t *testing.T) { + list := newList(t) + if list == nil { + t.Fatal("expected non-nil Skiplist") + } +} + +func TestCreateSkiplist_InitialSizeZero(t *testing.T) { + list := newList(t) + if list.size != 0 { + t.Errorf("expected size 0, got %d", list.size) + } +} + +func TestCreateSkiplist_HeadHasMaxLevels(t *testing.T) { + list := newList(t) + if len(list.head.levels) != MAX_LEVEL { + t.Errorf("expected head to have %d levels, got %d", MAX_LEVEL, len(list.head.levels)) + } +} + +// --- Insert --- + +func TestInsert_SingleElement(t *testing.T) { + list := newList(t) + list.Insert("a", []byte("val-a")) + if list.size != 1 { + t.Errorf("expected size 1, got %d", list.size) + } +} + +func TestInsert_MultipleElements(t *testing.T) { + list := newList(t) + keys := []string{"banana", "apple", "cherry"} + for _, k := range keys { + list.Insert(k, []byte(k+"-value")) + } + if list.size != len(keys) { + t.Errorf("expected size %d, got %d", len(keys), list.size) + } +} + +func TestInsert_UpdateExistingKey(t *testing.T) { + list := newList(t) + list.Insert("key", []byte("old")) + list.Insert("key", []byte("new")) + + // Size should not grow on update. + if list.size != 1 { + t.Errorf("expected size 1 after update, got %d", list.size) + } + + val, err := list.Search("key") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(val) != "new" { + t.Errorf("expected 'new', got %q", val) + } +} + +func TestInsert_NilValue(t *testing.T) { + list := newList(t) + list.Insert("nilkey", nil) + val, err := list.Search("nilkey") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if val != nil { + t.Errorf("expected nil value, got %v", val) + } +} + +func TestInsert_EmptyStringKey(t *testing.T) { + list := newList(t) + list.Insert("", []byte("empty-key-value")) + val, err := list.Search("") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(val) != "empty-key-value" { + t.Errorf("expected 'empty-key-value', got %q", val) + } +} + +func TestInsert_PreservesOrder(t *testing.T) { + list := newList(t) + // Insert out of alphabetical order. + list.Insert("c", []byte("c")) + list.Insert("a", []byte("a")) + list.Insert("b", []byte("b")) + + // Level-0 linked list must be sorted. + curr := list.head.levels[0] + var prev string + for curr != nil { + if curr.key < prev { + t.Errorf("out of order: %q after %q", curr.key, prev) + } + prev = curr.key + curr = curr.levels[0] + } +} + +// --- Search --- + +func TestSearch_ExistingKey(t *testing.T) { + list := newList(t) + list.Insert("hello", []byte("world")) + val, err := list.Search("hello") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(val) != "world" { + t.Errorf("expected 'world', got %q", val) + } +} + +func TestSearch_MissingKey(t *testing.T) { + list := newList(t) + _, err := list.Search("ghost") + if err == nil { + t.Error("expected error for missing key, got nil") + } +} + +func TestSearch_EmptyList(t *testing.T) { + list := newList(t) + _, err := list.Search("anything") + if err == nil { + t.Error("expected error on empty list, got nil") + } +} + +func TestSearch_AfterMultipleInserts(t *testing.T) { + list := newList(t) + pairs := map[string]string{ + "foo": "1", + "bar": "2", + "baz": "3", + } + for k, v := range pairs { + list.Insert(k, []byte(v)) + } + for k, want := range pairs { + got, err := list.Search(k) + if err != nil { + t.Errorf("Search(%q): unexpected error: %v", k, err) + continue + } + if string(got) != want { + t.Errorf("Search(%q): expected %q, got %q", k, want, got) + } + } +} + +// --- Delete --- + +func TestDelete_ExistingKey(t *testing.T) { + list := newList(t) + list.Insert("del-me", []byte("value")) + val, err := list.Delete("del-me") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(val) != "value" { + t.Errorf("expected returned value 'value', got %q", val) + } +} + +func TestDelete_DecreasesSize(t *testing.T) { + list := newList(t) + list.Insert("x", []byte("x")) + list.Insert("y", []byte("y")) + list.Delete("x") + if list.size != 1 { + t.Errorf("expected size 1 after delete, got %d", list.size) + } +} + +func TestDelete_MakesKeyUnsearchable(t *testing.T) { + list := newList(t) + list.Insert("gone", []byte("poof")) + list.Delete("gone") + _, err := list.Search("gone") + if err == nil { + t.Error("expected error searching deleted key, got nil") + } +} + +func TestDelete_MissingKey(t *testing.T) { + list := newList(t) + _, err := list.Delete("nope") + if err == nil { + t.Error("expected error deleting missing key, got nil") + } +} + +func TestDelete_OnEmptyList(t *testing.T) { + list := newList(t) + _, err := list.Delete("anything") + if err == nil { + t.Error("expected error deleting from empty list, got nil") + } +} + +func TestDelete_CorrectValueReturned(t *testing.T) { + list := newList(t) + list.Insert("k", []byte("expected-val")) + val, err := list.Delete("k") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(val) != "expected-val" { + t.Errorf("expected 'expected-val', got %q", val) + } +} + +// --- EntireList --- + +func TestEntireList_EmptyList(t *testing.T) { + list := newList(t) + res := list.EntireList() + // Result should not be nil; contents may vary by implementation. + if res == nil { + t.Error("expected non-nil slice from EntireList on empty list") + } +} + +func TestEntireList_ReturnsAllValues(t *testing.T) { + list := newList(t) + list.Insert("a", []byte("1")) + list.Insert("b", []byte("2")) + list.Insert("c", []byte("3")) + res := list.EntireList() + if len(res) < 3 { + t.Errorf("expected at least 3 entries, got %d", len(res)) + } +} + +// --- EmptyList --- + +func TestEmptyList_SizeAfterEmpty(t *testing.T) { + list := newList(t) + list.Insert("a", []byte("1")) + list.Insert("b", []byte("2")) + list.EmptyList() + // NOTE: EmptyList reassigns list internally; the caller's pointer is + // unchanged. This test documents the current (possibly surprising) + // behaviour — update if the implementation is fixed. + size := list.size // no panic expected + if size != 0 { + t.Error("Expected an empty list",size); + } +} \ No newline at end of file From bc44dadb4845d80d1dcc7f91e46edc5df2e4bde3 Mon Sep 17 00:00:00 2001 From: Ivers Date: Thu, 4 Jun 2026 13:37:16 -0400 Subject: [PATCH 13/34] Refactored and made unit tests for the GetContent function so that it is easier to use. --- internals/record/record.go | 33 +++++--- internals/record/record_test.go | 43 +++++++--- internals/sstable/writer.go | 140 ++++++++++++++++---------------- 3 files changed, 124 insertions(+), 92 deletions(-) diff --git a/internals/record/record.go b/internals/record/record.go index 953651b..4788744 100644 --- a/internals/record/record.go +++ b/internals/record/record.go @@ -13,6 +13,17 @@ import ( // The int value for the checksum is different depending on the data that we provide // As long as the data is the same between two variables they should always have the same checksum. + +type Content struct { + Timestamp time.Time + Checksum uint32 + Tombstone uint8 + Keysize uint32 + Payloadsize uint32 + Key string + Payload string +} + func checkSum(data string) uint32 { return crc32.ChecksumIEEE([]byte(data)) } @@ -91,19 +102,19 @@ func CreateRecord(key, payload, operation string) ([]byte, error) { return b, nil } -func GetContents(content []byte) (timestamp time.Time,checksum uint32,tombstone uint8,keySize uint32,payloadSize uint32,key string, payload string) { - timeByte := content[:8]; - timestamp = time.Unix(0, int64(binary.BigEndian.Uint64(timeByte))); +func GetContents(content []byte) (result Content) { + timeByte := content[:8] + result.Timestamp = time.Unix(0, int64(binary.BigEndian.Uint64(timeByte))) + + result.Checksum = binary.BigEndian.Uint32(content[8:12]) - checksum = binary.BigEndian.Uint32(content[8:12]); - - tombstone = uint8(content[12]); + result.Tombstone = uint8(content[12]) - keySize = binary.BigEndian.Uint32(content[13:17]); - payloadSize = binary.BigEndian.Uint32(content[17:21]); + result.Keysize = binary.BigEndian.Uint32(content[13:17]) + result.Payloadsize = binary.BigEndian.Uint32(content[17:21]) - key = string(content[21:keySize+21]); - payload = string(content[keySize+21 : (keySize+21)+payloadSize]); + result.Key = string(content[21 : result.Keysize+21]) + result.Payload = string(content[result.Keysize+21 : (result.Keysize+21)+result.Payloadsize]) - return; + return } diff --git a/internals/record/record_test.go b/internals/record/record_test.go index 34f8ad5..1a488d6 100644 --- a/internals/record/record_test.go +++ b/internals/record/record_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - records "github.com/Vince-maple-byte/KeyData/internals/record" + "github.com/Vince-maple-byte/KeyData/internals/record" ) func TestRecordTimeStamp(t *testing.T) { @@ -44,7 +44,7 @@ func TestRecordTimeStamp(t *testing.T) { } for _, test := range tests { - record, err := records.CreateRecord(test.key, test.payload, test.operation) + record, err := record.CreateRecord(test.key, test.payload, test.operation) now := time.Now() if err != nil { @@ -96,7 +96,7 @@ func TestRecordCheckSum(t *testing.T) { } for _, test := range tests { - record, err := records.CreateRecord(test.key, test.payload, test.operation) + record, err := record.CreateRecord(test.key, test.payload, test.operation) if err != nil { t.Errorf("Unable to create the record") @@ -136,7 +136,7 @@ func TestRecordTombstone(t *testing.T) { } for _, test := range tests { - record, err := records.CreateRecord(test.key, test.payload, test.operation) + record, err := record.CreateRecord(test.key, test.payload, test.operation) if err != nil { t.Errorf("Unable to create the record") @@ -176,7 +176,7 @@ func TestRecordKeySize(t *testing.T) { } for _, test := range tests { - record, err := records.CreateRecord(test.key, test.payload, test.operation) + record, err := record.CreateRecord(test.key, test.payload, test.operation) if err != nil { t.Errorf("Unable to create the record") @@ -216,7 +216,7 @@ func TestRecordPayloadSize(t *testing.T) { } for _, test := range tests { - record, err := records.CreateRecord(test.key, test.payload, test.operation) + record, err := record.CreateRecord(test.key, test.payload, test.operation) if err != nil { t.Errorf("Unable to create the record") @@ -256,7 +256,7 @@ func TestRecordKey(t *testing.T) { } for _, test := range tests { - record, err := records.CreateRecord(test.key, test.payload, test.operation) + record, err := record.CreateRecord(test.key, test.payload, test.operation) if err != nil { t.Errorf("Unable to create the record") @@ -297,19 +297,42 @@ func TestRecordPayload(t *testing.T) { } for _, test := range tests { - record, err := records.CreateRecord(test.key, test.payload, test.operation) + records, err := record.CreateRecord(test.key, test.payload, test.operation) if err != nil { t.Errorf("Unable to create the record") } //var parseTime time.Time; - keySize := binary.BigEndian.Uint32(record[13:17]) + keySize := binary.BigEndian.Uint32(records[13:17]) //payloadSize := binary.BigEndian.Uint32(record[17:21]) - payload := string(record[keySize+21:]) + payload := string(records[keySize+21:]) if payload != test.expected { t.Errorf("incorrect checksum for %v", test.testName) } } } + +func TestGetContents(t *testing.T) { + key := "helllpp" + payload := "hhhh" + records, err := record.CreateRecord(key, payload, "PUT") + + if err != nil { + t.Errorf("Unable to create the record") + } + + getContents := record.GetContents(records) + + // if time.Unix(0, int64(binary.BigEndian.Uint64(records[:8]))) != getContents.Timestamp { + // t.Errorf("Was not able to retrieve the proper timestamp\n", + // "Expected:", time.Unix(0, int64(binary.BigEndian.Uint64(records[:8]))), + // "Actual:", getContents.Timestamp) + // } + + if getContents.Checksum != binary.BigEndian.Uint32(records[8:12]) { + t.Errorf("Was not able to retrieve the proper checksum\nExpected:%v\nActual:%v", + binary.BigEndian.Uint32(records[8:12]), getContents.Timestamp) + } +} diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index b9f3320..a350e20 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -11,123 +11,121 @@ import ( ) const ( - COMPACTION_SIZE = 4; - INDEX_BLOCK = 20; + COMPACTION_SIZE = 4 + INDEX_BLOCK = 20 ) -//TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. -func WriteToFile(list [][] byte) (bool, error) { - filepath := "./internals/data"; +// TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. +func WriteToFile(list [][]byte) (bool, error) { + filepath := "./internals/data" filename := "" - files, err := os.ReadDir(filepath); + files, err := os.ReadDir(filepath) if err != nil { - return false, err; + return false, err } if len(files) < 1 { filename = "kd_1.sst" } else { - index,err := strconv.Atoi(strings.Split(files[len(files)-1].Name(), "_")[1]); + index, err := strconv.Atoi(strings.Split(files[len(files)-1].Name(), "_")[1]) if err != nil { - return false, err; + return false, err } - filename = "kd_"+strconv.Itoa(index+1)+".sst" + filename = "kd_" + strconv.Itoa(index+1) + ".sst" } - file, err := os.Create(filename); + file, err := os.Create(filename) if err != nil { - return false, err; + return false, err } - - contents := list; - offset := fileOffset(contents); - index := createIndexBlock(contents, offset); - - contents = append(contents, index); + contents := list + offset := fileOffset(contents) + index := createIndexBlock(contents, offset) - content := slices.Concat(contents...); - - _,err = file.Write(content); + contents = append(contents, index) + + content := slices.Concat(contents...) + + _, err = file.Write(content) if err != nil { - return false, err; + return false, err } - file.Sync(); + file.Sync() - compact(files); + compact(files) - return true, nil; + return true, nil } -//Create the indexing block for the file - /* - The indexing block is responsible for allowing us to perform reads much more efficiently. +//Create the indexing block for the file +/* + The indexing block is responsible for allowing us to perform reads much more efficiently. - How is it done: - The skiplist would be called to write into the file after 3200 entries are filled. - When this is done, we can categorize the file into 20 sections of 160 elements. + How is it done: + The skiplist would be called to write into the file after 3200 entries are filled. + When this is done, we can categorize the file into 20 sections of 160 elements. - In the indexing block, each entry would represent a specific block. - The entry will contain the key and the byte offset of the lowest key value in the block. - So it would look like this for example - keyA=0,keyB=202,keyC=403, etc. + In the indexing block, each entry would represent a specific block. + The entry will contain the key and the byte offset of the lowest key value in the block. + So it would look like this for example + keyA=0,keyB=202,keyC=403, etc. - Since all of the records in the file are organized in sorted order, we can use the block to traverse through the file much quicker + Since all of the records in the file are organized in sorted order, we can use the block to traverse through the file much quicker - For example, if we have a key called N, we would start by comparing the key of the 20th block, - if the key is greater than or equal to the key N we would start from the byte offset of that block, - else we keep on going backwards by 1 block until we find a block where the key is greater than or equal to the block. + For example, if we have a key called N, we would start by comparing the key of the 20th block, + if the key is greater than or equal to the key N we would start from the byte offset of that block, + else we keep on going backwards by 1 block until we find a block where the key is greater than or equal to the block. - Only issue with this method. In the case where the key is not located in the file, we would still need to do this traversal, - Solution using bloom filters. - - Format: - Size of index block: uint64 - Each entry will follow like this: - KeySize: uint32 bytes long; - Key: n bytes long (string) (n for keysize) - Byte offset: uint64 - */ + Only issue with this method. In the case where the key is not located in the file, we would still need to do this traversal, + Solution using bloom filters. + + Format: + Size of index block: uint64 + Each entry will follow like this: + KeySize: uint32 bytes long; + Key: n bytes long (string) (n for keysize) + Byte offset: uint64 +*/ func createIndexBlock(contentList [][]byte, offset []uint64) []byte { - index := make([]byte, 0, INDEX_BLOCK * 16); + index := make([]byte, 0, INDEX_BLOCK*16) + + for i := 0; i < len(contentList); i = i + (len(contentList) / INDEX_BLOCK) { + contents := record.GetContents(contentList[i]) - for i:=0; i < len(contentList); i=i+(len(contentList)/INDEX_BLOCK) { - _,_,_,keysize,_,key,_:= record.GetContents(contentList[i]); + index = binary.BigEndian.AppendUint32(index, contents.Keysize) - index = binary.BigEndian.AppendUint32(index, uint32(keysize)); - - index = append(index, key...); + index = append(index, contents.Key...) - index = binary.BigEndian.AppendUint64(index, offset[i]); + index = binary.BigEndian.AppendUint64(index, offset[i]) } - var size uint64 = uint64(len(index)); - index = append(binary.BigEndian.AppendUint64([]byte{}, size), index...); + var size uint64 = uint64(len(index)) + index = append(binary.BigEndian.AppendUint64([]byte{}, size), index...) - return index; + return index } func fileOffset(contentList [][]byte) []uint64 { - var offsetTracker uint64 = 0; - offset := make([]uint64, len(contentList)); - - for i, v := range(contentList) { - recordsize := len(v); - - offset[i] = offsetTracker; - - offsetTracker += uint64(recordsize); + var offsetTracker uint64 = 0 + offset := make([]uint64, len(contentList)) + + for i, v := range contentList { + recordsize := len(v) + + offset[i] = offsetTracker + + offsetTracker += uint64(recordsize) } - - return offset; + + return offset } func compact(files []os.DirEntry) { - } From e8a51e6fc0ce06447f287fbc28e8ada62d588dc3 Mon Sep 17 00:00:00 2001 From: Ivers Date: Thu, 4 Jun 2026 13:39:09 -0400 Subject: [PATCH 14/34] Combination of the previous commit --- internals/record/record_test.go | 37 +++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/internals/record/record_test.go b/internals/record/record_test.go index 1a488d6..a637281 100644 --- a/internals/record/record_test.go +++ b/internals/record/record_test.go @@ -325,14 +325,39 @@ func TestGetContents(t *testing.T) { getContents := record.GetContents(records) - // if time.Unix(0, int64(binary.BigEndian.Uint64(records[:8]))) != getContents.Timestamp { - // t.Errorf("Was not able to retrieve the proper timestamp\n", - // "Expected:", time.Unix(0, int64(binary.BigEndian.Uint64(records[:8]))), - // "Actual:", getContents.Timestamp) - // } + if time.Unix(0, int64(binary.BigEndian.Uint64(records[:8]))) != getContents.Timestamp { + t.Errorf("Was not able to retrieve the proper timestamp\nExpected:%v\nActual:%v", + time.Unix(0, int64(binary.BigEndian.Uint64(records[:8]))), + getContents.Timestamp) + } if getContents.Checksum != binary.BigEndian.Uint32(records[8:12]) { t.Errorf("Was not able to retrieve the proper checksum\nExpected:%v\nActual:%v", - binary.BigEndian.Uint32(records[8:12]), getContents.Timestamp) + binary.BigEndian.Uint32(records[8:12]), getContents.Checksum) + } + + if getContents.Tombstone != uint8(records[12]) { + t.Errorf("Was not able to retrieve the proper tombstone\nExpected:%v\nActual:%v", + uint8(records[12]), getContents.Tombstone) + } + + if getContents.Keysize != binary.BigEndian.Uint32(records[13:17]) { + t.Errorf("Was not able to retrieve the proper key size\nExpected:%v\nActual:%v", + binary.BigEndian.Uint32(records[13:17]), getContents.Keysize) + } + + if getContents.Payloadsize != binary.BigEndian.Uint32(records[17:21]) { + t.Errorf("Was not able to retrieve the proper payload size\nExpected:%v\nActual:%v", + binary.BigEndian.Uint32(records[17:21]), getContents.Payloadsize) + } + + if getContents.Key != string(records[21:getContents.Keysize+21]) { + t.Errorf("Was not able to retrieve the proper key\nExpected:%v\nActual:%v", + binary.BigEndian.Uint32(records[21:getContents.Keysize+21]), getContents.Key) + } + + if getContents.Payload != string(records[getContents.Keysize+21:]) { + t.Errorf("Was not able to retrieve the proper payload\nExpected:%v\nActual:%v", + string(records[getContents.Keysize+21:]), getContents.Payload) } } From a295f165e4f9a73e57f694d8db5cc36402127697 Mon Sep 17 00:00:00 2001 From: Ivers Date: Fri, 5 Jun 2026 15:21:27 -0400 Subject: [PATCH 15/34] Was able to make a test for the buckets method for the establishing the size based compaction. Just need to do a couple of more unit test for more skewed data types --- internals/sstable/writer.go | 77 +++++++++++++++++++++++++++++--- internals/sstable/writer_test.go | 75 +++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 internals/sstable/writer_test.go diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index a350e20..b06c938 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -2,6 +2,8 @@ package sstable import ( "encoding/binary" + "fmt" + "io/fs" "os" "slices" "strconv" @@ -10,14 +12,20 @@ import ( "github.com/Vince-maple-byte/KeyData/internals/record" ) +type file_buckets float64 + const ( - COMPACTION_SIZE = 4 - INDEX_BLOCK = 20 + COMPACTION_SIZE = 4 + INDEX_BLOCK = 20 + SMALL file_buckets = 0.5 + MEDIUM file_buckets = 1.0 + LARGE file_buckets = 1.5 + OVERSIZE file_buckets = 2.0 ) // TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. func WriteToFile(list [][]byte) (bool, error) { - filepath := "./internals/data" + filepath := "../data" filename := "" files, err := os.ReadDir(filepath) @@ -58,7 +66,7 @@ func WriteToFile(list [][]byte) (bool, error) { file.Sync() - compact(files) + buckets(files) return true, nil } @@ -126,6 +134,65 @@ func fileOffset(contentList [][]byte) []uint64 { return offset } -func compact(files []os.DirEntry) { +// We are going to be doing size based compaction for compacting these files +// The amount of files that need to be a similar size +func buckets(files []os.DirEntry) map[file_buckets][]fs.FileInfo { + // min_threshold := 4 + // max_threshold := 32 + var average_size int64 + var total_size int64 + //var largest_size int64 = 0 + //var min + + buckets := make(map[file_buckets][]fs.FileInfo) + + buckets[SMALL] = make([]fs.FileInfo, 0) + buckets[MEDIUM] = make([]fs.FileInfo, 0) + buckets[LARGE] = make([]fs.FileInfo, 0) + buckets[OVERSIZE] = make([]fs.FileInfo, 0) + + for _, file := range files { + + fileInfo, err := file.Info() + + if err != nil { + continue + } + + total_size += fileInfo.Size() + } + + average_size = total_size / int64(len(files)) + fmt.Println(average_size) + fmt.Println(total_size) + + for _, file := range files { + fileInfo, _ := file.Info() + size := fileInfo.Size() + + switch { + case float64(size) >= float64(average_size)*float64(OVERSIZE): + buckets[OVERSIZE] = append(buckets[OVERSIZE], fileInfo) + case float64(size) >= float64(average_size)*float64(LARGE): + buckets[LARGE] = append(buckets[LARGE], fileInfo) + case float64(size) >= float64(average_size)*float64(MEDIUM): + buckets[MEDIUM] = append(buckets[MEDIUM], fileInfo) + default: + buckets[SMALL] = append(buckets[SMALL], fileInfo) + } + } + + return buckets + // for _, bucket := range buckets { + // if len(bucket) >= min_threshold || len(bucket) <= max_threshold { + // //We do the compacting here + // fmt.Println("Compacting here", bucket[0].Name()) + // break + // } + // } +} + +func ExportCompact(files []os.DirEntry) map[file_buckets][]fs.FileInfo { + return compact(files) } diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go new file mode 100644 index 0000000..3df7b7e --- /dev/null +++ b/internals/sstable/writer_test.go @@ -0,0 +1,75 @@ +package sstable_test + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/Vince-maple-byte/KeyData/internals/record" + "github.com/Vince-maple-byte/KeyData/internals/sstable" +) + +func TestCheck(t *testing.T) { + if 2+2 == 5 { + t.Error() + } +} + +func startUp(data ...string) error { + for i := range 10 { + targetDir := "../test" + name := fmt.Sprintf("%s_%d.txt", "testfile", i) + fullPath := filepath.Join(targetDir, name) + file, err := os.Create(fullPath) + + if err != nil { + return err + } + + //This is 31 bytes long for the even method and the increasing method will be +1 + w, _ := record.CreateRecord(data[i], data[i], "PUT") + + file.Write(w) + file.Close() + } + + return nil +} + +func TestBucketsForEvenlyDistributedFiles(t *testing.T) { + err := startUp() + filePath := "../test" + if err != nil { + t.Fatalf("Could not start up the test") + } + + filePath = "../test" + files, err := os.ReadDir(filePath) + + if err != nil { + t.Fatalf("Could not make the file path") + } + + buckets := sstable.ExportCompact(files) + totalSize := 310 + average := totalSize / 10 + + for key, bucket := range buckets { + + if len(bucket) == 0 { + //t.Logf("This bucket: %v is empty", key) + } else { + t.Logf("This bucket %v is %d length\n", key, len(bucket)) + + for _, item := range bucket { + if float64(item.Size()) < float64(average)*float64(key) { + t.Errorf("This file should not be in this bucket:%v\nFile name:%s\tFile size:%d", + key, item.Name(), item.Size()) + } + //t.Logf("File %s is %d large\n", item.Name(), item.Size()) + } + } + } + +} From 8f7996312d823f2c2f496e95ea4c3e1cf8b548f7 Mon Sep 17 00:00:00 2001 From: Ivers Date: Sat, 6 Jun 2026 15:05:38 -0400 Subject: [PATCH 16/34] Able to finish with the unit tests regarding making the buckets to organize the sstables by size. Now we just need to make the function to compact these files into one file. --- internals/sstable/writer.go | 16 ++----- internals/sstable/writer_test.go | 76 +++++++++++++++++++++----------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index b06c938..689b36c 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -2,7 +2,6 @@ package sstable import ( "encoding/binary" - "fmt" "io/fs" "os" "slices" @@ -146,6 +145,8 @@ func buckets(files []os.DirEntry) map[file_buckets][]fs.FileInfo { //var min buckets := make(map[file_buckets][]fs.FileInfo) + // + // buckets[SMALL] = make([]fs.FileInfo, 0) buckets[MEDIUM] = make([]fs.FileInfo, 0) @@ -164,8 +165,6 @@ func buckets(files []os.DirEntry) map[file_buckets][]fs.FileInfo { } average_size = total_size / int64(len(files)) - fmt.Println(average_size) - fmt.Println(total_size) for _, file := range files { fileInfo, _ := file.Info() @@ -184,15 +183,8 @@ func buckets(files []os.DirEntry) map[file_buckets][]fs.FileInfo { } return buckets - // for _, bucket := range buckets { - // if len(bucket) >= min_threshold || len(bucket) <= max_threshold { - // //We do the compacting here - // fmt.Println("Compacting here", bucket[0].Name()) - // break - // } - // } } -func ExportCompact(files []os.DirEntry) map[file_buckets][]fs.FileInfo { - return compact(files) +func ExportBuckets(files []os.DirEntry) map[file_buckets][]fs.FileInfo { + return buckets(files) } diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go index 3df7b7e..836a584 100644 --- a/internals/sstable/writer_test.go +++ b/internals/sstable/writer_test.go @@ -16,7 +16,8 @@ func TestCheck(t *testing.T) { } } -func startUp(data ...string) error { +func startUp(data ...string) (int, error) { + size := 0 for i := range 10 { targetDir := "../test" name := fmt.Sprintf("%s_%d.txt", "testfile", i) @@ -24,50 +25,73 @@ func startUp(data ...string) error { file, err := os.Create(fullPath) if err != nil { - return err + return 0, err } //This is 31 bytes long for the even method and the increasing method will be +1 w, _ := record.CreateRecord(data[i], data[i], "PUT") + size += len(w) file.Write(w) file.Close() } - return nil + return size, nil } -func TestBucketsForEvenlyDistributedFiles(t *testing.T) { - err := startUp() - filePath := "../test" - if err != nil { - t.Fatalf("Could not start up the test") +func TestBucketsForFiles(t *testing.T) { + tests := []struct { + testName string + data []string + }{ + { + testName: "Same size for each file", + data: []string{"data", "data", "data", "data", "data", "data", "data", "data", "data", "data"}, + }, + { + testName: "Size increasing for each file", + data: []string{"data1", "data11", "data111", "data1111", "data11111", + "data111111", "data1111111", "data11111111", "data111111111", "data1111111111"}, + }, + { + testName: "Size decreasing for each file", + data: []string{"data1111111111", "data111111111", "data11111111", "data1111111", "data111111", + "data11111", "data1111", "data111", "data11", "data1"}, + }, + { + testName: "Random order of size for each file", + data: []string{"data111", "data1111", "data1", "data1111111", "data111111111", + "data11111", "data1111", "data1111111111", "data11", "data11111111"}, + }, } - filePath = "../test" - files, err := os.ReadDir(filePath) + for _, test := range tests { + totalSize, err := startUp(test.data...) + filePath := "../test" + if err != nil { + t.Fatalf("Could not start up the test") + } + files, err := os.ReadDir(filePath) - if err != nil { - t.Fatalf("Could not make the file path") - } + if err != nil { + t.Fatalf("Could not make the file path") + } - buckets := sstable.ExportCompact(files) - totalSize := 310 - average := totalSize / 10 + buckets := sstable.ExportBuckets(files) + average := totalSize / 10 - for key, bucket := range buckets { + for key, bucket := range buckets { - if len(bucket) == 0 { - //t.Logf("This bucket: %v is empty", key) - } else { - t.Logf("This bucket %v is %d length\n", key, len(bucket)) + if len(bucket) != 0 { + t.Logf("This bucket %v is %d length\n", key, len(bucket)) - for _, item := range bucket { - if float64(item.Size()) < float64(average)*float64(key) { - t.Errorf("This file should not be in this bucket:%v\nFile name:%s\tFile size:%d", - key, item.Name(), item.Size()) + for _, item := range bucket { + if float64(item.Size()) < float64(average)*float64(key) { + t.Errorf("For test:%s\nThis file should not be in this bucket:%v\nFile name:%s\tFile size:%d", + test.testName, + key, item.Name(), item.Size()) + } } - //t.Logf("File %s is %d large\n", item.Name(), item.Size()) } } } From 15a568a5abb5a95e9a392d49ee9855d626a56b97 Mon Sep 17 00:00:00 2001 From: Ivers Date: Sun, 7 Jun 2026 14:03:21 -0400 Subject: [PATCH 17/34] Changed the function of insert so that when a key that is being added to the skiplist already exist, we will check for which key/value pair is newer, the old one or the new one. From there we save the newest one into the database. --- internals/memtable/skiplist.go | 145 +++++++++++++++------------- internals/memtable/skiplist_test.go | 45 +++++++-- 2 files changed, 115 insertions(+), 75 deletions(-) diff --git a/internals/memtable/skiplist.go b/internals/memtable/skiplist.go index 0fa72fd..5a620df 100644 --- a/internals/memtable/skiplist.go +++ b/internals/memtable/skiplist.go @@ -3,13 +3,15 @@ package memtable import ( "errors" "math/rand/v2" + + "github.com/Vince-maple-byte/KeyData/internals/record" ) -const MAX_LEVEL = 32; +const MAX_LEVEL = 32 type Node struct { - key string - value []byte + key string + value []byte levels []*Node } @@ -20,108 +22,115 @@ type Skiplist struct { func CreateSkiplist() *Skiplist { head := &Node{ - key: "", - value: nil, + key: "", + value: nil, levels: make([]*Node, MAX_LEVEL), - }; + } return &Skiplist{ head: head, size: 0, - }; + } } func (list *Skiplist) Search(key string) ([]byte, error) { - curr := list.head; - var next *Node; + curr := list.head + var next *Node + + for i := len(list.head.levels) - 1; i >= 0; i-- { - for i := len(list.head.levels)-1; i >= 0; i-- { - for next = curr.levels[i]; next != nil && key > next.key; next = curr.levels[i] { - curr = next; + curr = next } if next != nil && next.key == key { - return next.value, nil; + return next.value, nil } - } + } - return nil, errors.New("Element doesn't exist"); + return nil, errors.New("Element doesn't exist") } func (list *Skiplist) Insert(key string, value []byte) { - var update [MAX_LEVEL]*Node; - curr := list.head; + var update [MAX_LEVEL]*Node + curr := list.head - for i:= MAX_LEVEL-1; i >= 0; i-- { + for i := MAX_LEVEL - 1; i >= 0; i-- { for curr.levels[i] != nil && key > curr.levels[i].key { - curr = curr.levels[i]; + curr = curr.levels[i] } - update[i] = curr; + update[i] = curr } - + //This code block checks if the key is already inside of the skiplist - next := curr.levels[0]; + // If the key is already inside then we can check for whichever key is the newest by time and then have that value in the skiplist + next := curr.levels[0] if next != nil && next.key == key { - next.value = value; - return; + newTime := record.GetContents(value).Timestamp.UTC() + prevTime := record.GetContents(next.value).Timestamp.UTC() + + if newTime.After(prevTime) || newTime.Equal(prevTime) { + next.value = value + } + + return } newNode := &Node{ - key: key, - value: value, + key: key, + value: value, levels: make([]*Node, randLevel()), } for i := 0; i < len(newNode.levels); i++ { - newNode.levels[i] = update[i].levels[i]; - update[i].levels[i] = newNode; + newNode.levels[i] = update[i].levels[i] + update[i].levels[i] = newNode } - list.size++; + list.size++ } -func (list *Skiplist) Delete(key string) ([]byte,error) { - deleteNode, err := list.searchNode(key); +func (list *Skiplist) Delete(key string) ([]byte, error) { + deleteNode, err := list.searchNode(key) if err != nil { - return nil, err; + return nil, err } - update := make([]*Node, len(deleteNode.levels)); - curr := list.head; - val := deleteNode.value; + update := make([]*Node, len(deleteNode.levels)) + curr := list.head + val := deleteNode.value - for i:= len(update)-1; i>=0; i-- { + for i := len(update) - 1; i >= 0; i-- { for curr.levels[i] != nil && key > curr.levels[i].key { - curr = curr.levels[i] - } + curr = curr.levels[i] + } - update[i] = curr; + update[i] = curr } - for i := range(len(update)) { - update[i].levels[i] = deleteNode.levels[i]; + for i := range len(update) { + update[i].levels[i] = deleteNode.levels[i] } - - list.size--; - return val, nil; + + list.size-- + return val, nil } func (list Skiplist) searchNode(key string) (*Node, error) { - curr := list.head; - var next *Node; + curr := list.head + var next *Node + + for i := len(list.head.levels) - 1; i >= 0; i-- { - for i := len(list.head.levels)-1; i >= 0; i-- { - for next = curr.levels[i]; next != nil && key > next.key; next = curr.levels[i] { - curr = next; + curr = next } if next != nil && next.key == key { - return next, nil; + return next, nil } - } + } - return nil, errors.New("Element doesn't exist"); + return nil, errors.New("Element doesn't exist") } func (list *Skiplist) EmptyList() { @@ -134,31 +143,31 @@ func (list *Skiplist) EmptyList() { // head: head, // size: 0, // }; - list.head.key = ""; - list.head.value = nil; - list.head.levels = make([]*Node, MAX_LEVEL); + list.head.key = "" + list.head.value = nil + list.head.levels = make([]*Node, MAX_LEVEL) - list.size = 0; + list.size = 0 } -func (list *Skiplist) EntireList() ([][]byte) { - curr := list.head.levels[0]; - res := make([][]byte, list.size); - - for range(list.size) { - res = append(res, curr.value); - curr = curr.levels[0]; +func (list *Skiplist) EntireList() [][]byte { + curr := list.head.levels[0] + res := make([][]byte, list.size) + + for range list.size { + res = append(res, curr.value) + curr = curr.levels[0] } - return res; + return res } func randLevel() int { - var level int = 1; + var level int = 1 - for(rand.IntN(2) == 1 && level < MAX_LEVEL){ - level++; + for rand.IntN(2) == 1 && level < MAX_LEVEL { + level++ } - return level; + return level } diff --git a/internals/memtable/skiplist_test.go b/internals/memtable/skiplist_test.go index 8283f9a..b49d8f5 100644 --- a/internals/memtable/skiplist_test.go +++ b/internals/memtable/skiplist_test.go @@ -2,6 +2,9 @@ package memtable import ( "testing" + "time" + + "github.com/Vince-maple-byte/KeyData/internals/record" ) // --- helpers --- @@ -55,10 +58,36 @@ func TestInsert_MultipleElements(t *testing.T) { } } -func TestInsert_UpdateExistingKey(t *testing.T) { +func TestInsert_UpdateExistingKeyForDifferentTimesItWasInserted(t *testing.T) { + list := newList(t) + old, _ := record.CreateRecord("key", "old", "PUT") + time.Sleep(2 * time.Millisecond) + new, _ := record.CreateRecord("key", "new", "PUT") + list.Insert("key", old) + list.Insert("key", new) + + // Size should not grow on update. + if list.size != 1 { + t.Errorf("expected size 1 after update, got %d", list.size) + } + + val, err := list.Search("key") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if record.GetContents(val).Payload != record.GetContents(new).Payload { + t.Errorf("expected 'new', got %q\nTime for old:%v\nTime for new:%v", + val, record.GetContents(old).Timestamp, record.GetContents(new).Timestamp) + + } +} + +func TestInsert_UpdateExistingKeyWhenInsertedAtTheSameTime(t *testing.T) { list := newList(t) - list.Insert("key", []byte("old")) - list.Insert("key", []byte("new")) + old, _ := record.CreateRecord("key", "old", "PUT") + new, _ := record.CreateRecord("key", "new", "PUT") + list.Insert("key", old) + list.Insert("key", new) // Size should not grow on update. if list.size != 1 { @@ -69,8 +98,10 @@ func TestInsert_UpdateExistingKey(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if string(val) != "new" { - t.Errorf("expected 'new', got %q", val) + if record.GetContents(val).Payload != record.GetContents(new).Payload { + t.Errorf("expected 'new', got %q\nTime for old:%v\nTime for new:%v", + val, record.GetContents(old).Timestamp, record.GetContents(new).Timestamp) + } } @@ -265,6 +296,6 @@ func TestEmptyList_SizeAfterEmpty(t *testing.T) { // behaviour — update if the implementation is fixed. size := list.size // no panic expected if size != 0 { - t.Error("Expected an empty list",size); + t.Error("Expected an empty list", size) } -} \ No newline at end of file +} From 3fb5e067a52c385e2f11b2ac771c178ca133fef9 Mon Sep 17 00:00:00 2001 From: Ivers Date: Sun, 7 Jun 2026 22:28:50 -0400 Subject: [PATCH 18/34] Completed the initial implementation of the compaction function. Next up testing the code. --- internals/memtable/memtable.go | 40 ++++++------- internals/memtable/skiplist.go | 1 + internals/record/record.go | 8 +-- internals/sstable/merger.go | 8 +++ internals/sstable/writer.go | 99 +++++++++++++++++++++++++++++--- internals/sstable/writer_test.go | 7 +-- main.go | 16 ++++-- 7 files changed, 135 insertions(+), 44 deletions(-) create mode 100644 internals/sstable/merger.go diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index 7984e2e..8d8a8a9 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -7,7 +7,7 @@ import ( const MAX_SIZE = 3200 -type Memtable struct{ +type Memtable struct { list *Skiplist size int } @@ -20,33 +20,33 @@ func CreateMemtable() *Memtable { } func (memtable *Memtable) Write(key, value, operation string) (bool, error) { - record, err := record.CreateRecord(key, value, operation); + record, err := record.CreateRecord(key, value, operation) if err != nil { - return false, err; + return false, err } - memtable.list.Insert(key, record); - memtable.size += 1; + memtable.list.Insert(key, record) + memtable.size += 1 - if(memtable.size >= MAX_SIZE) { - content := memtable.list.EntireList(); - _, errF := sstable.WriteToFile(content) + if memtable.size >= MAX_SIZE { + content := memtable.list.EntireList() + _, errF := sstable.WriteToFile(content) if errF != nil { - return false, errF; + return false, errF } - memtable.list.EmptyList(); - memtable.size = 0; - + memtable.list.EmptyList() + memtable.size = 0 + } - - return true, nil; + + return true, nil } func (memtable *Memtable) Read(key string) ([]byte, error) { - record, err := memtable.list.Search(key); + record, err := memtable.list.Search(key) if err != nil { //Call the method to perform the file i/o operation for reads @@ -54,15 +54,15 @@ func (memtable *Memtable) Read(key string) ([]byte, error) { //If we still return an error we would return a nil/empty byte slice and a error } - return record, nil; + return record, nil } func (memtable *Memtable) Get(key string) ([]byte, error) { - record, err := memtable.list.Search(key); + record, err := memtable.list.Search(key) if err != nil { - return nil,err; + return nil, err } - return record, nil; -} + return record, nil +} diff --git a/internals/memtable/skiplist.go b/internals/memtable/skiplist.go index 5a620df..c55cb2d 100644 --- a/internals/memtable/skiplist.go +++ b/internals/memtable/skiplist.go @@ -49,6 +49,7 @@ func (list *Skiplist) Search(key string) ([]byte, error) { return nil, errors.New("Element doesn't exist") } +// The value is the entire record including all of the information on the timestamp, etc. func (list *Skiplist) Insert(key string, value []byte) { var update [MAX_LEVEL]*Node curr := list.head diff --git a/internals/record/record.go b/internals/record/record.go index 4788744..07d0b73 100644 --- a/internals/record/record.go +++ b/internals/record/record.go @@ -24,12 +24,12 @@ type Content struct { Payload string } -func checkSum(data string) uint32 { - return crc32.ChecksumIEEE([]byte(data)) +func checkSum(payload string) uint32 { + return crc32.ChecksumIEEE([]byte(payload)) } -func ChecksumChecker(data []byte, checksum uint32) bool { - valid := crc32.ChecksumIEEE(data) +func ChecksumChecker(payload []byte, checksum uint32) bool { + valid := crc32.ChecksumIEEE(payload) if valid == checksum { return true diff --git a/internals/sstable/merger.go b/internals/sstable/merger.go new file mode 100644 index 0000000..4e3a751 --- /dev/null +++ b/internals/sstable/merger.go @@ -0,0 +1,8 @@ +package sstable + +type ListMerger interface { + Insert(key string, value []byte) + EntireList() [][]byte +} + +var MergeList func() ListMerger diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index 689b36c..e80dd64 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -65,7 +65,7 @@ func WriteToFile(list [][]byte) (bool, error) { file.Sync() - buckets(files) + //buckets(files) return true, nil } @@ -136,13 +136,16 @@ func fileOffset(contentList [][]byte) []uint64 { // We are going to be doing size based compaction for compacting these files // The amount of files that need to be a similar size -func buckets(files []os.DirEntry) map[file_buckets][]fs.FileInfo { - // min_threshold := 4 - // max_threshold := 32 +// TODO: Need to make the bucket map into a persistent map that is used throughout the entire state of the +// program.w +func buckets(filePath string) (map[file_buckets][]fs.FileInfo, error) { + files, err := os.ReadDir(filePath) + + if err != nil { + return nil, err + } var average_size int64 var total_size int64 - //var largest_size int64 = 0 - //var min buckets := make(map[file_buckets][]fs.FileInfo) // @@ -182,9 +185,87 @@ func buckets(files []os.DirEntry) map[file_buckets][]fs.FileInfo { } } - return buckets + return buckets, nil } -func ExportBuckets(files []os.DirEntry) map[file_buckets][]fs.FileInfo { - return buckets(files) +// When we are doing the concurrency option. Compaction and the buckets will be done in a separate class, +// and in it's own separate thread that will be run periodically every time interval that we decide +func Compact(filePath string) error { + bucketMap, err := buckets(filePath) + + if err != nil { + return err + } + minThreshold := 4 + maxThreshold := 32 + + //Slight potential optimization: We can have this continue compacting each bucket of similar sizes, + // but we have to recalculate the average file size and bucket arrangement for Each of the files + for _, val := range bucketMap { + bucketSize := len(val) + + if bucketSize >= minThreshold && bucketSize <= maxThreshold { + //compactMap := make(map[string][]byte, 0); + skiplist := MergeList() + for _, fileInfo := range val { + fileData, err := os.ReadFile(fileInfo.Name()) + + if err != nil { + return err + } + + //We are going through each file and from there we will save each key/value pair record into a skiplist + for i := 0; i < len(fileData); i++ { + //header := fileData[i : i+21] + + checksum := binary.BigEndian.Uint32(fileData[i+8 : i+12]) + + keySize := int(binary.BigEndian.Uint32(fileData[i+13 : i+17])) + payloadSize := int(binary.BigEndian.Uint32(fileData[i+17 : i+21])) + + key := fileData[i+21 : i+keySize+21] + payload := fileData[i+keySize+21 : i+(keySize+21)+payloadSize] + + //If the checksum is invalid, we ignore the rest of the file + if !record.ChecksumChecker(payload, checksum) { + break + } + + skiplist.Insert(string(key), fileData[i:i+(keySize+21)+payloadSize]) + i = i + (keySize + 21) + payloadSize + } + } + + //We take the entire skiplist and write it into a new file + fileContents := skiplist.EntireList() + + //Technically speaking we can just recall the write to file again to make the new file + // Since the Entire write operation is there. + // So I'm planning on calling the Compact function after the write to file function goes through in the Memtable class + _, err := WriteToFile(fileContents) + + if err != nil { + return err + } + + // Delete all of the old files once the new file is committed + for _, fileInfo := range val { + err := os.Remove(fileInfo.Name()) + + if err != nil { + return err + } + } + break + } + + } + + return nil +} + +func ExportBuckets(filePath string) map[file_buckets][]fs.FileInfo { + result, _ := buckets(filePath) + + return result } diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go index 836a584..3964e5d 100644 --- a/internals/sstable/writer_test.go +++ b/internals/sstable/writer_test.go @@ -71,13 +71,8 @@ func TestBucketsForFiles(t *testing.T) { if err != nil { t.Fatalf("Could not start up the test") } - files, err := os.ReadDir(filePath) - if err != nil { - t.Fatalf("Could not make the file path") - } - - buckets := sstable.ExportBuckets(files) + buckets := sstable.ExportBuckets(filePath) average := totalSize / 10 for key, bucket := range buckets { diff --git a/main.go b/main.go index e9bfff6..fed5096 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,10 @@ package main +import ( + "github.com/Vince-maple-byte/KeyData/internals/memtable" + "github.com/Vince-maple-byte/KeyData/internals/sstable" +) + //"time" // "github.com/Vince-maple-byte/KeyData/internals/file" //"github.com/Vince-maple-byte/KeyData/internals/record" @@ -18,9 +23,9 @@ time stamp: 8 bytes CRC32: 4 bytes Tombstone: 1 byte key size: 4 bytes -payload size: 4 bytes +payload size: 4 bytes -> The header of the record is 21 bytes long key: (key size) bytes -payload: (payload size) bytes +payload: (payload size) bytes -> The rest of the record is 21 + keySize + payloadSize bytes long Have it so that in the Write method after we are done adding the record into the file, make sure to add flush(), we update the index map with the key and byte offset @@ -34,7 +39,8 @@ From there we can move on to SSTable, compaction, LSM Tables, and finally bloom */ func main() { - - - + //We need to use this interface here to avoid an import cycle between sstable and memtable + sstable.MergeList = func() sstable.ListMerger { + return memtable.CreateSkiplist() + } } From c7f2b8856f71d3c733f2c7a57c2a00ddca8315bb Mon Sep 17 00:00:00 2001 From: Ivers Date: Mon, 8 Jun 2026 10:19:35 -0400 Subject: [PATCH 19/34] Able to make tests for the memtable class. To test the flush we just need to make sure that the compact function works --- internals/memtable/memtable.go | 7 ++ internals/memtable/memtable_test.go | 99 +++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 internals/memtable/memtable_test.go diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index 8d8a8a9..29504b7 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -37,6 +37,13 @@ func (memtable *Memtable) Write(key, value, operation string) (bool, error) { return false, errF } + //For now, when running test we just replace the folder path as the test folder. + err = sstable.Compact("../data") + + if err != nil { + return false, err + } + memtable.list.EmptyList() memtable.size = 0 diff --git a/internals/memtable/memtable_test.go b/internals/memtable/memtable_test.go new file mode 100644 index 0000000..6f8ac52 --- /dev/null +++ b/internals/memtable/memtable_test.go @@ -0,0 +1,99 @@ +package memtable + +import ( + "testing" + + "github.com/Vince-maple-byte/KeyData/internals/record" +) + +func TestWriteForSingleEntryInMemtable(t *testing.T) { + mem := CreateMemtable() + prevSize := mem.size + _, err := mem.Write("key", "val", "PUT") + + if err != nil { + t.Fatalf("Was not able to process the delete operation for the memtable") + } + + if mem.size <= prevSize { + t.Errorf("The write operation did not successfully increase the memtable size") + } +} + +func TestWriteForMultipleEntriesInMemtable(t *testing.T) { + mem := CreateMemtable() + + tests := []struct { + key, value, op string + }{ + {"key1", "val1", "PUT"}, + {"key2", "val2", "PUT"}, + {"key3", "val3", "PUT"}, + {"key4", "val4", "PUT"}, + {"key5", "val5", "PUT"}, + } + + for _, test := range tests { + prevSize := mem.size + _, err := mem.Write(test.key, test.value, test.op) + + if err != nil { + t.Fatalf("Was not able to process the delete operation for the memtable") + } + + if mem.size <= prevSize { + t.Errorf("The delete operation did not successfully increase the memtable size") + } + } + +} + +func TestReadForMemtable(t *testing.T) { + mem := CreateMemtable() + + mem.Write("key", "val", "PUT") + actual, err := mem.Read("key") + + if err != nil { + t.Errorf("The read was not able to complete (could mean that the key was not saved into the memtable)") + } + + if record.GetContents(actual).Key != "key" { + t.Errorf("The key that was retrieved was not the same as the key/value pair that was written to the memtable") + } +} + +func TestDeleteForMemtable(t *testing.T) { + mem := CreateMemtable() + + mem.Write("key", "val", "PUT") + prevSize := mem.size + _, err := mem.Write("key", "", "DELETE") + + if err != nil { + t.Fatalf("Was not able to process the delete operation for the memtable") + } + + if mem.size <= prevSize { + t.Errorf("The delete operation did not successfully increase the memtable size") + } + +} + +func TestWrite_FlushResetsState(t *testing.T) { + t.Skip("requires sstable wiring and writable data dir — run as integration test") + + mt := CreateMemtable() + + for i := 0; i < MAX_SIZE; i++ { + key := "key" + string(rune(i)) + _, err := mt.Write(key, "value", "PUT") + if err != nil { + t.Fatalf("unexpected error at write %d: %v", i, err) + } + } + + if mt.size != 0 { + t.Errorf("expected size to reset to 0 after flush, got %d", mt.size) + } +} From 54f0dd50d13856df004f70d724826cfd23f57618 Mon Sep 17 00:00:00 2001 From: Ivers Date: Mon, 8 Jun 2026 13:28:01 -0400 Subject: [PATCH 20/34] Finised with the compaction function and testing it. Next up the reading from file --- internals/memtable/skiplist.go | 2 +- internals/sstable/writer.go | 51 ++++++++++++++++++--------- internals/sstable/writer_test.go | 59 +++++++++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 18 deletions(-) diff --git a/internals/memtable/skiplist.go b/internals/memtable/skiplist.go index c55cb2d..e1a955b 100644 --- a/internals/memtable/skiplist.go +++ b/internals/memtable/skiplist.go @@ -153,7 +153,7 @@ func (list *Skiplist) EmptyList() { func (list *Skiplist) EntireList() [][]byte { curr := list.head.levels[0] - res := make([][]byte, list.size) + res := make([][]byte, 0, list.size) for range list.size { res = append(res, curr.value) diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index e80dd64..4866342 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -2,8 +2,10 @@ package sstable import ( "encoding/binary" + "fmt" "io/fs" "os" + "path/filepath" "slices" "strconv" "strings" @@ -24,9 +26,9 @@ const ( // TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. func WriteToFile(list [][]byte) (bool, error) { - filepath := "../data" + filePath := "../test" filename := "" - files, err := os.ReadDir(filepath) + files, err := os.ReadDir(filePath) if err != nil { return false, err @@ -35,7 +37,11 @@ func WriteToFile(list [][]byte) (bool, error) { if len(files) < 1 { filename = "kd_1.sst" } else { - index, err := strconv.Atoi(strings.Split(files[len(files)-1].Name(), "_")[1]) + //This gives us something like 9.txt + str := strings.Split(files[len(files)-1].Name(), "_")[1] + //Spliting it by the . and returning the first element will give us the 9 as a string + str = strings.Split(str, ".")[0] + index, err := strconv.Atoi(str) if err != nil { return false, err @@ -43,19 +49,19 @@ func WriteToFile(list [][]byte) (bool, error) { filename = "kd_" + strconv.Itoa(index+1) + ".sst" } - file, err := os.Create(filename) + file, err := os.Create(filepath.Join(filePath, filename)) if err != nil { return false, err } - contents := list - offset := fileOffset(contents) - index := createIndexBlock(contents, offset) + //contents := list + offset := fileOffset(list) + index := createIndexBlock(list, offset) - contents = append(contents, index) + list = append(list, index) - content := slices.Concat(contents...) + content := slices.Concat(list...) _, err = file.Write(content) @@ -101,16 +107,27 @@ func WriteToFile(list [][]byte) (bool, error) { */ func createIndexBlock(contentList [][]byte, offset []uint64) []byte { - index := make([]byte, 0, INDEX_BLOCK*16) + index := make([]byte, 0, INDEX_BLOCK*160) - for i := 0; i < len(contentList); i = i + (len(contentList) / INDEX_BLOCK) { - contents := record.GetContents(contentList[i]) + if len(contentList) < INDEX_BLOCK*160 { + contents := record.GetContents(contentList[0]) index = binary.BigEndian.AppendUint32(index, contents.Keysize) index = append(index, contents.Key...) - index = binary.BigEndian.AppendUint64(index, offset[i]) + index = binary.BigEndian.AppendUint64(index, offset[0]) + } else { + for i := 0; i < len(contentList); i = i + (len(contentList) / INDEX_BLOCK) { + fmt.Println(i) + contents := record.GetContents(contentList[i]) + + index = binary.BigEndian.AppendUint32(index, contents.Keysize) + + index = append(index, contents.Key...) + + index = binary.BigEndian.AppendUint64(index, offset[i]) + } } var size uint64 = uint64(len(index)) @@ -208,14 +225,15 @@ func Compact(filePath string) error { //compactMap := make(map[string][]byte, 0); skiplist := MergeList() for _, fileInfo := range val { - fileData, err := os.ReadFile(fileInfo.Name()) + + fileData, err := os.ReadFile(filepath.Join(filePath, fileInfo.Name())) if err != nil { return err } //We are going through each file and from there we will save each key/value pair record into a skiplist - for i := 0; i < len(fileData); i++ { + for i := 0; i < len(fileData); { //header := fileData[i : i+21] checksum := binary.BigEndian.Uint32(fileData[i+8 : i+12]) @@ -233,6 +251,7 @@ func Compact(filePath string) error { skiplist.Insert(string(key), fileData[i:i+(keySize+21)+payloadSize]) i = i + (keySize + 21) + payloadSize + } } @@ -250,7 +269,7 @@ func Compact(filePath string) error { // Delete all of the old files once the new file is committed for _, fileInfo := range val { - err := os.Remove(fileInfo.Name()) + err := os.Remove(filepath.Join(filePath, fileInfo.Name())) if err != nil { return err diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go index 3964e5d..edf7789 100644 --- a/internals/sstable/writer_test.go +++ b/internals/sstable/writer_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "testing" + "github.com/Vince-maple-byte/KeyData/internals/memtable" "github.com/Vince-maple-byte/KeyData/internals/record" "github.com/Vince-maple-byte/KeyData/internals/sstable" ) @@ -20,7 +21,7 @@ func startUp(data ...string) (int, error) { size := 0 for i := range 10 { targetDir := "../test" - name := fmt.Sprintf("%s_%d.txt", "testfile", i) + name := fmt.Sprintf("%s_%d.sst", "testfile", i) fullPath := filepath.Join(targetDir, name) file, err := os.Create(fullPath) @@ -39,6 +40,14 @@ func startUp(data ...string) (int, error) { return size, nil } +func tearDown(filePath string) { + files, _ := os.ReadDir(filePath) + + for _, file := range files { + os.Remove(filepath.Join(filePath, file.Name())) + } +} + func TestBucketsForFiles(t *testing.T) { tests := []struct { testName string @@ -91,4 +100,52 @@ func TestBucketsForFiles(t *testing.T) { } } + tearDown("../test") +} + +func TestCompactFilesForFiles(t *testing.T) { + data := []string{"data", "data", "data", "data", "data", "data", "data", "data", "data", "data"} + + startUp(data...) + + sstable.MergeList = func() sstable.ListMerger { + return memtable.CreateSkiplist() + } + + err := sstable.Compact("../test") + + if err != nil { + t.Errorf("Not able to complete the compaction\n Recieved this error code:\n%v", err) + } + + files, err := os.ReadDir("../test") + + if err != nil { + t.Fatalf("Not able to access the directory for testing: ../test") + } + + if len(files) != 1 { + t.Errorf("Improper amount of files inside of the test directory:\nExpected: %d; Actual:%d", 1, len(files)) + } + + tearDown("../test") +} + +func TestWriteToFile(t *testing.T) { + fileContents := make([][]byte, 0, 10) + + for range 10 { + r, _ := record.CreateRecord("a", "1", "PUT") + fileContents = append(fileContents, r) + } + + ok, err := sstable.WriteToFile(fileContents) + + if err != nil { + t.Errorf("Error encountered: %v\n", err) + } + + if !ok { + t.Errorf("Not able to create the file") + } } From c94120d63fa9907e30bfddfe06e2d19c7645c623 Mon Sep 17 00:00:00 2001 From: Ivers Date: Tue, 9 Jun 2026 14:25:41 -0400 Subject: [PATCH 21/34] Finished with testing the writer methods. Now we can confidently move on to the reader i/o --- internals/sstable/writer.go | 56 +++++++++++---- internals/sstable/writer_test.go | 119 ++++++++++++++++++++++++++++--- 2 files changed, 149 insertions(+), 26 deletions(-) diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index 4866342..1635bf1 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -2,6 +2,7 @@ package sstable import ( "encoding/binary" + "errors" "fmt" "io/fs" "os" @@ -37,19 +38,20 @@ func WriteToFile(list [][]byte) (bool, error) { if len(files) < 1 { filename = "kd_1.sst" } else { - //This gives us something like 9.txt - str := strings.Split(files[len(files)-1].Name(), "_")[1] - //Spliting it by the . and returning the first element will give us the 9 as a string - str = strings.Split(str, ".")[0] - index, err := strconv.Atoi(str) - - if err != nil { - return false, err + maxIndex := 0 + for _, f := range files { + str := strings.Split(f.Name(), "_")[1] + str = strings.Split(str, ".")[0] + idx, err := strconv.Atoi(str) + if err == nil && idx > maxIndex { + maxIndex = idx + } } - filename = "kd_" + strconv.Itoa(index+1) + ".sst" + filename = "kd_" + strconv.Itoa(maxIndex+1) + ".sst" } file, err := os.Create(filepath.Join(filePath, filename)) + defer file.Close() if err != nil { return false, err @@ -58,8 +60,14 @@ func WriteToFile(list [][]byte) (bool, error) { //contents := list offset := fileOffset(list) index := createIndexBlock(list, offset) + footer, err := createFooter(list) + + if err != nil { + return false, err + } list = append(list, index) + list = append(list, footer) content := slices.Concat(list...) @@ -71,8 +79,6 @@ func WriteToFile(list [][]byte) (bool, error) { file.Sync() - //buckets(files) - return true, nil } @@ -130,8 +136,8 @@ func createIndexBlock(contentList [][]byte, offset []uint64) []byte { } } - var size uint64 = uint64(len(index)) - index = append(binary.BigEndian.AppendUint64([]byte{}, size), index...) + //var size uint64 = uint64(len(index)) + //index = append(binary.BigEndian.AppendUint64([]byte{}, size), index...) return index } @@ -151,6 +157,23 @@ func fileOffset(contentList [][]byte) []uint64 { return offset } +func createFooter(list [][]byte) ([]byte, error) { + if len(list) <= 0 { + return nil, errors.New("Not able to create the footer because the record list is too small") + } + footer := make([]byte, 0, 24) + footer = binary.BigEndian.AppendUint64(footer, 0) + var size uint64 + + for _, rec := range list { + size += uint64(len(rec)) + } + footer = binary.BigEndian.AppendUint64(footer, size) + footer = binary.BigEndian.AppendUint64(footer, uint64(0xDEADBEEFDEADBEEF)) + + return footer, nil +} + // We are going to be doing size based compaction for compacting these files // The amount of files that need to be a similar size // TODO: Need to make the bucket map into a persistent map that is used throughout the entire state of the @@ -222,18 +245,20 @@ func Compact(filePath string) error { bucketSize := len(val) if bucketSize >= minThreshold && bucketSize <= maxThreshold { - //compactMap := make(map[string][]byte, 0); skiplist := MergeList() for _, fileInfo := range val { fileData, err := os.ReadFile(filepath.Join(filePath, fileInfo.Name())) + //We do this so that we only take into account the file block, and not the index or footer + footer := fileData[len(fileData)-24:] + fileBlockEnds := binary.BigEndian.Uint64(footer[8:16]) if err != nil { return err } //We are going through each file and from there we will save each key/value pair record into a skiplist - for i := 0; i < len(fileData); { + for i := 0; i < int(fileBlockEnds); { //header := fileData[i : i+21] checksum := binary.BigEndian.Uint32(fileData[i+8 : i+12]) @@ -269,6 +294,7 @@ func Compact(filePath string) error { // Delete all of the old files once the new file is committed for _, fileInfo := range val { + fmt.Println(filepath.Join(filePath, fileInfo.Name())) err := os.Remove(filepath.Join(filePath, fileInfo.Name())) if err != nil { diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go index edf7789..aa6c354 100644 --- a/internals/sstable/writer_test.go +++ b/internals/sstable/writer_test.go @@ -1,6 +1,7 @@ package sstable_test import ( + "encoding/binary" "fmt" "os" "path/filepath" @@ -18,23 +19,31 @@ func TestCheck(t *testing.T) { } func startUp(data ...string) (int, error) { + sstable.MergeList = func() sstable.ListMerger { + return memtable.CreateSkiplist() + } + size := 0 for i := range 10 { - targetDir := "../test" - name := fmt.Sprintf("%s_%d.sst", "testfile", i) - fullPath := filepath.Join(targetDir, name) - file, err := os.Create(fullPath) + // targetDir := "../test" + // name := fmt.Sprintf("%s_%d.sst", "testfile", i) + // fullPath := filepath.Join(targetDir, name) + // file, err := os.Create(fullPath) - if err != nil { - return 0, err - } + // if err != nil { + // return 0, err + // } //This is 31 bytes long for the even method and the increasing method will be +1 w, _ := record.CreateRecord(data[i], data[i], "PUT") size += len(w) + sstable.MergeList().Insert(data[i], w) + } + + _, err := sstable.WriteToFile(sstable.MergeList().EntireList()) - file.Write(w) - file.Close() + if err != nil { + return 0, nil } return size, nil @@ -45,6 +54,7 @@ func tearDown(filePath string) { for _, file := range files { os.Remove(filepath.Join(filePath, file.Name())) + } } @@ -103,15 +113,32 @@ func TestBucketsForFiles(t *testing.T) { tearDown("../test") } -func TestCompactFilesForFiles(t *testing.T) { +func TestCompactFiles(t *testing.T) { data := []string{"data", "data", "data", "data", "data", "data", "data", "data", "data", "data"} - startUp(data...) + //startUp(data...) sstable.MergeList = func() sstable.ListMerger { return memtable.CreateSkiplist() } + //size := 0 + for range 10 { + file := make([][]byte, 0, 10) + for i := range 10 { + w, _ := record.CreateRecord(data[i], data[i], "PUT") + file = append(file, w) + } + _, err := sstable.WriteToFile(file) + + t.Logf("Size: %d", len(file)) + + if err != nil || len(file) == 0 { + t.Fatalf("Not to able properly make the file: size=%d", len(file)) + } + + } + err := sstable.Compact("../test") if err != nil { @@ -148,4 +175,74 @@ func TestWriteToFile(t *testing.T) { if !ok { t.Errorf("Not able to create the file") } + + file, _ := os.Open("../test/kd_1.sst") + info, _ := file.Stat() + + if info.Size() == 0 { + t.Error("Did not populate file") + } + + tearDown("../test") +} + +func TestFooter(t *testing.T) { + fileContents := make([][]byte, 0, 32) + + for i := range 32 { + c, _ := record.CreateRecord(fmt.Sprintf("d%v", i), fmt.Sprintf("d%v", i), "PUT") + fileContents = append(fileContents, c) + } + + _, err := sstable.WriteToFile(fileContents) + + if err != nil { + t.Fatal(err.Error()) + } + + file, err := os.Open("../test/kd_1.sst") + //defer file.Close() + + if err != nil { + t.Fatal(err.Error()) + } + + //fmt.Println() + + footer := make([]byte, 24) + fileInfo, _ := file.Stat() + size := fileInfo.Size() + fmt.Println(int64(size - 24)) + _, err = file.ReadAt(footer, int64(size-24)) + + if err != nil { + t.Fatal(err.Error()) + } + + if binary.BigEndian.Uint64(footer[:8]) != uint64(0) { + t.Errorf("Footer saved incorrect offset for the start of the file:%d", binary.BigEndian.Uint64(footer[:8])) + } + + // //The first offset for the index block should always return 0 + // //indexBlock := binary.BigEndian.Uint64(footer[8:16]) + firstKey := make([]byte, 4) + file.ReadAt(firstKey, int64(binary.BigEndian.Uint64(footer[8:16]))) + keySize := binary.BigEndian.Uint32(firstKey) + key := make([]byte, keySize) + file.ReadAt(key, int64(binary.BigEndian.Uint64(footer[8:16])+4)) + offset := make([]byte, 8) + file.ReadAt(offset, int64(binary.BigEndian.Uint64(footer[8:16])+4+uint64(keySize))) + keyOffset := binary.BigEndian.Uint64(offset) + + //keySize := binary.BigEndian.Uint64(); + if keyOffset != uint64(0) { + t.Errorf("Footer saved incorrect offset for the start of the index block:%d", binary.BigEndian.Uint64(footer[8:16])) + } + fmt.Println(len(footer)) + if binary.BigEndian.Uint64(footer[16:24]) != uint64(0xDEADBEEFDEADBEEF) { + t.Errorf("Footer saved incorrect magic number: %v", binary.BigEndian.Uint64(footer[16:24])) + } + + file.Close() + tearDown("../test") } From bcd88b7c17a82852d443884fd20564a944d0784c Mon Sep 17 00:00:00 2001 From: Ivers Date: Tue, 9 Jun 2026 14:38:45 -0400 Subject: [PATCH 22/34] Forgot to mention that we need to create a footer function to be able to read in the file. That's why we weren't done before. Required serious refactoring. --- internals/sstable/reader.go | 6 ++++++ internals/sstable/reader_test.go | 1 + 2 files changed, 7 insertions(+) create mode 100644 internals/sstable/reader.go create mode 100644 internals/sstable/reader_test.go diff --git a/internals/sstable/reader.go b/internals/sstable/reader.go new file mode 100644 index 0000000..a6f0d46 --- /dev/null +++ b/internals/sstable/reader.go @@ -0,0 +1,6 @@ +package sstable + +func ReadFromFile(filePath, key string) ([]byte, error) { + + return nil, nil +} diff --git a/internals/sstable/reader_test.go b/internals/sstable/reader_test.go new file mode 100644 index 0000000..2f6a0eb --- /dev/null +++ b/internals/sstable/reader_test.go @@ -0,0 +1 @@ +package sstable_test From cc7935877c2e476819aadd47b8771c2ecb4beb7f Mon Sep 17 00:00:00 2001 From: Ivers Date: Wed, 10 Jun 2026 09:08:07 -0400 Subject: [PATCH 23/34] Fixed some testing bugs --- internals/sstable/writer_test.go | 41 ++++++++++---------------------- 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go index aa6c354..61fcbec 100644 --- a/internals/sstable/writer_test.go +++ b/internals/sstable/writer_test.go @@ -12,38 +12,22 @@ import ( "github.com/Vince-maple-byte/KeyData/internals/sstable" ) -func TestCheck(t *testing.T) { - if 2+2 == 5 { - t.Error() - } -} - func startUp(data ...string) (int, error) { - sstable.MergeList = func() sstable.ListMerger { - return memtable.CreateSkiplist() - } size := 0 - for i := range 10 { - // targetDir := "../test" - // name := fmt.Sprintf("%s_%d.sst", "testfile", i) - // fullPath := filepath.Join(targetDir, name) - // file, err := os.Create(fullPath) - - // if err != nil { - // return 0, err - // } - - //This is 31 bytes long for the even method and the increasing method will be +1 - w, _ := record.CreateRecord(data[i], data[i], "PUT") - size += len(w) - sstable.MergeList().Insert(data[i], w) - } + for range 10 { + file := make([][]byte, 0, 10) + for i := range 10 { + w, _ := record.CreateRecord(data[i], data[i], "PUT") + file = append(file, w) + } + _, err := sstable.WriteToFile(file) - _, err := sstable.WriteToFile(sstable.MergeList().EntireList()) + if err != nil { + return 0, err + } + size++ - if err != nil { - return 0, nil } return size, nil @@ -87,7 +71,7 @@ func TestBucketsForFiles(t *testing.T) { for _, test := range tests { totalSize, err := startUp(test.data...) filePath := "../test" - if err != nil { + if err != nil || totalSize == 0 { t.Fatalf("Could not start up the test") } @@ -183,6 +167,7 @@ func TestWriteToFile(t *testing.T) { t.Error("Did not populate file") } + file.Close() tearDown("../test") } From 7aac18f70c4d991a8cf70da91c689affb2db27e7 Mon Sep 17 00:00:00 2001 From: Ivers Date: Thu, 11 Jun 2026 14:00:58 -0400 Subject: [PATCH 24/34] Able to fully right the entire read file function, and fix the functionality of the checksum to better determine if a file is corrupted by adding the time, key size, payload size, key, and payload into the data that create the checksum. --- internals/record/record.go | 25 +++++---- internals/record/record_test.go | 35 ++++++++++-- internals/sstable/reader.go | 91 +++++++++++++++++++++++++++++++- internals/sstable/writer.go | 6 +-- internals/sstable/writer_test.go | 22 ++++---- 5 files changed, 146 insertions(+), 33 deletions(-) diff --git a/internals/record/record.go b/internals/record/record.go index 07d0b73..a97cb37 100644 --- a/internals/record/record.go +++ b/internals/record/record.go @@ -24,18 +24,21 @@ type Content struct { Payload string } -func checkSum(payload string) uint32 { - return crc32.ChecksumIEEE([]byte(payload)) -} +func checkSum(key, payload []byte, timestamp uint64) uint32 { + buf := make([]byte, 16+len(key)+len(payload)) -func ChecksumChecker(payload []byte, checksum uint32) bool { - valid := crc32.ChecksumIEEE(payload) + binary.BigEndian.PutUint64(buf[0:8], timestamp) + binary.BigEndian.PutUint32(buf[8:12], uint32(len(key))) + binary.BigEndian.PutUint32(buf[12:16], uint32(len(payload))) - if valid == checksum { - return true - } else { - return false - } + copy(buf[16:], key) + copy(buf[16+len(key):], payload) + + return crc32.ChecksumIEEE(buf) +} + +func ChecksumChecker(key, payload []byte, timestamp uint64, checksum uint32) bool { + return checkSum(key, payload, timestamp) == checksum } func createTimeStamp() int64 { @@ -64,7 +67,7 @@ func CreateRecord(key, payload, operation string) ([]byte, error) { //fmt.Println(len(b), time); //Remember to use BigEndian for reverting it back to a uint32 - b = binary.BigEndian.AppendUint32(b, checkSum(payload)) + b = binary.BigEndian.AppendUint32(b, checkSum([]byte(key), []byte(payload), uint64(time))) //We are adding the tombstone value into the record here var Tombstone uint8 diff --git a/internals/record/record_test.go b/internals/record/record_test.go index a637281..a851d94 100644 --- a/internals/record/record_test.go +++ b/internals/record/record_test.go @@ -77,21 +77,48 @@ func TestRecordCheckSum(t *testing.T) { key: "a", payload: "abc", operation: "PUT", - expected: crc32.ChecksumIEEE([]byte("abc")), + expected: func() uint32 { + data, _ := record.CreateRecord("a", "abc", "PUT") + f := make([]byte, 20) + copy(f[0:8], data[:8]) + binary.BigEndian.PutUint32(f[8:12], uint32(len("a"))) + binary.BigEndian.PutUint32(f[12:16], uint32(len("abc"))) + copy(f[16:17], []byte("a")) + copy(f[17:], []byte("abc")) + return crc32.ChecksumIEEE(f) + }(), }, { testName: "Test 2", key: "a", payload: "", operation: "DELETE", - expected: crc32.ChecksumIEEE([]byte("")), + expected: func() uint32 { + data, _ := record.CreateRecord("a", "", "PUT") + f := make([]byte, 17) + copy(f[0:8], data[:8]) + binary.BigEndian.PutUint32(f[8:12], uint32(len("a"))) + binary.BigEndian.PutUint32(f[12:16], uint32(len(""))) + copy(f[16:17], []byte("a")) + copy(f[17:], []byte("")) + return crc32.ChecksumIEEE(f) + }(), }, { testName: "Test 3", key: "a", payload: "a", operation: "PUT", - expected: crc32.ChecksumIEEE([]byte("a")), + expected: func() uint32 { + data, _ := record.CreateRecord("a", "a", "PUT") + f := make([]byte, 18) + copy(f[0:8], data[:8]) + binary.BigEndian.PutUint32(f[8:12], uint32(len("a"))) + binary.BigEndian.PutUint32(f[12:16], uint32(len("a"))) + copy(f[16:17], []byte("a")) + copy(f[17:], []byte("a")) + return crc32.ChecksumIEEE(f) + }(), }, } @@ -106,7 +133,7 @@ func TestRecordCheckSum(t *testing.T) { checksum := binary.BigEndian.Uint32(record[8:12]) if checksum != test.expected { - t.Errorf("incorrect checksum for %v", test.testName) + t.Errorf("incorrect checksum for %v:\nExpected checksum %v\nActual checksum %v", test.testName, test.expected, checksum) } } } diff --git a/internals/sstable/reader.go b/internals/sstable/reader.go index a6f0d46..0a42a31 100644 --- a/internals/sstable/reader.go +++ b/internals/sstable/reader.go @@ -1,6 +1,95 @@ package sstable +import ( + "encoding/binary" + "errors" + "os" + "path/filepath" + + "github.com/Vince-maple-byte/KeyData/internals/record" +) + +// How to read from the file +// We first checking the magic number inside of the footer to see if the file is valid +// We get the byte offset of the index block, and proceed to do: +// We check if the key inside of the byte offset is greater than or equal to our key +// If it is greater: we go to the previous byte offset inside of the index and search between the previous byte offset +// and the byte offset were it is greater than the key. +// if it is equal than we can just return that key/value pair inside of the file at that byte offset +// If the key is not inside of the range in which we stated before, than we can just return nil and an error message +// stating that the key can't be found func ReadFromFile(filePath, key string) ([]byte, error) { + file, err := os.Open(filepath.Join("../test", filePath)) + + defer file.Close() + if err != nil { + return nil, err + } + fileInfo, _ := file.Stat() + footer := make([]byte, 24) + + file.ReadAt(footer, fileInfo.Size()-24) + + //Checking the magic number first + if binary.BigEndian.Uint64(footer[16:]) != uint64(0xDEADBEEFDEADBEEF) { + err = errors.New("Incorrect magic number inside of the file. The file is invalid") + return nil, err + } + + //Getting the index block offset + indexBlockByte := make([]byte, 8) + file.ReadAt(indexBlockByte, int64(binary.BigEndian.Uint64(footer[8:16]))) + indexBlockLoc := binary.BigEndian.Uint64(indexBlockByte) + lowKeyOffset := indexBlockLoc + highKeyOffset := indexBlockLoc + + for i := indexBlockLoc; i < uint64(fileInfo.Size()-24); { + lowKeyOffset = i + keySize := make([]byte, 4) + file.ReadAt(keySize, int64(i)) + offsetKey := make([]byte, int64(binary.BigEndian.Uint32(keySize))) + file.ReadAt(offsetKey, int64(i)+4) + offset := make([]byte, 8) + file.ReadAt(offset, int64(i+4+uint64(binary.BigEndian.Uint32(keySize)))) + //keyOffset := binary.BigEndian.Uint64(offset) + + i = i + 4 + uint64(binary.BigEndian.Uint32(keySize)) + 8 + if key <= string(offsetKey) { + break + } else { + lowKeyOffset = highKeyOffset + highKeyOffset = i + } + } + + if lowKeyOffset == highKeyOffset { + return nil, errors.New("Key is not inside of the file") + } + + //TODO: Make the range to + indexBlockRange := make([]byte, highKeyOffset-lowKeyOffset) + file.ReadAt(indexBlockRange, int64(lowKeyOffset)) + + for i := lowKeyOffset; i < highKeyOffset; { + keySize := binary.BigEndian.Uint32(indexBlockRange[i+13 : i+17]) + payloadSize := binary.BigEndian.Uint32(indexBlockRange[i+17 : i+21]) + + currRecord := record.GetContents(indexBlockRange[i : i+21+uint64(keySize)+uint64(payloadSize)]) + + if !record.ChecksumChecker( + []byte(currRecord.Key), + []byte(currRecord.Payload), + uint64(currRecord.Timestamp.UnixNano()), + currRecord.Checksum) { + return nil, errors.New("This section of the file is corrupted, can not retrieve the key/value pairs from here") + } + + if currRecord.Key == key { + return indexBlockRange[i : i+21+uint64(keySize)+uint64(payloadSize)], nil + } + + i += 21 + uint64(keySize) + uint64(payloadSize) + } - return nil, nil + return nil, errors.New("Key could not be found") } diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index 1635bf1..893dbd3 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -3,7 +3,6 @@ package sstable import ( "encoding/binary" "errors" - "fmt" "io/fs" "os" "path/filepath" @@ -125,7 +124,6 @@ func createIndexBlock(contentList [][]byte, offset []uint64) []byte { index = binary.BigEndian.AppendUint64(index, offset[0]) } else { for i := 0; i < len(contentList); i = i + (len(contentList) / INDEX_BLOCK) { - fmt.Println(i) contents := record.GetContents(contentList[i]) index = binary.BigEndian.AppendUint32(index, contents.Keysize) @@ -260,6 +258,7 @@ func Compact(filePath string) error { //We are going through each file and from there we will save each key/value pair record into a skiplist for i := 0; i < int(fileBlockEnds); { //header := fileData[i : i+21] + time := fileData[i : i+8] checksum := binary.BigEndian.Uint32(fileData[i+8 : i+12]) @@ -270,7 +269,7 @@ func Compact(filePath string) error { payload := fileData[i+keySize+21 : i+(keySize+21)+payloadSize] //If the checksum is invalid, we ignore the rest of the file - if !record.ChecksumChecker(payload, checksum) { + if !record.ChecksumChecker(key, payload, binary.BigEndian.Uint64(time), checksum) { break } @@ -294,7 +293,6 @@ func Compact(filePath string) error { // Delete all of the old files once the new file is committed for _, fileInfo := range val { - fmt.Println(filepath.Join(filePath, fileInfo.Name())) err := os.Remove(filepath.Join(filePath, fileInfo.Name())) if err != nil { diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go index 61fcbec..54ee7fe 100644 --- a/internals/sstable/writer_test.go +++ b/internals/sstable/writer_test.go @@ -192,12 +192,9 @@ func TestFooter(t *testing.T) { t.Fatal(err.Error()) } - //fmt.Println() - footer := make([]byte, 24) fileInfo, _ := file.Stat() size := fileInfo.Size() - fmt.Println(int64(size - 24)) _, err = file.ReadAt(footer, int64(size-24)) if err != nil { @@ -208,22 +205,21 @@ func TestFooter(t *testing.T) { t.Errorf("Footer saved incorrect offset for the start of the file:%d", binary.BigEndian.Uint64(footer[:8])) } - // //The first offset for the index block should always return 0 - // //indexBlock := binary.BigEndian.Uint64(footer[8:16]) - firstKey := make([]byte, 4) - file.ReadAt(firstKey, int64(binary.BigEndian.Uint64(footer[8:16]))) - keySize := binary.BigEndian.Uint32(firstKey) - key := make([]byte, keySize) - file.ReadAt(key, int64(binary.BigEndian.Uint64(footer[8:16])+4)) + indexBlockLoc := make([]byte, 8) + file.ReadAt(indexBlockLoc, int64(binary.BigEndian.Uint64(footer[8:16]))) + keySize := make([]byte, 4) + + file.ReadAt(keySize, int64(binary.BigEndian.Uint64(footer[8:16]))) + key := make([]byte, int64(binary.BigEndian.Uint32(keySize))) + file.ReadAt(key, int64(binary.BigEndian.Uint64(footer[8:16]))+4) offset := make([]byte, 8) - file.ReadAt(offset, int64(binary.BigEndian.Uint64(footer[8:16])+4+uint64(keySize))) + file.ReadAt(offset, int64(binary.BigEndian.Uint64(footer[8:16])+4+uint64(binary.BigEndian.Uint32(keySize)))) keyOffset := binary.BigEndian.Uint64(offset) //keySize := binary.BigEndian.Uint64(); - if keyOffset != uint64(0) { + if keyOffset != uint64(0) || string(key) != "d0" { t.Errorf("Footer saved incorrect offset for the start of the index block:%d", binary.BigEndian.Uint64(footer[8:16])) } - fmt.Println(len(footer)) if binary.BigEndian.Uint64(footer[16:24]) != uint64(0xDEADBEEFDEADBEEF) { t.Errorf("Footer saved incorrect magic number: %v", binary.BigEndian.Uint64(footer[16:24])) } From 5c98d38fe9a38320924f7dd76319636ba2c72136 Mon Sep 17 00:00:00 2001 From: Ivers Date: Tue, 16 Jun 2026 11:08:02 -0400 Subject: [PATCH 25/34] Finished with all of the testing. Next up creating grpc server/client communication for the database communication --- internals/sstable/corruption_test.go | 432 +++++++++++++++++++++++++++ internals/sstable/reader.go | 62 ++-- internals/sstable/reader_test.go | 147 +++++++++ internals/sstable/writer.go | 24 +- internals/sstable/writer_test.go | 18 +- 5 files changed, 649 insertions(+), 34 deletions(-) create mode 100644 internals/sstable/corruption_test.go diff --git a/internals/sstable/corruption_test.go b/internals/sstable/corruption_test.go new file mode 100644 index 0000000..0fc7e13 --- /dev/null +++ b/internals/sstable/corruption_test.go @@ -0,0 +1,432 @@ +package sstable_test + +import ( + "encoding/binary" + "os" + "path/filepath" + "testing" + + "github.com/Vince-maple-byte/KeyData/internals/memtable" + "github.com/Vince-maple-byte/KeyData/internals/sstable" +) + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// buildValidRecord constructs a minimal, valid SSTable record byte slice. +// Layout (matches Compact reader): +// +// [0:8] timestamp (uint64 big-endian) +// [8:12] checksum (uint32 big-endian) -- intentionally zeroed for helpers +// [12] flags byte (1 byte, unused but present) +// [13:17] keySize (uint32 big-endian) +// [17:21] payloadSize(uint32 big-endian) +// [21:21+keySize] key +// [21+keySize : ...] payload +func buildValidRecord(key, payload string, timestamp uint64) []byte { + ks := uint32(len(key)) + ps := uint32(len(payload)) + + rec := make([]byte, 0, 21+ks+ps) + rec = binary.BigEndian.AppendUint64(rec, timestamp) // [0:8] timestamp + rec = binary.BigEndian.AppendUint32(rec, 0) // [8:12] checksum placeholder + rec = append(rec, 0x00) // [12] flags + rec = binary.BigEndian.AppendUint32(rec, ks) // [13:17] keySize + rec = binary.BigEndian.AppendUint32(rec, ps) // [17:21] payloadSize + rec = append(rec, []byte(key)...) + rec = append(rec, []byte(payload)...) + return rec +} + +// buildValidFooter builds a 24-byte footer. +// +// [0:8] reserved / bloom-filter offset (zeroed) +// [8:16] end-of-data-block offset +// [16:24] magic number 0xDEADBEEFDEADBEEF +func buildValidFooter(dataBlockSize uint64) []byte { + f := make([]byte, 0, 24) + f = binary.BigEndian.AppendUint64(f, 0) + f = binary.BigEndian.AppendUint64(f, dataBlockSize) + f = binary.BigEndian.AppendUint64(f, 0xDEADBEEFDEADBEEF) + return f +} + +// writeTempFile writes data to a uniquely-named temp file inside dir and +// returns the full path. +func writeTempFile(t *testing.T, dir string, data []byte) string { + t.Helper() + f, err := os.CreateTemp(dir, "kd_*.sst") + if err != nil { + t.Fatalf("writeTempFile: CreateTemp: %v", err) + } + defer f.Close() + if _, err := f.Write(data); err != nil { + t.Fatalf("writeTempFile: Write: %v", err) + } + return f.Name() +} + +// buildSSTFile assembles: records || indexBlock || footer +// and returns the raw bytes. +func buildSSTFile(records [][]byte, indexBlock []byte, footer []byte) []byte { + var out []byte + for _, r := range records { + out = append(out, r...) + } + out = append(out, indexBlock...) + out = append(out, footer...) + return out +} + +// --------------------------------------------------------------------------- +// createFooter tests +// --------------------------------------------------------------------------- + +func TestCreateFooter_EmptyList(t *testing.T) { + _, err := sstable.ExportFooter([][]byte{}) + if err == nil { + t.Fatal("expected error for empty record list, got nil") + } +} + +func TestCreateFooter_MagicNumber(t *testing.T) { + rec := buildValidRecord("hello", "world", 1) + footer, err := sstable.ExportFooter([][]byte{rec}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(footer) != 24 { + t.Fatalf("footer length: want 24, got %d", len(footer)) + } + magic := binary.BigEndian.Uint64(footer[16:24]) + const want = uint64(0xDEADBEEFDEADBEEF) + if magic != want { + t.Errorf("magic: want 0x%X, got 0x%X", want, magic) + } +} + +func TestCreateFooter_DataSizeMatchesInput(t *testing.T) { + r1 := buildValidRecord("key1", "val1", 1) + r2 := buildValidRecord("key2", "val2", 2) + list := [][]byte{r1, r2} + + footer, err := sstable.ExportFooter(list) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + wantSize := uint64(len(r1) + len(r2)) + gotSize := binary.BigEndian.Uint64(footer[8:16]) + if gotSize != wantSize { + t.Errorf("dataBlockSize: want %d, got %d", wantSize, gotSize) + } +} + +// --------------------------------------------------------------------------- +// Truncated-file tests +// (These simulate what Compact would encounter when reading a corrupt file.) +// --------------------------------------------------------------------------- + +func TestReadFile_TruncatedFooter(t *testing.T) { + rec := buildValidRecord("key", "value", 42) + footer := buildValidFooter(uint64(len(rec))) + full := buildSSTFile([][]byte{rec}, []byte{}, footer) + + // Chop the last 4 bytes so the footer is incomplete. + truncated := full[:len(full)-4] + + dir := t.TempDir() + path := writeTempFile(t, dir, truncated) + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + + if len(data) < 24 { + // Correct: caller should detect the file is too short to hold a footer. + t.Log("correctly detected truncated footer: file shorter than 24 bytes") + return + } + + footer2 := data[len(data)-24:] + magic := binary.BigEndian.Uint64(footer2[16:24]) + if magic == 0xDEADBEEFDEADBEEF { + t.Error("truncated file should NOT produce a valid magic number") + } +} + +func TestReadFile_TruncatedMidRecord(t *testing.T) { + // Build a record but only write the first half of it. + rec := buildValidRecord("somekey", "somevalue", 99) + half := rec[:len(rec)/2] + + dir := t.TempDir() + path := writeTempFile(t, dir, half) + + data, _ := os.ReadFile(path) + + // Simulate the Compact inner loop: try to read keySize starting at offset 0. + const headerEnd = 17 + if len(data) < headerEnd { + t.Log("correctly detected truncated record: header incomplete") + return + } + + keySize := int(binary.BigEndian.Uint32(data[13:17])) + payloadSize := int(binary.BigEndian.Uint32(data[17:21])) + expectedEnd := 21 + keySize + payloadSize + + if len(data) < expectedEnd { + t.Logf("correctly detected truncated record: need %d bytes, have %d", expectedEnd, len(data)) + } else { + t.Error("expected truncation to be detected but full record appears readable") + } +} + +func TestReadFile_ZeroByteFile(t *testing.T) { + dir := t.TempDir() + path := writeTempFile(t, dir, []byte{}) + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + if len(data) != 0 { + t.Fatalf("expected 0 bytes, got %d", len(data)) + } + if len(data) < 24 { + t.Log("correctly detected zero-byte file: cannot contain a valid footer") + } +} + +func TestReadFile_FooterOnlyNoData(t *testing.T) { + // A file containing only a footer (dataBlockSize = 0) — edge case. + footer := buildValidFooter(0) + dir := t.TempDir() + path := writeTempFile(t, dir, footer) + + data, _ := os.ReadFile(path) + footer2 := data[len(data)-24:] + magic := binary.BigEndian.Uint64(footer2[16:24]) + + if magic != 0xDEADBEEFDEADBEEF { + t.Errorf("magic mismatch: got 0x%X", magic) + } + + dataEnd := binary.BigEndian.Uint64(footer2[8:16]) + if dataEnd != 0 { + t.Errorf("expected dataBlockSize=0, got %d", dataEnd) + } +} + +func TestReadFile_TruncatedAtHeaderBoundary(t *testing.T) { + // Write exactly 21 bytes (header only, no key/payload). + header := make([]byte, 21) + binary.BigEndian.PutUint64(header[0:8], 12345) // timestamp + binary.BigEndian.PutUint32(header[8:12], 0) // checksum + header[12] = 0x00 // flags + binary.BigEndian.PutUint32(header[13:17], 5) // keySize = 5 + binary.BigEndian.PutUint32(header[17:21], 10) // payloadSize = 10 + + dir := t.TempDir() + path := writeTempFile(t, dir, header) + data, _ := os.ReadFile(path) + + keySize := int(binary.BigEndian.Uint32(data[13:17])) + payloadSize := int(binary.BigEndian.Uint32(data[17:21])) + need := 21 + keySize + payloadSize + + if len(data) < need { + t.Logf("correctly detected record truncated at header boundary: need %d, have %d", need, len(data)) + } else { + t.Error("expected boundary truncation to be detected") + } +} + +// --------------------------------------------------------------------------- +// Swapped-bytes / bit-flip tests +// --------------------------------------------------------------------------- + +func TestReadFile_SwappedMagicBytes(t *testing.T) { + rec := buildValidRecord("key", "val", 1) + footer := buildValidFooter(uint64(len(rec))) + + // Swap the first two bytes of the magic number region (footer[16] <-> footer[17]). + footer[16], footer[17] = footer[17], footer[16] + + full := buildSSTFile([][]byte{rec}, []byte{}, footer) + dir := t.TempDir() + path := writeTempFile(t, dir, full) + + data, _ := os.ReadFile(path) + magic := binary.BigEndian.Uint64(data[len(data)-8:]) + + if magic == 0xDEADBEEFDEADBEEF { + t.Error("swapped magic bytes should invalidate the magic number") + } else { + t.Logf("correctly detected swapped magic bytes: got 0x%X", magic) + } +} + +func TestReadFile_SwappedKeySizeBytes(t *testing.T) { + rec := buildValidRecord("hello", "world", 7) + + // Swap bytes [13] and [14] — the high bytes of keySize (uint32). + rec[13], rec[16] = rec[16], rec[13] + + dir := t.TempDir() + path := writeTempFile(t, dir, []byte{}) + os.WriteFile(path, rec, 0644) + + data, _ := os.ReadFile(path) + keySize := int(binary.BigEndian.Uint32(data[13:17])) + + // Original keySize was 5 ("hello"); swapped bytes should differ. + if keySize == 5 { + t.Error("swapped keySize bytes should produce a different (wrong) keySize") + } else { + t.Logf("correctly detected swapped keySize bytes: parsed keySize=%d (true=5)", keySize) + } +} + +func TestReadFile_SwappedTimestampBytes(t *testing.T) { + const originalTS uint64 = 0x0000000100000002 + rec := buildValidRecord("k", "v", originalTS) + + // Swap bytes [0] and [7] — MSB and LSB of the timestamp. + rec[0], rec[7] = rec[7], rec[0] + + ts := binary.BigEndian.Uint64(rec[0:8]) + if ts == originalTS { + t.Error("swapped timestamp bytes should produce a different value") + } else { + t.Logf("correctly detected swapped timestamp: got 0x%X (original 0x%X)", ts, originalTS) + } +} + +func TestReadFile_SwappedDataBlockSizeInFooter(t *testing.T) { + rec := buildValidRecord("testkey", "testval", 55) + correctSize := uint64(len(rec)) + footer := buildValidFooter(correctSize) + + // Swap bytes [8] and [9] inside the footer (high bytes of dataBlockSize). + footer[8], footer[15] = footer[15], footer[8] + + full := buildSSTFile([][]byte{rec}, []byte{}, footer) + dir := t.TempDir() + path := writeTempFile(t, dir, full) + + data, _ := os.ReadFile(path) + footerSlice := data[len(data)-24:] + parsedSize := binary.BigEndian.Uint64(footerSlice[8:16]) + + if parsedSize == correctSize { + t.Logf("incorrectly detected swapped dataBlockSize: got %d (correct %d)", parsedSize, correctSize) + t.Error("swapped dataBlockSize bytes should produce a wrong size") + } else { + t.Logf("correctly detected swapped dataBlockSize: got %d (correct %d)", parsedSize, correctSize) + } +} + +func TestReadFile_BitFlipInPayload(t *testing.T) { + rec := buildValidRecord("mykey", "myvalue", 100) + payloadStart := 21 + len("mykey") + + // Flip all bits in the first payload byte. + rec[payloadStart] ^= 0xFF + + payload := rec[payloadStart : payloadStart+len("myvalue")] + if string(payload) == "myvalue" { + t.Error("bit-flipped payload should not equal original") + } else { + t.Logf("correctly detected payload corruption: %q", payload) + } +} + +// --------------------------------------------------------------------------- +// fileOffset tests +// --------------------------------------------------------------------------- + +func TestFileOffset_CorrectOffsets(t *testing.T) { + r1 := []byte("abcde") // 5 bytes → offset 0 + r2 := []byte("fghij") // 5 bytes → offset 5 + r3 := []byte("klmnopqr") // 8 bytes → offset 10 + + offsets := sstable.ExportFileOffset([][]byte{r1, r2, r3}) + + expected := []uint64{0, 5, 10} + for i, want := range expected { + if offsets[i] != want { + t.Errorf("offset[%d]: want %d, got %d", i, want, offsets[i]) + } + } +} + +func TestFileOffset_EmptyList(t *testing.T) { + offsets := sstable.ExportFileOffset([][]byte{}) + if len(offsets) != 0 { + t.Errorf("expected empty offsets, got %v", offsets) + } +} + +// --------------------------------------------------------------------------- +// Compact-level integration: corrupt file in temp dir +// --------------------------------------------------------------------------- + +func TestCompact_IgnoresFileWithTruncatedRecord(t *testing.T) { + dir := t.TempDir() + + // Write 4 files so Compact's minThreshold (4) is met. + // Three are valid; one has its record body truncated. + writeValidSST := func(name, key, val string) { + rec := buildValidRecord(key, val, 1) + footer := buildValidFooter(uint64(len(rec))) + data := buildSSTFile([][]byte{rec}, []byte{}, footer) + os.WriteFile(filepath.Join(dir, name), data, 0644) + } + + writeValidSST("kd_1.sst", "apple", "pie") + writeValidSST("kd_2.sst", "banana", "split") + writeValidSST("kd_3.sst", "cherry", "jam") + + // Fourth file: valid header, truncated body. + rec := buildValidRecord("date", "fruit", 1) + truncated := rec[:len(rec)-3] + footer := buildValidFooter(uint64(len(truncated))) + corrupt := buildSSTFile([][]byte{truncated}, []byte{}, footer) + os.WriteFile(filepath.Join(dir, "kd_4.sst"), corrupt, 0644) + + // Compact should not panic; corruption causes the inner loop to break early. + err := sstable.Compact(dir) + // We accept either nil or an error — the key requirement is no panic / index OOB. + t.Logf("Compact returned: %v", err) +} + +func TestCompact_FileWithSwappedMagic(t *testing.T) { + dir := t.TempDir() + + sstable.MergeList = func() sstable.ListMerger { + return memtable.CreateSkiplist() + } + + writeSST := func(name, key, val string, corruptMagic bool) { + rec := buildValidRecord(key, val, 1) + footer := buildValidFooter(uint64(len(rec))) + if corruptMagic { + // Swap the last two bytes of the magic number. + footer[22], footer[23] = footer[23], footer[22] + } + data := buildSSTFile([][]byte{rec}, []byte{}, footer) + os.WriteFile(filepath.Join(dir, name), data, 0644) + } + + writeSST("kd_1.sst", "alpha", "1", false) + writeSST("kd_2.sst", "beta", "2", false) + writeSST("kd_3.sst", "gamma", "3", false) + writeSST("kd_4.sst", "delta", "4", true) // corrupted magic + + err := sstable.Compact(dir) + t.Logf("Compact with swapped magic returned: %v", err) +} diff --git a/internals/sstable/reader.go b/internals/sstable/reader.go index 0a42a31..4991373 100644 --- a/internals/sstable/reader.go +++ b/internals/sstable/reader.go @@ -37,44 +37,58 @@ func ReadFromFile(filePath, key string) ([]byte, error) { } //Getting the index block offset - indexBlockByte := make([]byte, 8) - file.ReadAt(indexBlockByte, int64(binary.BigEndian.Uint64(footer[8:16]))) - indexBlockLoc := binary.BigEndian.Uint64(indexBlockByte) - lowKeyOffset := indexBlockLoc - highKeyOffset := indexBlockLoc - - for i := indexBlockLoc; i < uint64(fileInfo.Size()-24); { - lowKeyOffset = i + indexBlockLoc := binary.BigEndian.Uint64(footer[8:16]) + lowKeyOffset := uint64(0) + highKeyOffset := uint64(0) + + for i := indexBlockLoc; i <= uint64(fileInfo.Size()-24); { + //highKeyOffset = i keySize := make([]byte, 4) file.ReadAt(keySize, int64(i)) offsetKey := make([]byte, int64(binary.BigEndian.Uint32(keySize))) file.ReadAt(offsetKey, int64(i)+4) - offset := make([]byte, 8) - file.ReadAt(offset, int64(i+4+uint64(binary.BigEndian.Uint32(keySize)))) - //keyOffset := binary.BigEndian.Uint64(offset) + //This gives us the location of the offset where it is saved in the data portion of the file + offsetLoc := make([]byte, 8) + file.ReadAt(offsetLoc, int64(i+4+uint64(binary.BigEndian.Uint32(keySize)))) + keyOffset := binary.BigEndian.Uint64(offsetLoc) + //Go to the next location in the index block i = i + 4 + uint64(binary.BigEndian.Uint32(keySize)) + 8 - if key <= string(offsetKey) { + + if key == string(offsetKey) { + curr := make([]byte, 21) + file.ReadAt(curr, int64(keyOffset)) + payloadSize := binary.BigEndian.Uint32(curr[17:21]) + + entireRecord := make([]byte, 21+binary.BigEndian.Uint32(keySize)+payloadSize) + file.ReadAt(entireRecord, int64(keyOffset)) + return entireRecord, nil + } + + lowKeyOffset = highKeyOffset + highKeyOffset = keyOffset + + if key < string(offsetKey) { break - } else { - lowKeyOffset = highKeyOffset - highKeyOffset = i } } if lowKeyOffset == highKeyOffset { - return nil, errors.New("Key is not inside of the file") + return nil, errors.New("Key is not inside of the file: Key is larger than any key in the file") } //TODO: Make the range to - indexBlockRange := make([]byte, highKeyOffset-lowKeyOffset) - file.ReadAt(indexBlockRange, int64(lowKeyOffset)) for i := lowKeyOffset; i < highKeyOffset; { - keySize := binary.BigEndian.Uint32(indexBlockRange[i+13 : i+17]) - payloadSize := binary.BigEndian.Uint32(indexBlockRange[i+17 : i+21]) + curr := make([]byte, 21) + file.ReadAt(curr, int64(i)) + keySize := binary.BigEndian.Uint32(curr[13:17]) + payloadSize := binary.BigEndian.Uint32(curr[17:21]) + + entireRecord := make([]byte, 21+keySize+payloadSize) + file.ReadAt(entireRecord, int64(i)) - currRecord := record.GetContents(indexBlockRange[i : i+21+uint64(keySize)+uint64(payloadSize)]) + currRecord := record.GetContents(entireRecord) if !record.ChecksumChecker( []byte(currRecord.Key), @@ -85,7 +99,11 @@ func ReadFromFile(filePath, key string) ([]byte, error) { } if currRecord.Key == key { - return indexBlockRange[i : i+21+uint64(keySize)+uint64(payloadSize)], nil + return entireRecord, nil + } + + if currRecord.Key > key { + break } i += 21 + uint64(keySize) + uint64(payloadSize) diff --git a/internals/sstable/reader_test.go b/internals/sstable/reader_test.go index 2f6a0eb..1046ed8 100644 --- a/internals/sstable/reader_test.go +++ b/internals/sstable/reader_test.go @@ -1 +1,148 @@ package sstable_test + +import ( + "os" + "strconv" + "testing" + + "github.com/Vince-maple-byte/KeyData/internals/memtable" + "github.com/Vince-maple-byte/KeyData/internals/record" + "github.com/Vince-maple-byte/KeyData/internals/sstable" +) + +func TestReadFile(t *testing.T) { + tests := []struct { + testName string + data [][]byte + key string + expected string + }{ + { + testName: "key_size_1", + data: func() [][]byte { + d := memtable.CreateSkiplist() + for i := range 3200 { + r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") + d.Insert(strconv.Itoa(i), r) + } + + return d.EntireList() + }(), + key: "8", + expected: "9", + }, + { + testName: "key_size_2", + data: func() [][]byte { + d := memtable.CreateSkiplist() + for i := range 3200 { + + r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") + d.Insert(strconv.Itoa(i), r) + } + return d.EntireList() + }(), + key: "31", + expected: "32", + }, + { + testName: "key_size_3", + data: func() [][]byte { + d := memtable.CreateSkiplist() + for i := range 3200 { + + r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") + d.Insert(strconv.Itoa(i), r) + } + return d.EntireList() + }(), + key: "432", + expected: "433", + }, + { + testName: "key_size_4", + data: func() [][]byte { + d := memtable.CreateSkiplist() + for i := range 3200 { + + r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") + d.Insert(strconv.Itoa(i), r) + } + return d.EntireList() + }(), + key: "3197", + expected: "3198", + }, + { + testName: "key_is_a_direct_match_to_the_index", + data: func() [][]byte { + d := memtable.CreateSkiplist() + for i := range 3200 { + + r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") + d.Insert(strconv.Itoa(i), r) + } + return d.EntireList() + }(), + key: "1140", + expected: "1141", + }, + { + testName: "key_if_file_truncated", + data: func() [][]byte { + d := memtable.CreateSkiplist() + for i := range 240 { + + r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") + d.Insert(strconv.Itoa(i), r) + } + return d.EntireList() + }(), + key: "104", + expected: "105", + }, + { + testName: "key_if_larger_than_normal", + data: func() [][]byte { + d := memtable.CreateSkiplist() + for i := range 7000 { + + r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") + d.Insert(strconv.Itoa(i), r) + } + return d.EntireList() + }(), + key: "4006", + expected: "4007", + }, + } + + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + _, err := sstable.WriteToFile(test.data) + + if err != nil { + t.Fatalf("Error in writing the file: %s", err.Error()) + } + fileInfo, err := os.Lstat("../test/kd_1.sst") + size := fileInfo.Size() + if err != nil { + tearDown("../test") + t.Fatalf("Error in accessing the file stats: %s", err.Error()) + } + rec, err := sstable.ReadFromFile("kd_1.sst", test.key) + + if rec == nil || err != nil { + tearDown("../test") + t.Fatalf("Error recieved: %v\nSize of the file: %d", err.Error(), size) + } + if record.GetContents(rec).Payload != test.expected { + t.Errorf("Test failed for %s\nExpected:%v\nActual:%v", + test.testName, test.expected, record.GetContents(rec).Payload) + } + + tearDown("../test") + }) + + } +} diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index 893dbd3..0323d76 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -3,6 +3,7 @@ package sstable import ( "encoding/binary" "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -114,14 +115,15 @@ func WriteToFile(list [][]byte) (bool, error) { func createIndexBlock(contentList [][]byte, offset []uint64) []byte { index := make([]byte, 0, INDEX_BLOCK*160) + //I made this specific change for the index block since in the if len(contentList) < INDEX_BLOCK*160 { - contents := record.GetContents(contentList[0]) + contents := record.GetContents(contentList[len(contentList)/2]) index = binary.BigEndian.AppendUint32(index, contents.Keysize) index = append(index, contents.Key...) - index = binary.BigEndian.AppendUint64(index, offset[0]) + index = binary.BigEndian.AppendUint64(index, offset[len(offset)/2]) } else { for i := 0; i < len(contentList); i = i + (len(contentList) / INDEX_BLOCK) { contents := record.GetContents(contentList[i]) @@ -247,13 +249,17 @@ func Compact(filePath string) error { for _, fileInfo := range val { fileData, err := os.ReadFile(filepath.Join(filePath, fileInfo.Name())) - //We do this so that we only take into account the file block, and not the index or footer - footer := fileData[len(fileData)-24:] - fileBlockEnds := binary.BigEndian.Uint64(footer[8:16]) if err != nil { return err } + //We do this so that we only take into account the file block, and not the index or footer + footer := fileData[len(fileData)-24:] + if binary.BigEndian.Uint64(footer[16:]) != 0xDEADBEEFDEADBEEF { + //os.Remove(fileInfo.Name()) + return fmt.Errorf("invalid footer magic") + } + fileBlockEnds := binary.BigEndian.Uint64(footer[8:16]) //We are going through each file and from there we will save each key/value pair record into a skiplist for i := 0; i < int(fileBlockEnds); { @@ -312,3 +318,11 @@ func ExportBuckets(filePath string) map[file_buckets][]fs.FileInfo { return result } + +func ExportFooter(list [][]byte) ([]byte, error) { + return createFooter(list) +} + +func ExportFileOffset(contentList [][]byte) []uint64 { + return fileOffset(contentList) +} diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go index 54ee7fe..2923ec0 100644 --- a/internals/sstable/writer_test.go +++ b/internals/sstable/writer_test.go @@ -145,7 +145,7 @@ func TestCompactFiles(t *testing.T) { func TestWriteToFile(t *testing.T) { fileContents := make([][]byte, 0, 10) - for range 10 { + for range 1 { r, _ := record.CreateRecord("a", "1", "PUT") fileContents = append(fileContents, r) } @@ -172,9 +172,9 @@ func TestWriteToFile(t *testing.T) { } func TestFooter(t *testing.T) { - fileContents := make([][]byte, 0, 32) + fileContents := make([][]byte, 0) - for i := range 32 { + for i := range 3200 { c, _ := record.CreateRecord(fmt.Sprintf("d%v", i), fmt.Sprintf("d%v", i), "PUT") fileContents = append(fileContents, c) } @@ -210,15 +210,19 @@ func TestFooter(t *testing.T) { keySize := make([]byte, 4) file.ReadAt(keySize, int64(binary.BigEndian.Uint64(footer[8:16]))) - key := make([]byte, int64(binary.BigEndian.Uint32(keySize))) + key := make([]byte, binary.BigEndian.Uint32(keySize)) file.ReadAt(key, int64(binary.BigEndian.Uint64(footer[8:16]))+4) offset := make([]byte, 8) file.ReadAt(offset, int64(binary.BigEndian.Uint64(footer[8:16])+4+uint64(binary.BigEndian.Uint32(keySize)))) - keyOffset := binary.BigEndian.Uint64(offset) + //keyOffset := binary.BigEndian.Uint64(offset) //keySize := binary.BigEndian.Uint64(); - if keyOffset != uint64(0) || string(key) != "d0" { - t.Errorf("Footer saved incorrect offset for the start of the index block:%d", binary.BigEndian.Uint64(footer[8:16])) + fmt.Printf("Footer bytes: %x\n", footer) + t.Logf("key size=%d", binary.BigEndian.Uint32(keySize)) + t.Logf("Index Block Loc=%v", binary.BigEndian.Uint64(footer[8:16])) + if binary.BigEndian.Uint64(offset) != 0 { + t.Errorf("Footer saved incorrect offset for the start of the index block:%d\nkey=%v", + binary.BigEndian.Uint64(offset), string(key)) } if binary.BigEndian.Uint64(footer[16:24]) != uint64(0xDEADBEEFDEADBEEF) { t.Errorf("Footer saved incorrect magic number: %v", binary.BigEndian.Uint64(footer[16:24])) From 9489131610fac221213f76bfe810d6fc13e34e91 Mon Sep 17 00:00:00 2001 From: Ivers Date: Mon, 22 Jun 2026 14:29:38 -0400 Subject: [PATCH 26/34] Made a function to read through all of the files in the directory --- internals/sstable/reader.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/internals/sstable/reader.go b/internals/sstable/reader.go index 4991373..1d11b22 100644 --- a/internals/sstable/reader.go +++ b/internals/sstable/reader.go @@ -19,7 +19,7 @@ import ( // If the key is not inside of the range in which we stated before, than we can just return nil and an error message // stating that the key can't be found func ReadFromFile(filePath, key string) ([]byte, error) { - file, err := os.Open(filepath.Join("../test", filePath)) + file, err := os.Open(filepath.Join("../data", filePath)) defer file.Close() if err != nil { @@ -42,7 +42,6 @@ func ReadFromFile(filePath, key string) ([]byte, error) { highKeyOffset := uint64(0) for i := indexBlockLoc; i <= uint64(fileInfo.Size()-24); { - //highKeyOffset = i keySize := make([]byte, 4) file.ReadAt(keySize, int64(i)) offsetKey := make([]byte, int64(binary.BigEndian.Uint32(keySize))) @@ -111,3 +110,21 @@ func ReadFromFile(filePath, key string) ([]byte, error) { return nil, errors.New("Key could not be found") } + +func ReadFromAllFiles(key string) ([]byte, error) { + fileDir, err := os.ReadDir("../data") + + if err != nil { + return nil, err + } + + for _, file := range fileDir { + res, err := ReadFromFile(file.Name(), key) + + if err == nil { + return res, nil + } + } + + return nil, errors.New("The key does not exist in any of the files") +} From f5a25f8200d33556c11e494a3bdb78edc7b93e63 Mon Sep 17 00:00:00 2001 From: Ivers Date: Wed, 24 Jun 2026 11:06:23 -0400 Subject: [PATCH 27/34] Made the reader function to read from all of the files --- internals/sstable/reader.go | 41 +++++++++++++++++++++++++++++--- internals/sstable/reader_test.go | 24 +++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/internals/sstable/reader.go b/internals/sstable/reader.go index 1d11b22..90148a7 100644 --- a/internals/sstable/reader.go +++ b/internals/sstable/reader.go @@ -5,10 +5,18 @@ import ( "errors" "os" "path/filepath" + "sort" + "strconv" + "strings" "github.com/Vince-maple-byte/KeyData/internals/record" ) +type SSTFile struct { + Generation int + FileName string +} + // How to read from the file // We first checking the magic number inside of the footer to see if the file is valid // We get the byte offset of the index block, and proceed to do: @@ -19,7 +27,7 @@ import ( // If the key is not inside of the range in which we stated before, than we can just return nil and an error message // stating that the key can't be found func ReadFromFile(filePath, key string) ([]byte, error) { - file, err := os.Open(filepath.Join("../data", filePath)) + file, err := os.Open(filepath.Join("../test", filePath)) defer file.Close() if err != nil { @@ -112,14 +120,33 @@ func ReadFromFile(filePath, key string) ([]byte, error) { } func ReadFromAllFiles(key string) ([]byte, error) { - fileDir, err := os.ReadDir("../data") + fileDir, err := os.ReadDir("../test") + + var files []SSTFile if err != nil { return nil, err } for _, file := range fileDir { - res, err := ReadFromFile(file.Name(), key) + gen, err := parseGeneration(file.Name()) + + if err != nil { + continue + } + + files = append(files, SSTFile{ + Generation: gen, + FileName: file.Name(), + }) + } + + sort.Slice(files, func(i, j int) bool { + return files[i].Generation < files[j].Generation + }) + + for _, file := range files { + res, err := ReadFromFile(file.FileName, key) if err == nil { return res, nil @@ -128,3 +155,11 @@ func ReadFromAllFiles(key string) ([]byte, error) { return nil, errors.New("The key does not exist in any of the files") } + +func parseGeneration(fileName string) (int, error) { + str := strings.Split(fileName, "_")[1] + str = strings.Split(str, ".")[0] + idx, err := strconv.Atoi(str) + + return idx, err +} diff --git a/internals/sstable/reader_test.go b/internals/sstable/reader_test.go index 1046ed8..b58f7ed 100644 --- a/internals/sstable/reader_test.go +++ b/internals/sstable/reader_test.go @@ -146,3 +146,27 @@ func TestReadFile(t *testing.T) { } } + +func TestReadFromAllFiles(t *testing.T) { + for i := range 10 { + d := memtable.CreateSkiplist() + for i := range 3000 * (i + 1) { + + r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") + d.Insert(strconv.Itoa(i), r) + } + _, err := sstable.WriteToFile(d.EntireList()) + + if err != nil { + t.Fatalf("Error in writing the file: %s", err.Error()) + } + } + + _, err := sstable.ReadFromAllFiles("29994") + + if err != nil { + t.Errorf("Was not able to find the valid record") + } + + tearDown("../test") +} From 731abb85cfd410f75a29c965bfbf4cf74424b0ab Mon Sep 17 00:00:00 2001 From: Ivers Date: Thu, 25 Jun 2026 11:11:21 -0400 Subject: [PATCH 28/34] Almost done with the grpc server just need to rewrite the request handler since the file was corrupted --- go.mod | 12 ++ go.sum | 38 ++++ internals/memtable/memtable.go | 12 -- internals/memtable/memtable_test.go | 4 +- internals/sstable/writer.go | 2 +- main.go | 28 +++ network/data.pb.go | 311 ++++++++++++++++++++++++++++ network/data.proto | 30 +++ network/data_grpc.pb.go | 159 ++++++++++++++ network/internals.go | 9 + 10 files changed, 590 insertions(+), 15 deletions(-) create mode 100644 go.sum create mode 100644 network/data.pb.go create mode 100644 network/data.proto create mode 100644 network/data_grpc.pb.go create mode 100644 network/internals.go diff --git a/go.mod b/go.mod index 3946826..3a9e041 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,15 @@ module github.com/Vince-maple-byte/KeyData go 1.25.4 + +require ( + google.golang.org/grpc v1.81.1 + google.golang.org/protobuf v1.36.11 +) + +require ( + golang.org/x/net v0.51.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..44c671d --- /dev/null +++ b/go.sum @@ -0,0 +1,38 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index 29504b7..e0f4b04 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -52,18 +52,6 @@ func (memtable *Memtable) Write(key, value, operation string) (bool, error) { return true, nil } -func (memtable *Memtable) Read(key string) ([]byte, error) { - record, err := memtable.list.Search(key) - - if err != nil { - //Call the method to perform the file i/o operation for reads - - //If we still return an error we would return a nil/empty byte slice and a error - } - - return record, nil -} - func (memtable *Memtable) Get(key string) ([]byte, error) { record, err := memtable.list.Search(key) diff --git a/internals/memtable/memtable_test.go b/internals/memtable/memtable_test.go index 6f8ac52..88a0468 100644 --- a/internals/memtable/memtable_test.go +++ b/internals/memtable/memtable_test.go @@ -48,11 +48,11 @@ func TestWriteForMultipleEntriesInMemtable(t *testing.T) { } -func TestReadForMemtable(t *testing.T) { +func TestGetForMemtable(t *testing.T) { mem := CreateMemtable() mem.Write("key", "val", "PUT") - actual, err := mem.Read("key") + actual, err := mem.Get("key") if err != nil { t.Errorf("The read was not able to complete (could mean that the key was not saved into the memtable)") diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index 0323d76..986e75d 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -256,7 +256,7 @@ func Compact(filePath string) error { //We do this so that we only take into account the file block, and not the index or footer footer := fileData[len(fileData)-24:] if binary.BigEndian.Uint64(footer[16:]) != 0xDEADBEEFDEADBEEF { - //os.Remove(fileInfo.Name()) + os.Remove(filepath.Join(filePath, fileInfo.Name())) return fmt.Errorf("invalid footer magic") } fileBlockEnds := binary.BigEndian.Uint64(footer[8:16]) diff --git a/main.go b/main.go index fed5096..412d736 100644 --- a/main.go +++ b/main.go @@ -1,8 +1,14 @@ package main import ( + "fmt" + "log" + "net" + "github.com/Vince-maple-byte/KeyData/internals/memtable" "github.com/Vince-maple-byte/KeyData/internals/sstable" + "github.com/Vince-maple-byte/KeyData/network" + "google.golang.org/grpc" ) //"time" @@ -43,4 +49,26 @@ func main() { sstable.MergeList = func() sstable.ListMerger { return memtable.CreateSkiplist() } + + mem := memtable.CreateMemtable() + + network.NetworkMemtable = func() network.InternalMemtable { + return mem + } + + port := 5773 + + lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port)) + + if err != nil { + log.Fatalf("Failed to listen: %v", err) + } + + var opts []grpc.ServerOption + + grpcServer := grpc.NewServer(opts...) + network.pb.RegisterDataServer(grpcServer, newServer()) + //pb.RegisterDataServer(grpcServer, newServer()) + // + grpcServer.Server(lis) } diff --git a/network/data.pb.go b/network/data.pb.go new file mode 100644 index 0000000..ab59dfc --- /dev/null +++ b/network/data.pb.go @@ -0,0 +1,311 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v7.35.0 +// source: network/data.proto + +package network + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type CreateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Operation string `protobuf:"bytes,1,opt,name=operation,proto3" json:"operation,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Payload string `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateRequest) Reset() { + *x = CreateRequest{} + mi := &file_network_data_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateRequest) ProtoMessage() {} + +func (x *CreateRequest) ProtoReflect() protoreflect.Message { + mi := &file_network_data_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateRequest.ProtoReflect.Descriptor instead. +func (*CreateRequest) Descriptor() ([]byte, []int) { + return file_network_data_proto_rawDescGZIP(), []int{0} +} + +func (x *CreateRequest) GetOperation() string { + if x != nil { + return x.Operation + } + return "" +} + +func (x *CreateRequest) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *CreateRequest) GetPayload() string { + if x != nil { + return x.Payload + } + return "" +} + +type CreateResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateResponse) Reset() { + *x = CreateResponse{} + mi := &file_network_data_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateResponse) ProtoMessage() {} + +func (x *CreateResponse) ProtoReflect() protoreflect.Message { + mi := &file_network_data_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateResponse.ProtoReflect.Descriptor instead. +func (*CreateResponse) Descriptor() ([]byte, []int) { + return file_network_data_proto_rawDescGZIP(), []int{1} +} + +func (x *CreateResponse) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} + +type SearchRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SearchRequest) Reset() { + *x = SearchRequest{} + mi := &file_network_data_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SearchRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchRequest) ProtoMessage() {} + +func (x *SearchRequest) ProtoReflect() protoreflect.Message { + mi := &file_network_data_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead. +func (*SearchRequest) Descriptor() ([]byte, []int) { + return file_network_data_proto_rawDescGZIP(), []int{2} +} + +func (x *SearchRequest) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +type SearchResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Payload string `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SearchResponse) Reset() { + *x = SearchResponse{} + mi := &file_network_data_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SearchResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SearchResponse) ProtoMessage() {} + +func (x *SearchResponse) ProtoReflect() protoreflect.Message { + mi := &file_network_data_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchResponse.ProtoReflect.Descriptor instead. +func (*SearchResponse) Descriptor() ([]byte, []int) { + return file_network_data_proto_rawDescGZIP(), []int{3} +} + +func (x *SearchResponse) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *SearchResponse) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *SearchResponse) GetPayload() string { + if x != nil { + return x.Payload + } + return "" +} + +var File_network_data_proto protoreflect.FileDescriptor + +const file_network_data_proto_rawDesc = "" + + "\n" + + "\x12network/data.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"Y\n" + + "\rCreateRequest\x12\x1c\n" + + "\toperation\x18\x01 \x01(\tR\toperation\x12\x10\n" + + "\x03key\x18\x02 \x01(\tR\x03key\x12\x18\n" + + "\apayload\x18\x03 \x01(\tR\apayload\"*\n" + + "\x0eCreateResponse\x12\x18\n" + + "\asuccess\x18\x01 \x01(\bR\asuccess\"!\n" + + "\rSearchRequest\x12\x10\n" + + "\x03key\x18\x02 \x01(\tR\x03key\"w\n" + + "\x0eSearchResponse\x129\n" + + "\n" + + "created_at\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x12\x10\n" + + "\x03key\x18\x02 \x01(\tR\x03key\x12\x18\n" + + "\apayload\x18\x03 \x01(\tR\apayload2`\n" + + "\x04Data\x12+\n" + + "\x06Search\x12\x0e.SearchRequest\x1a\x0f.SearchResponse\"\x00\x12+\n" + + "\x06Create\x12\x0e.CreateRequest\x1a\x0f.CreateResponse\"\x00B-Z+github.com/Vince-maple-byte/KeyData/networkb\x06proto3" + +var ( + file_network_data_proto_rawDescOnce sync.Once + file_network_data_proto_rawDescData []byte +) + +func file_network_data_proto_rawDescGZIP() []byte { + file_network_data_proto_rawDescOnce.Do(func() { + file_network_data_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_network_data_proto_rawDesc), len(file_network_data_proto_rawDesc))) + }) + return file_network_data_proto_rawDescData +} + +var file_network_data_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_network_data_proto_goTypes = []any{ + (*CreateRequest)(nil), // 0: CreateRequest + (*CreateResponse)(nil), // 1: CreateResponse + (*SearchRequest)(nil), // 2: SearchRequest + (*SearchResponse)(nil), // 3: SearchResponse + (*timestamppb.Timestamp)(nil), // 4: google.protobuf.Timestamp +} +var file_network_data_proto_depIdxs = []int32{ + 4, // 0: SearchResponse.created_at:type_name -> google.protobuf.Timestamp + 2, // 1: Data.Search:input_type -> SearchRequest + 0, // 2: Data.Create:input_type -> CreateRequest + 3, // 3: Data.Search:output_type -> SearchResponse + 1, // 4: Data.Create:output_type -> CreateResponse + 3, // [3:5] is the sub-list for method output_type + 1, // [1:3] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_network_data_proto_init() } +func file_network_data_proto_init() { + if File_network_data_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_network_data_proto_rawDesc), len(file_network_data_proto_rawDesc)), + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_network_data_proto_goTypes, + DependencyIndexes: file_network_data_proto_depIdxs, + MessageInfos: file_network_data_proto_msgTypes, + }.Build() + File_network_data_proto = out.File + file_network_data_proto_goTypes = nil + file_network_data_proto_depIdxs = nil +} diff --git a/network/data.proto b/network/data.proto new file mode 100644 index 0000000..479ec53 --- /dev/null +++ b/network/data.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/Vince-maple-byte/KeyData/network"; + +service Data { + rpc Search(SearchRequest) returns (SearchResponse) {} + rpc Create(CreateRequest) returns (CreateResponse) {} +} + +message CreateRequest { + string operation = 1; + string key = 2; + string payload = 3; +} + +message CreateResponse { + bool success = 1; +} + +message SearchRequest { + string key = 2; +} + +message SearchResponse { + google.protobuf.Timestamp created_at = 1; + string key = 2; + string payload = 3; +} diff --git a/network/data_grpc.pb.go b/network/data_grpc.pb.go new file mode 100644 index 0000000..01ff269 --- /dev/null +++ b/network/data_grpc.pb.go @@ -0,0 +1,159 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc v7.35.0 +// source: network/data.proto + +package network + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Data_Search_FullMethodName = "/Data/Search" + Data_Create_FullMethodName = "/Data/Create" +) + +// DataClient is the client API for Data service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type DataClient interface { + Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) + Create(ctx context.Context, in *CreateRequest, opts ...grpc.CallOption) (*CreateResponse, error) +} + +type dataClient struct { + cc grpc.ClientConnInterface +} + +func NewDataClient(cc grpc.ClientConnInterface) DataClient { + return &dataClient{cc} +} + +func (c *dataClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SearchResponse) + err := c.cc.Invoke(ctx, Data_Search_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dataClient) Create(ctx context.Context, in *CreateRequest, opts ...grpc.CallOption) (*CreateResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CreateResponse) + err := c.cc.Invoke(ctx, Data_Create_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// DataServer is the server API for Data service. +// All implementations must embed UnimplementedDataServer +// for forward compatibility. +type DataServer interface { + Search(context.Context, *SearchRequest) (*SearchResponse, error) + Create(context.Context, *CreateRequest) (*CreateResponse, error) + mustEmbedUnimplementedDataServer() +} + +// UnimplementedDataServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedDataServer struct{} + +func (UnimplementedDataServer) Search(context.Context, *SearchRequest) (*SearchResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Search not implemented") +} +func (UnimplementedDataServer) Create(context.Context, *CreateRequest) (*CreateResponse, error) { + return nil, status.Error(codes.Unimplemented, "method Create not implemented") +} +func (UnimplementedDataServer) mustEmbedUnimplementedDataServer() {} +func (UnimplementedDataServer) testEmbeddedByValue() {} + +// UnsafeDataServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to DataServer will +// result in compilation errors. +type UnsafeDataServer interface { + mustEmbedUnimplementedDataServer() +} + +func RegisterDataServer(s grpc.ServiceRegistrar, srv DataServer) { + // If the following call panics, it indicates UnimplementedDataServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Data_ServiceDesc, srv) +} + +func _Data_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SearchRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DataServer).Search(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Data_Search_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DataServer).Search(ctx, req.(*SearchRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Data_Create_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DataServer).Create(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Data_Create_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DataServer).Create(ctx, req.(*CreateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Data_ServiceDesc is the grpc.ServiceDesc for Data service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Data_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "Data", + HandlerType: (*DataServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Search", + Handler: _Data_Search_Handler, + }, + { + MethodName: "Create", + Handler: _Data_Create_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "network/data.proto", +} diff --git a/network/internals.go b/network/internals.go new file mode 100644 index 0000000..73e7959 --- /dev/null +++ b/network/internals.go @@ -0,0 +1,9 @@ +package network + +// This is the memtable +type InternalMemtable interface { + Write(key, value, operation string) (bool, error) + Get(key string) ([]byte, error) +} + +var NetworkMemtable func() InternalMemtable From 611cb2c330cc860526d30bf62fe09b7f0898dbda Mon Sep 17 00:00:00 2001 From: Ivers Date: Thu, 25 Jun 2026 11:27:50 -0400 Subject: [PATCH 29/34] completed the request_handler, next thing to do, fix the main class --- network/request_handler.go | 55 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 network/request_handler.go diff --git a/network/request_handler.go b/network/request_handler.go new file mode 100644 index 0000000..a885e77 --- /dev/null +++ b/network/request_handler.go @@ -0,0 +1,55 @@ +package network + +import ( + context "context" + + "github.com/Vince-maple-byte/KeyData/internals/record" + "github.com/Vince-maple-byte/KeyData/internals/sstable" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type Server struct { + UnimplementedDataServer +} + +func (s *Server) Search(ctx context.Context, search *SearchRequest) (*SearchResponse, error) { + res, err := searchHelper(search.GetKey()) + + if err != nil { + return &SearchResponse{CreatedAt: nil, Key: "", Payload: ""}, status.Error(codes.NotFound, err.Error()) + } + + contents := record.GetContents(res) + + return &SearchResponse{ + CreatedAt: timestamppb.New(contents.Timestamp), + Key: contents.Key, + Payload: contents.Payload, + }, nil +} + +func (s *Server) Create(ctx context.Context, create *CreateRequest) (*CreateResponse, error) { + mem := NetworkMemtable() + + ok, err := mem.Write(create.GetKey(), create.GetOperation(), create.Payload) + + if !ok { + return &CreateResponse{Success: ok}, status.Error(codes.Unknown, err.Error()) + } + + return &CreateResponse{Success: ok}, nil +} + +func searchHelper(key string) ([]byte, error) { + mem := NetworkMemtable() + + res, err := mem.Get(key) + + if err != nil { + return sstable.ReadFromAllFiles(key) + } + + return res, err +} From 32fa2a038a5c46a5255a9e00559f7480ab79f71c Mon Sep 17 00:00:00 2001 From: Ivers Date: Thu, 25 Jun 2026 12:17:34 -0400 Subject: [PATCH 30/34] Finished with the entire grpc server implementation. Just need to do some integration tests to make sure everything is working fine, and then I should be able to merge with main --- internals/data/kd_1.sst | 0 internals/memtable/memtable.go | 12 ++++++++++-- internals/record/record.go | 3 ++- internals/sstable/reader.go | 12 +++++++++--- internals/sstable/writer.go | 2 +- main.go | 14 +++++++++++--- network/request_handler.go | 2 +- 7 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 internals/data/kd_1.sst diff --git a/internals/data/kd_1.sst b/internals/data/kd_1.sst new file mode 100644 index 0000000..e69de29 diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index e0f4b04..d93a9f9 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -1,6 +1,8 @@ package memtable import ( + "errors" + "github.com/Vince-maple-byte/KeyData/internals/record" "github.com/Vince-maple-byte/KeyData/internals/sstable" ) @@ -53,11 +55,17 @@ func (memtable *Memtable) Write(key, value, operation string) (bool, error) { } func (memtable *Memtable) Get(key string) ([]byte, error) { - record, err := memtable.list.Search(key) + records, err := memtable.list.Search(key) if err != nil { return nil, err } - return record, nil + content := record.GetContents(records) + + if content.Tombstone != 0 { + return nil, errors.New("Key/Value pair doesn't exist") + } + + return records, nil } diff --git a/internals/record/record.go b/internals/record/record.go index a97cb37..aa6c588 100644 --- a/internals/record/record.go +++ b/internals/record/record.go @@ -3,6 +3,7 @@ package record import ( "encoding/binary" "errors" + "fmt" "hash/crc32" "time" //"fmt" @@ -54,7 +55,7 @@ func CreateRecord(key, payload, operation string) ([]byte, error) { //Don't need to check for the GET operation since that should automatically go to the ReadFile section. if operation == "" || (operation != "PUT" && operation != "DELETE") { - return nil, errors.New("Invalid Operation") + return nil, errors.New(fmt.Sprintf("Invalid operation: %v", operation)) } //Make sure to use time.AppendBinary https://pkg.go.dev/time#example-Time.AppendBinary diff --git a/internals/sstable/reader.go b/internals/sstable/reader.go index 90148a7..c8baae9 100644 --- a/internals/sstable/reader.go +++ b/internals/sstable/reader.go @@ -27,7 +27,7 @@ type SSTFile struct { // If the key is not inside of the range in which we stated before, than we can just return nil and an error message // stating that the key can't be found func ReadFromFile(filePath, key string) ([]byte, error) { - file, err := os.Open(filepath.Join("../test", filePath)) + file, err := os.Open(filepath.Join("../data", filePath)) defer file.Close() if err != nil { @@ -120,7 +120,7 @@ func ReadFromFile(filePath, key string) ([]byte, error) { } func ReadFromAllFiles(key string) ([]byte, error) { - fileDir, err := os.ReadDir("../test") + fileDir, err := os.ReadDir("../data") var files []SSTFile @@ -129,7 +129,7 @@ func ReadFromAllFiles(key string) ([]byte, error) { } for _, file := range fileDir { - gen, err := parseGeneration(file.Name()) + gen, err := parseGeneration(filepath.Join("../data", file.Name())) if err != nil { continue @@ -149,6 +149,12 @@ func ReadFromAllFiles(key string) ([]byte, error) { res, err := ReadFromFile(file.FileName, key) if err == nil { + contents := record.GetContents(res) + + if contents.Tombstone != 0 { + return nil, errors.New("Key/Value pair doesn't exist") + } + return res, nil } } diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index 986e75d..9de85d4 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -27,7 +27,7 @@ const ( // TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. func WriteToFile(list [][]byte) (bool, error) { - filePath := "../test" + filePath := "../data" filename := "" files, err := os.ReadDir(filePath) diff --git a/main.go b/main.go index 412d736..12c7b47 100644 --- a/main.go +++ b/main.go @@ -9,6 +9,7 @@ import ( "github.com/Vince-maple-byte/KeyData/internals/sstable" "github.com/Vince-maple-byte/KeyData/network" "google.golang.org/grpc" + "google.golang.org/grpc/reflection" ) //"time" @@ -67,8 +68,15 @@ func main() { var opts []grpc.ServerOption grpcServer := grpc.NewServer(opts...) - network.pb.RegisterDataServer(grpcServer, newServer()) - //pb.RegisterDataServer(grpcServer, newServer()) + //network.pb.RegisterDataServer(grpcServer, newServer()) + network.RegisterDataServer(grpcServer, &network.Server{}) + reflection.Register(grpcServer) // - grpcServer.Server(lis) + err = grpcServer.Serve(lis) + + if err != nil { + log.Fatalf("Failed to serve: %v", err) + } + + fmt.Printf("logging in port: %d", port) } diff --git a/network/request_handler.go b/network/request_handler.go index a885e77..56993f3 100644 --- a/network/request_handler.go +++ b/network/request_handler.go @@ -33,7 +33,7 @@ func (s *Server) Search(ctx context.Context, search *SearchRequest) (*SearchResp func (s *Server) Create(ctx context.Context, create *CreateRequest) (*CreateResponse, error) { mem := NetworkMemtable() - ok, err := mem.Write(create.GetKey(), create.GetOperation(), create.Payload) + ok, err := mem.Write(create.GetKey(), create.GetPayload(), create.GetOperation()) if !ok { return &CreateResponse{Success: ok}, status.Error(codes.Unknown, err.Error()) From 3da213781d7af3de50c8767c6ff127ccc2409f45 Mon Sep 17 00:00:00 2001 From: Ivers Date: Mon, 29 Jun 2026 19:17:31 -0400 Subject: [PATCH 31/34] all of the changes we made so far. --- filepath.go | 6 +++ internals/data/kd_1.sst | 0 internals/memtable/memtable.go | 71 +++++++++++++++++++++++++++++++- internals/memtable/skiplist.go | 9 ---- internals/sstable/filepath.go | 9 ++++ internals/sstable/reader.go | 6 +-- internals/sstable/reader_test.go | 15 +++---- internals/sstable/writer.go | 5 +-- internals/sstable/writer_test.go | 42 ++++++++++--------- main.go | 14 +++---- 10 files changed, 127 insertions(+), 50 deletions(-) create mode 100644 filepath.go delete mode 100644 internals/data/kd_1.sst create mode 100644 internals/sstable/filepath.go diff --git a/filepath.go b/filepath.go new file mode 100644 index 0000000..95db806 --- /dev/null +++ b/filepath.go @@ -0,0 +1,6 @@ +package main + +type FilePath struct { + prod string + test string +} diff --git a/internals/data/kd_1.sst b/internals/data/kd_1.sst deleted file mode 100644 index e69de29..0000000 diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index d93a9f9..84ca1c6 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -1,7 +1,9 @@ package memtable import ( + "encoding/binary" "errors" + "os" "github.com/Vince-maple-byte/KeyData/internals/record" "github.com/Vince-maple-byte/KeyData/internals/sstable" @@ -28,12 +30,18 @@ func (memtable *Memtable) Write(key, value, operation string) (bool, error) { return false, err } + ok, err := WriteToWal(record, "") + + if !ok { + return false, err + } + memtable.list.Insert(key, record) memtable.size += 1 if memtable.size >= MAX_SIZE { content := memtable.list.EntireList() - _, errF := sstable.WriteToFile(content) + _, errF := sstable.WriteToFile(content, "./internals/") if errF != nil { return false, errF @@ -69,3 +77,64 @@ func (memtable *Memtable) Get(key string) ([]byte, error) { return records, nil } + +func WriteToWal(record []byte, filePath string) (bool, error) { + file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + defer file.Close() + + if err != nil { + return false, errors.New("Not able to create/open the wal file") + } + + _, err = file.Write(record) + + if err != nil { + return false, err + } + + return true, nil +} + +func (m *Memtable) MemtableStartUp(filePath string) (bool, error) { + file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + defer file.Close() + + if err != nil { + return false, err + } + + fileInfo, err := file.Stat() + + if err != nil { + return false, err + } + + for i := int64(0); i < fileInfo.Size(); { + header := make([]byte, 21) + file.ReadAt(header, i) + + keySize := binary.BigEndian.Uint32(header[13:17]) + payloadSize := binary.BigEndian.Uint32(header[17:21]) + + keyValuePair := make([]byte, keySize+payloadSize) + file.ReadAt(keyValuePair, i+21) + + ok := record.ChecksumChecker( + keyValuePair[:keySize], + keyValuePair[keySize:], + binary.BigEndian.Uint64(header[0:8]), + binary.BigEndian.Uint32(header[8:12]), + ) + + if !ok { + break + } + + rec := append(header, keyValuePair...) + m.list.Insert(string(keyValuePair[:keySize]), rec) + } + + return true, nil +} diff --git a/internals/memtable/skiplist.go b/internals/memtable/skiplist.go index e1a955b..28c3eb8 100644 --- a/internals/memtable/skiplist.go +++ b/internals/memtable/skiplist.go @@ -135,15 +135,6 @@ func (list Skiplist) searchNode(key string) (*Node, error) { } func (list *Skiplist) EmptyList() { - // head := &Node{ - // key: "", - // value: nil, - // levels: make([]*Node, MAX_LEVEL), - // }; - // return &Skiplist{ - // head: head, - // size: 0, - // }; list.head.key = "" list.head.value = nil list.head.levels = make([]*Node, MAX_LEVEL) diff --git a/internals/sstable/filepath.go b/internals/sstable/filepath.go new file mode 100644 index 0000000..2e989c5 --- /dev/null +++ b/internals/sstable/filepath.go @@ -0,0 +1,9 @@ +package sstable + +type FilePath struct { + Path string +} + +func NewPath(path string) *FilePath { + return &FilePath{Path: path} +} diff --git a/internals/sstable/reader.go b/internals/sstable/reader.go index c8baae9..1cde974 100644 --- a/internals/sstable/reader.go +++ b/internals/sstable/reader.go @@ -27,7 +27,7 @@ type SSTFile struct { // If the key is not inside of the range in which we stated before, than we can just return nil and an error message // stating that the key can't be found func ReadFromFile(filePath, key string) ([]byte, error) { - file, err := os.Open(filepath.Join("../data", filePath)) + file, err := os.Open(filePath) defer file.Close() if err != nil { @@ -119,8 +119,8 @@ func ReadFromFile(filePath, key string) ([]byte, error) { return nil, errors.New("Key could not be found") } -func ReadFromAllFiles(key string) ([]byte, error) { - fileDir, err := os.ReadDir("../data") +func ReadFromAllFiles(key string, dir string) ([]byte, error) { + fileDir, err := os.ReadDir(dir) var files []SSTFile diff --git a/internals/sstable/reader_test.go b/internals/sstable/reader_test.go index b58f7ed..05e5254 100644 --- a/internals/sstable/reader_test.go +++ b/internals/sstable/reader_test.go @@ -2,6 +2,7 @@ package sstable_test import ( "os" + "path/filepath" "strconv" "testing" @@ -119,29 +120,29 @@ func TestReadFile(t *testing.T) { for _, test := range tests { t.Run(test.testName, func(t *testing.T) { - _, err := sstable.WriteToFile(test.data) + dir := t.TempDir() + _, err := sstable.WriteToFile(test.data, dir) if err != nil { t.Fatalf("Error in writing the file: %s", err.Error()) } - fileInfo, err := os.Lstat("../test/kd_1.sst") + fileInfo, err := os.Lstat(filepath.Join(dir, "kd_1.sst")) size := fileInfo.Size() if err != nil { tearDown("../test") t.Fatalf("Error in accessing the file stats: %s", err.Error()) } - rec, err := sstable.ReadFromFile("kd_1.sst", test.key) + rec, err := sstable.ReadFromFile(filepath.Join(dir, "kd_1.sst"), test.key) if rec == nil || err != nil { - tearDown("../test") - t.Fatalf("Error recieved: %v\nSize of the file: %d", err.Error(), size) + t.Fatalf("Error received: %v\nSize of the file: %d", err.Error(), size) } if record.GetContents(rec).Payload != test.expected { t.Errorf("Test failed for %s\nExpected:%v\nActual:%v", test.testName, test.expected, record.GetContents(rec).Payload) } - tearDown("../test") + //tearDown("../test") }) } @@ -155,7 +156,7 @@ func TestReadFromAllFiles(t *testing.T) { r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") d.Insert(strconv.Itoa(i), r) } - _, err := sstable.WriteToFile(d.EntireList()) + _, err := sstable.WriteToFile(d.EntireList(), "../test") if err != nil { t.Fatalf("Error in writing the file: %s", err.Error()) diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index 9de85d4..8ebad9b 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -26,8 +26,7 @@ const ( ) // TODO:Finish with the write method for the file i/o; use the diagram that I made as a guide. -func WriteToFile(list [][]byte) (bool, error) { - filePath := "../data" +func WriteToFile(list [][]byte, filePath string) (bool, error) { filename := "" files, err := os.ReadDir(filePath) @@ -291,7 +290,7 @@ func Compact(filePath string) error { //Technically speaking we can just recall the write to file again to make the new file // Since the Entire write operation is there. // So I'm planning on calling the Compact function after the write to file function goes through in the Memtable class - _, err := WriteToFile(fileContents) + _, err := WriteToFile(fileContents, filePath) if err != nil { return err diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go index 2923ec0..f4b01b4 100644 --- a/internals/sstable/writer_test.go +++ b/internals/sstable/writer_test.go @@ -12,7 +12,7 @@ import ( "github.com/Vince-maple-byte/KeyData/internals/sstable" ) -func startUp(data ...string) (int, error) { +func startUp(filePath string, data ...string) (int, error) { size := 0 for range 10 { @@ -21,7 +21,7 @@ func startUp(data ...string) (int, error) { w, _ := record.CreateRecord(data[i], data[i], "PUT") file = append(file, w) } - _, err := sstable.WriteToFile(file) + _, err := sstable.WriteToFile(file, filePath) if err != nil { return 0, err @@ -69,8 +69,9 @@ func TestBucketsForFiles(t *testing.T) { } for _, test := range tests { - totalSize, err := startUp(test.data...) - filePath := "../test" + filePath := t.TempDir() + totalSize, err := startUp(filePath, test.data...) + if err != nil || totalSize == 0 { t.Fatalf("Could not start up the test") } @@ -94,7 +95,7 @@ func TestBucketsForFiles(t *testing.T) { } } - tearDown("../test") + //tearDown("../test") } func TestCompactFiles(t *testing.T) { @@ -106,6 +107,8 @@ func TestCompactFiles(t *testing.T) { return memtable.CreateSkiplist() } + dir := t.TempDir() + //size := 0 for range 10 { file := make([][]byte, 0, 10) @@ -113,7 +116,8 @@ func TestCompactFiles(t *testing.T) { w, _ := record.CreateRecord(data[i], data[i], "PUT") file = append(file, w) } - _, err := sstable.WriteToFile(file) + + _, err := sstable.WriteToFile(file, dir) t.Logf("Size: %d", len(file)) @@ -123,34 +127,35 @@ func TestCompactFiles(t *testing.T) { } - err := sstable.Compact("../test") + err := sstable.Compact(dir) if err != nil { t.Errorf("Not able to complete the compaction\n Recieved this error code:\n%v", err) } - files, err := os.ReadDir("../test") + files, err := os.ReadDir(dir) if err != nil { - t.Fatalf("Not able to access the directory for testing: ../test") + t.Fatalf("Not able to access the directory for testing: %v", dir) } if len(files) != 1 { t.Errorf("Improper amount of files inside of the test directory:\nExpected: %d; Actual:%d", 1, len(files)) } - tearDown("../test") + //tearDown("../test") } func TestWriteToFile(t *testing.T) { fileContents := make([][]byte, 0, 10) + dir := t.TempDir() for range 1 { r, _ := record.CreateRecord("a", "1", "PUT") fileContents = append(fileContents, r) } - ok, err := sstable.WriteToFile(fileContents) + ok, err := sstable.WriteToFile(fileContents, dir) if err != nil { t.Errorf("Error encountered: %v\n", err) @@ -160,33 +165,34 @@ func TestWriteToFile(t *testing.T) { t.Errorf("Not able to create the file") } - file, _ := os.Open("../test/kd_1.sst") + file, _ := os.Open(filepath.Join(dir, "kd_1.sst")) + defer file.Close() info, _ := file.Stat() if info.Size() == 0 { t.Error("Did not populate file") } - file.Close() - tearDown("../test") + //tearDown("../test") } func TestFooter(t *testing.T) { fileContents := make([][]byte, 0) + dir := t.TempDir() for i := range 3200 { c, _ := record.CreateRecord(fmt.Sprintf("d%v", i), fmt.Sprintf("d%v", i), "PUT") fileContents = append(fileContents, c) } - _, err := sstable.WriteToFile(fileContents) + _, err := sstable.WriteToFile(fileContents, dir) if err != nil { t.Fatal(err.Error()) } - file, err := os.Open("../test/kd_1.sst") - //defer file.Close() + file, err := os.Open(filepath.Join(dir, "kd_1.sst")) + defer file.Close() if err != nil { t.Fatal(err.Error()) @@ -228,6 +234,4 @@ func TestFooter(t *testing.T) { t.Errorf("Footer saved incorrect magic number: %v", binary.BigEndian.Uint64(footer[16:24])) } - file.Close() - tearDown("../test") } diff --git a/main.go b/main.go index 12c7b47..8719343 100644 --- a/main.go +++ b/main.go @@ -12,16 +12,8 @@ import ( "google.golang.org/grpc/reflection" ) -//"time" -// "github.com/Vince-maple-byte/KeyData/internals/file" -//"github.com/Vince-maple-byte/KeyData/internals/record" -// "unsafe" - /* -TODO: -Refactor the file folder to handle file operations for both in memory data store (skiplist) and the file i/o in case we have a file miss. So get rid of the map, and the everything else. - How the structure of the timestamp would look like Timestamp | CRC32 error checksum| Tombstone (It's one byte long; basically 0 and 1 to determine whether this is a deleted key or not) | Key Size | Payload(Value) Size | Key Value | Payload @@ -43,6 +35,8 @@ Make some unit tests to test the methods From there we can move on to SSTable, compaction, LSM Tables, and finally bloom tables +TODO: Fix the filePath for the project so that it doesn't have to be hardcoded. +./internals/data works for the production while ../data works for unit tests */ func main() { @@ -51,8 +45,12 @@ func main() { return memtable.CreateSkiplist() } + filePath := + mem := memtable.CreateMemtable() + mem.MemtableStartUp() + network.NetworkMemtable = func() network.InternalMemtable { return mem } From 4c2e2c15feaf5b4a15f01967c8ec1493380f964f Mon Sep 17 00:00:00 2001 From: Ivers Date: Thu, 2 Jul 2026 22:49:29 -0400 Subject: [PATCH 32/34] Fixed some file path naming conventions. --- filepath.go | 6 ------ internals/sstable/reader_test.go | 8 +++++--- main.go | 6 +++--- network/internals.go | 7 +++++++ network/request_handler.go | 3 ++- 5 files changed, 17 insertions(+), 13 deletions(-) delete mode 100644 filepath.go diff --git a/filepath.go b/filepath.go deleted file mode 100644 index 95db806..0000000 --- a/filepath.go +++ /dev/null @@ -1,6 +0,0 @@ -package main - -type FilePath struct { - prod string - test string -} diff --git a/internals/sstable/reader_test.go b/internals/sstable/reader_test.go index 05e5254..60c5cb0 100644 --- a/internals/sstable/reader_test.go +++ b/internals/sstable/reader_test.go @@ -149,6 +149,8 @@ func TestReadFile(t *testing.T) { } func TestReadFromAllFiles(t *testing.T) { + + dir := t.TempDir() for i := range 10 { d := memtable.CreateSkiplist() for i := range 3000 * (i + 1) { @@ -156,18 +158,18 @@ func TestReadFromAllFiles(t *testing.T) { r, _ := record.CreateRecord(strconv.Itoa(i), strconv.Itoa(i+1), "PUT") d.Insert(strconv.Itoa(i), r) } - _, err := sstable.WriteToFile(d.EntireList(), "../test") + _, err := sstable.WriteToFile(d.EntireList(), dir) if err != nil { t.Fatalf("Error in writing the file: %s", err.Error()) } } - _, err := sstable.ReadFromAllFiles("29994") + _, err := sstable.ReadFromAllFiles("29994", dir) if err != nil { t.Errorf("Was not able to find the valid record") } - tearDown("../test") + //tearDown("../test") } diff --git a/main.go b/main.go index 8719343..2441d64 100644 --- a/main.go +++ b/main.go @@ -45,16 +45,16 @@ func main() { return memtable.CreateSkiplist() } - filePath := - mem := memtable.CreateMemtable() - mem.MemtableStartUp() + mem.MemtableStartUp("./internals/wal") network.NetworkMemtable = func() network.InternalMemtable { return mem } + network.FileDir = &network.Dir{Path: "./internals/data"} + port := 5773 lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port)) diff --git a/network/internals.go b/network/internals.go index 73e7959..bf3f453 100644 --- a/network/internals.go +++ b/network/internals.go @@ -7,3 +7,10 @@ type InternalMemtable interface { } var NetworkMemtable func() InternalMemtable + +// This is the filepath struct +type Dir struct { + Path string +} + +var FileDir *Dir diff --git a/network/request_handler.go b/network/request_handler.go index 56993f3..ee0eb53 100644 --- a/network/request_handler.go +++ b/network/request_handler.go @@ -47,8 +47,9 @@ func searchHelper(key string) ([]byte, error) { res, err := mem.Get(key) + //Got to figure out how to pass in the filePath for the directory while it still being reusable if err != nil { - return sstable.ReadFromAllFiles(key) + return sstable.ReadFromAllFiles(key, FileDir.Path) } return res, err From 87c146bbaca36589a605685b7fa3690359ee21a8 Mon Sep 17 00:00:00 2001 From: Ivers Date: Thu, 2 Jul 2026 23:32:45 -0400 Subject: [PATCH 33/34] Refactored the memtable and the network packages to be more readable and maintainable. --- internals/memtable/memtable.go | 47 ++++++++++++++------------- internals/memtable/memtable_test.go | 49 ++++++++++++++++++++++++++--- main.go | 16 +++++----- network/internals.go | 2 +- network/request_handler.go | 17 +++++----- 5 files changed, 89 insertions(+), 42 deletions(-) diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index 84ca1c6..5d5e6db 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -12,58 +12,63 @@ import ( const MAX_SIZE = 3200 type Memtable struct { - list *Skiplist - size int + list *Skiplist + size int + WalPath string + Dir string } -func CreateMemtable() *Memtable { +func CreateMemtable(wal, dir string) *Memtable { return &Memtable{ - list: CreateSkiplist(), - size: 0, + list: CreateSkiplist(), + size: 0, + WalPath: wal, + Dir: dir, } } -func (memtable *Memtable) Write(key, value, operation string) (bool, error) { +func (m *Memtable) Write(key, value, operation string) (bool, error) { record, err := record.CreateRecord(key, value, operation) if err != nil { return false, err } - ok, err := WriteToWal(record, "") + ok, err := m.writeToWal(record) if !ok { return false, err } - memtable.list.Insert(key, record) - memtable.size += 1 + m.list.Insert(key, record) + m.size += 1 - if memtable.size >= MAX_SIZE { - content := memtable.list.EntireList() - _, errF := sstable.WriteToFile(content, "./internals/") + if m.size >= MAX_SIZE { + content := m.list.EntireList() + _, errF := sstable.WriteToFile(content, m.Dir) if errF != nil { return false, errF } //For now, when running test we just replace the folder path as the test folder. - err = sstable.Compact("../data") + err = sstable.Compact(m.Dir) if err != nil { return false, err } - memtable.list.EmptyList() - memtable.size = 0 + m.list.EmptyList() + m.size = 0 + os.Remove(m.WalPath) } return true, nil } -func (memtable *Memtable) Get(key string) ([]byte, error) { - records, err := memtable.list.Search(key) +func (m *Memtable) Get(key string) ([]byte, error) { + records, err := m.list.Search(key) if err != nil { return nil, err @@ -78,8 +83,8 @@ func (memtable *Memtable) Get(key string) ([]byte, error) { return records, nil } -func WriteToWal(record []byte, filePath string) (bool, error) { - file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) +func (m Memtable) writeToWal(record []byte) (bool, error) { + file, err := os.OpenFile(m.WalPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) defer file.Close() @@ -96,8 +101,8 @@ func WriteToWal(record []byte, filePath string) (bool, error) { return true, nil } -func (m *Memtable) MemtableStartUp(filePath string) (bool, error) { - file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) +func (m *Memtable) MemtableStartUp() (bool, error) { + file, err := os.OpenFile(m.WalPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) defer file.Close() diff --git a/internals/memtable/memtable_test.go b/internals/memtable/memtable_test.go index 88a0468..be4ebe5 100644 --- a/internals/memtable/memtable_test.go +++ b/internals/memtable/memtable_test.go @@ -1,13 +1,24 @@ package memtable import ( + "os" + "path/filepath" "testing" "github.com/Vince-maple-byte/KeyData/internals/record" ) func TestWriteForSingleEntryInMemtable(t *testing.T) { - mem := CreateMemtable() + + baseDir := t.TempDir() + + walDir := filepath.Join(baseDir, "dir1") + os.Mkdir(walDir, 0700) + + dataDir := filepath.Join(baseDir, "dir2") + os.Mkdir(dataDir, 0700) + + mem := CreateMemtable(walDir, dataDir) prevSize := mem.size _, err := mem.Write("key", "val", "PUT") @@ -21,7 +32,14 @@ func TestWriteForSingleEntryInMemtable(t *testing.T) { } func TestWriteForMultipleEntriesInMemtable(t *testing.T) { - mem := CreateMemtable() + baseDir := t.TempDir() + + walDir := filepath.Join(baseDir, "dir1") + os.Mkdir(walDir, 0700) + + dataDir := filepath.Join(baseDir, "dir2") + os.Mkdir(dataDir, 0700) + mem := CreateMemtable(walDir, dataDir) tests := []struct { key, value, op string @@ -49,7 +67,14 @@ func TestWriteForMultipleEntriesInMemtable(t *testing.T) { } func TestGetForMemtable(t *testing.T) { - mem := CreateMemtable() + baseDir := t.TempDir() + + walDir := filepath.Join(baseDir, "dir1") + os.Mkdir(walDir, 0700) + + dataDir := filepath.Join(baseDir, "dir2") + os.Mkdir(dataDir, 0700) + mem := CreateMemtable(walDir, dataDir) mem.Write("key", "val", "PUT") actual, err := mem.Get("key") @@ -64,7 +89,14 @@ func TestGetForMemtable(t *testing.T) { } func TestDeleteForMemtable(t *testing.T) { - mem := CreateMemtable() + baseDir := t.TempDir() + + walDir := filepath.Join(baseDir, "dir1") + os.Mkdir(walDir, 0700) + + dataDir := filepath.Join(baseDir, "dir2") + os.Mkdir(dataDir, 0700) + mem := CreateMemtable(walDir, dataDir) mem.Write("key", "val", "PUT") prevSize := mem.size @@ -83,7 +115,14 @@ func TestDeleteForMemtable(t *testing.T) { func TestWrite_FlushResetsState(t *testing.T) { t.Skip("requires sstable wiring and writable data dir — run as integration test") - mt := CreateMemtable() + baseDir := t.TempDir() + + walDir := filepath.Join(baseDir, "dir1") + os.Mkdir(walDir, 0700) + + dataDir := filepath.Join(baseDir, "dir2") + os.Mkdir(dataDir, 0700) + mt := CreateMemtable(walDir, dataDir) for i := 0; i < MAX_SIZE; i++ { key := "key" + string(rune(i)) diff --git a/main.go b/main.go index 2441d64..27ef79f 100644 --- a/main.go +++ b/main.go @@ -45,16 +45,17 @@ func main() { return memtable.CreateSkiplist() } - mem := memtable.CreateMemtable() + walPath := "./internals/wal/mem1.wal" + dataDir := "./internals/data" + mem := memtable.CreateMemtable(walPath, dataDir) - mem.MemtableStartUp("./internals/wal") + mem.MemtableStartUp() - network.NetworkMemtable = func() network.InternalMemtable { - return mem + server := &network.Server{ + Memtable: mem, + DataDir: dataDir, } - network.FileDir = &network.Dir{Path: "./internals/data"} - port := 5773 lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port)) @@ -67,9 +68,8 @@ func main() { grpcServer := grpc.NewServer(opts...) //network.pb.RegisterDataServer(grpcServer, newServer()) - network.RegisterDataServer(grpcServer, &network.Server{}) + network.RegisterDataServer(grpcServer, server) reflection.Register(grpcServer) - // err = grpcServer.Serve(lis) if err != nil { diff --git a/network/internals.go b/network/internals.go index bf3f453..9a3dad3 100644 --- a/network/internals.go +++ b/network/internals.go @@ -13,4 +13,4 @@ type Dir struct { Path string } -var FileDir *Dir +var FileDir Dir diff --git a/network/request_handler.go b/network/request_handler.go index ee0eb53..df18243 100644 --- a/network/request_handler.go +++ b/network/request_handler.go @@ -12,10 +12,13 @@ import ( type Server struct { UnimplementedDataServer + + Memtable InternalMemtable + DataDir string } func (s *Server) Search(ctx context.Context, search *SearchRequest) (*SearchResponse, error) { - res, err := searchHelper(search.GetKey()) + res, err := s.searchHelper(search.GetKey()) if err != nil { return &SearchResponse{CreatedAt: nil, Key: "", Payload: ""}, status.Error(codes.NotFound, err.Error()) @@ -31,9 +34,9 @@ func (s *Server) Search(ctx context.Context, search *SearchRequest) (*SearchResp } func (s *Server) Create(ctx context.Context, create *CreateRequest) (*CreateResponse, error) { - mem := NetworkMemtable() + //mem := NetworkMemtable() - ok, err := mem.Write(create.GetKey(), create.GetPayload(), create.GetOperation()) + ok, err := s.Memtable.Write(create.GetKey(), create.GetPayload(), create.GetOperation()) if !ok { return &CreateResponse{Success: ok}, status.Error(codes.Unknown, err.Error()) @@ -42,14 +45,14 @@ func (s *Server) Create(ctx context.Context, create *CreateRequest) (*CreateResp return &CreateResponse{Success: ok}, nil } -func searchHelper(key string) ([]byte, error) { - mem := NetworkMemtable() +func (s Server) searchHelper(key string) ([]byte, error) { + //mem := NetworkMemtable() - res, err := mem.Get(key) + res, err := s.Memtable.Get(key) //Got to figure out how to pass in the filePath for the directory while it still being reusable if err != nil { - return sstable.ReadFromAllFiles(key, FileDir.Path) + return sstable.ReadFromAllFiles(key, s.DataDir) } return res, err From 01fe3e45d75502e5b5f418880dacdf15150bdaae Mon Sep 17 00:00:00 2001 From: Vince-maple-byte <73848683+Vince-maple-byte@users.noreply.github.com> Date: Mon, 6 Jul 2026 02:46:28 -0400 Subject: [PATCH 34/34] I have completed phase2 :) Should make some benchmark tests in order to make a video explaining performance gains etc. --- .gitignore | 5 +- internals/memtable/memtable.go | 41 ++++++++------ internals/memtable/memtable_test.go | 88 ++++++++++++++++++++++++----- internals/sstable/writer.go | 5 +- internals/sstable/writer_test.go | 4 ++ 5 files changed, 110 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 2963b31..f3df9a6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ #Ignore .txt and log files since these are for testing. *.txt -*.log \ No newline at end of file +*.log +*.sst +*.wal + diff --git a/internals/memtable/memtable.go b/internals/memtable/memtable.go index 5d5e6db..455a772 100644 --- a/internals/memtable/memtable.go +++ b/internals/memtable/memtable.go @@ -5,6 +5,7 @@ import ( "errors" "os" + "fmt" "github.com/Vince-maple-byte/KeyData/internals/record" "github.com/Vince-maple-byte/KeyData/internals/sstable" ) @@ -14,16 +15,16 @@ const MAX_SIZE = 3200 type Memtable struct { list *Skiplist size int - WalPath string - Dir string + WalFilePath string + DataDir string } func CreateMemtable(wal, dir string) *Memtable { return &Memtable{ list: CreateSkiplist(), size: 0, - WalPath: wal, - Dir: dir, + WalFilePath: wal, + DataDir: dir, } } @@ -45,14 +46,14 @@ func (m *Memtable) Write(key, value, operation string) (bool, error) { if m.size >= MAX_SIZE { content := m.list.EntireList() - _, errF := sstable.WriteToFile(content, m.Dir) + _, errF := sstable.WriteToFile(content, m.DataDir) if errF != nil { return false, errF } - //For now, when running test we just replace the folder path as the test folder. - err = sstable.Compact(m.Dir) + //For now, when running test we just + err = sstable.Compact(m.DataDir) if err != nil { return false, err @@ -60,7 +61,7 @@ func (m *Memtable) Write(key, value, operation string) (bool, error) { m.list.EmptyList() m.size = 0 - os.Remove(m.WalPath) + os.Remove(m.WalFilePath) } @@ -84,14 +85,15 @@ func (m *Memtable) Get(key string) ([]byte, error) { } func (m Memtable) writeToWal(record []byte) (bool, error) { - file, err := os.OpenFile(m.WalPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - - defer file.Close() - + file, err := os.OpenFile(m.WalFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { - return false, errors.New("Not able to create/open the wal file") + errMessage := fmt.Sprintf("Not able to create/open the wal file %s", m.WalFilePath) + return false, errors.New(errMessage); } + defer file.Close() + _, err = file.Write(record) if err != nil { @@ -102,14 +104,16 @@ func (m Memtable) writeToWal(record []byte) (bool, error) { } func (m *Memtable) MemtableStartUp() (bool, error) { - file, err := os.OpenFile(m.WalPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - + file, err := os.OpenFile(m.WalFilePath, os.O_APPEND|os.O_CREATE|os.O_RDONLY, 0644) + defer file.Close() if err != nil { return false, err } + + fileInfo, err := file.Stat() if err != nil { @@ -137,8 +141,11 @@ func (m *Memtable) MemtableStartUp() (bool, error) { break } - rec := append(header, keyValuePair...) - m.list.Insert(string(keyValuePair[:keySize]), rec) + rec := append(header, keyValuePair...) + m.list.Insert(string(rec[21: keySize + 21]), rec) + + m.size += 1 + i += int64(len(rec)) } return true, nil diff --git a/internals/memtable/memtable_test.go b/internals/memtable/memtable_test.go index be4ebe5..649c90c 100644 --- a/internals/memtable/memtable_test.go +++ b/internals/memtable/memtable_test.go @@ -4,7 +4,7 @@ import ( "os" "path/filepath" "testing" - + "fmt" "github.com/Vince-maple-byte/KeyData/internals/record" ) @@ -12,18 +12,18 @@ func TestWriteForSingleEntryInMemtable(t *testing.T) { baseDir := t.TempDir() - walDir := filepath.Join(baseDir, "dir1") + walDir := filepath.Join(baseDir, "wal") os.Mkdir(walDir, 0700) - - dataDir := filepath.Join(baseDir, "dir2") + walFile := filepath.Join(walDir, "mem1.wal") + dataDir := filepath.Join(baseDir, "data") os.Mkdir(dataDir, 0700) - mem := CreateMemtable(walDir, dataDir) + mem := CreateMemtable(walFile, dataDir) prevSize := mem.size _, err := mem.Write("key", "val", "PUT") if err != nil { - t.Fatalf("Was not able to process the delete operation for the memtable") + t.Fatalf("Was not able to process the PUT operation for the memtable\n %v", err) } if mem.size <= prevSize { @@ -36,10 +36,11 @@ func TestWriteForMultipleEntriesInMemtable(t *testing.T) { walDir := filepath.Join(baseDir, "dir1") os.Mkdir(walDir, 0700) + walFile := filepath.Join(walDir, "mem1.wal") dataDir := filepath.Join(baseDir, "dir2") os.Mkdir(dataDir, 0700) - mem := CreateMemtable(walDir, dataDir) + mem := CreateMemtable(walFile, dataDir) tests := []struct { key, value, op string @@ -56,11 +57,11 @@ func TestWriteForMultipleEntriesInMemtable(t *testing.T) { _, err := mem.Write(test.key, test.value, test.op) if err != nil { - t.Fatalf("Was not able to process the delete operation for the memtable") + t.Fatalf("Was not able to process the PUT operation for the memtable") } if mem.size <= prevSize { - t.Errorf("The delete operation did not successfully increase the memtable size") + t.Errorf("The PUT operation did not successfully increase the memtable size") } } @@ -71,10 +72,11 @@ func TestGetForMemtable(t *testing.T) { walDir := filepath.Join(baseDir, "dir1") os.Mkdir(walDir, 0700) + walFile := filepath.Join(walDir, "mem1.wal") dataDir := filepath.Join(baseDir, "dir2") os.Mkdir(dataDir, 0700) - mem := CreateMemtable(walDir, dataDir) + mem := CreateMemtable(walFile, dataDir) mem.Write("key", "val", "PUT") actual, err := mem.Get("key") @@ -93,10 +95,11 @@ func TestDeleteForMemtable(t *testing.T) { walDir := filepath.Join(baseDir, "dir1") os.Mkdir(walDir, 0700) + walFile := filepath.Join(walDir, "mem1.wal") dataDir := filepath.Join(baseDir, "dir2") os.Mkdir(dataDir, 0700) - mem := CreateMemtable(walDir, dataDir) + mem := CreateMemtable(walFile, dataDir) mem.Write("key", "val", "PUT") prevSize := mem.size @@ -113,16 +116,17 @@ func TestDeleteForMemtable(t *testing.T) { } func TestWrite_FlushResetsState(t *testing.T) { - t.Skip("requires sstable wiring and writable data dir — run as integration test") + // t.Skip("requires sstable wiring and writable data dir — run as integration test") baseDir := t.TempDir() walDir := filepath.Join(baseDir, "dir1") os.Mkdir(walDir, 0700) + walFile := filepath.Join(walDir, "mem1.wal") dataDir := filepath.Join(baseDir, "dir2") os.Mkdir(dataDir, 0700) - mt := CreateMemtable(walDir, dataDir) + mt:= CreateMemtable(walFile, dataDir) for i := 0; i < MAX_SIZE; i++ { key := "key" + string(rune(i)) @@ -136,3 +140,61 @@ func TestWrite_FlushResetsState(t *testing.T) { t.Errorf("expected size to reset to 0 after flush, got %d", mt.size) } } + +func TestMemtableStartUp(t *testing.T) { + baseDir := t.TempDir() + + walDir := filepath.Join(baseDir, "dir1") + os.Mkdir(walDir, 0700) + walFile := filepath.Join(walDir, "mem1.wal") + + dataDir := filepath.Join(baseDir, "dir2") + os.Mkdir(dataDir, 0700) + mt:= CreateMemtable(walFile, dataDir) + + ok,err := mt.Write("key", "val23", "PUT"); + + if !ok { + t.Fatalf("Not able to write into the memtable, %v", err); + } + + for i := range 10 { + mt.Write(fmt.Sprintf("key%d",i), fmt.Sprintf("val%d", i), "PUT") + } + + fileInfo,_ := os.Lstat(walFile) + + fmt.Println(fileInfo.Size()) + + mm := CreateMemtable(walFile, dataDir) + _,err = mm.MemtableStartUp() + + if err != nil { + t.Errorf("From trying to startup the memtable:%s\n", err.Error()) + } + + data, err := mm.Get("key") + + if err != nil { + t.Error(err.Error()) + } + + if len(data) < 21 { + t.Fatalf("Was Not able to retrieve the data from the wal file, %v", data) + } + + contents := record.GetContents(data) + + if contents.Payload != "val23" && contents.Key == "key" { + t.Errorf("Did not return the valid key/value pair\nKey:%s\nValue:%s", contents.Key, contents.Payload) + } + + for i := range 10 { + content := record.GetContents(data) + + if content.Payload != fmt.Sprintf("val%d", i) && contents.Key == fmt.Sprintf("key%d",i) { + t.Errorf("Did not return the valid key/value pair\nKey:%s\nValue:%s", contents.Key, contents.Payload) + } + + } +} diff --git a/internals/sstable/writer.go b/internals/sstable/writer.go index 8ebad9b..49e622f 100644 --- a/internals/sstable/writer.go +++ b/internals/sstable/writer.go @@ -50,13 +50,14 @@ func WriteToFile(list [][]byte, filePath string) (bool, error) { } file, err := os.Create(filepath.Join(filePath, filename)) - defer file.Close() + if err != nil { return false, err } - //contents := list + defer file.Close() + offset := fileOffset(list) index := createIndexBlock(list, offset) footer, err := createFooter(list) diff --git a/internals/sstable/writer_test.go b/internals/sstable/writer_test.go index f4b01b4..e0433bc 100644 --- a/internals/sstable/writer_test.go +++ b/internals/sstable/writer_test.go @@ -235,3 +235,7 @@ func TestFooter(t *testing.T) { } } + +func TestCorrectKeyValue(t *testing.T) { + +}