riyyi/gram
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add more tests and some impl improvements from their results

Browse Source
This commit is contained in:
Riyyi
2026-08-21 20:43:46 +02:00
parent ba9c95a09f
commit 6e81c87501
7 changed files with 412 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/chunks/read_assets.odin
+79 -52
View File
@@ -7,7 +7,7 @@ import "core:strings"
// -----------------------------------------
file_exists :: proc(
asset_exists :: proc(
r: ^Reader,
path: string,
allocator := context.allocator,
@@ -17,6 +17,13 @@ file_exists :: proc(
error: Error,
) {
clean_path := os.clean_path(path, allocator) or_return
// Check filesystem first
if os.exists(clean_path) {
return true, max(u32), nil
}
// Check chunk
for i in 0 ..< r.header.number_of_assets {
str := string(r.indices[i].relpath[:])
relpath := strings.trim_right_null(str)
@@ -27,7 +34,7 @@ file_exists :: proc(
}
// Returning `[]u8` is owned by the caller.
read_file :: proc(
read_asset :: proc(
r: ^Reader,
path: string,
allocator := context.allocator,
@@ -37,8 +44,15 @@ read_file :: proc(
) {
clean_path := os.clean_path(path, allocator) or_return
found, i := asset_exists(r, clean_path) or_return
if !found {
fmt.eprintln("asset: ", clean_path)
return nil, .Asset_Not_Exist
}
// Check filesystem first, so chunks can be overriden
if os.exists(clean_path) {
if i == max(u32) {
// Open file
file := os.open(clean_path, {.Read}) or_return
defer os.close(file)
@@ -51,56 +65,69 @@ read_file :: proc(
return bytes, nil
}
// Check chunk
if found, i := file_exists(r, clean_path) or_return; found {
output_path := os.name(r.output)
entry := r.indices[i]
// Otherwise, read from chunk
return read_asset_from_chunk(r, i, allocator)
}
// Find correct chunk
chunk_index := entry.offset / r.header.size_per_chunk
// Find offset within this chunk
chunk_offset := entry.offset % r.header.size_per_chunk
bytes = make([]u8, entry.size, allocator)
n_left: u64
n_read: u64
for entry.size > n_read {
// Chunk path
chunk_path := compute_chunk_path(
output_path,
cast(int)chunk_index,
allocator,
) or_return
defer delete(chunk_path, allocator)
// Open chunk
file := os.open(chunk_path, {.Read}) or_return
defer os.close(file)
// Calculate how much of the current chunk should be read
if entry.size > n_read + r.header.size_per_chunk - chunk_offset {
n_left = r.header.size_per_chunk - chunk_offset
} else {
n_left = entry.size - n_read
}
// Read
n_read += cast(u64)os.read_at(
file,
bytes[n_read:n_read + n_left],
cast(i64)chunk_offset,
) or_return
// Overflow to next chunk
chunk_index += 1
chunk_offset = 0
}
return bytes, nil
// Returning `[]u8` is owned by the caller.
read_asset_from_chunk :: proc(
r: ^Reader,
index: u32,
allocator: runtime.Allocator,
) -> (
bytes: []u8,
err: Error,
) {
if cast(u32)len(r.indices) < index {
fmt.eprintf("asset[{}]\n", index)
return nil, .Asset_Index_Out_Of_Bounds
}
fmt.eprintln("asset: ", clean_path)
return nil, .Asset_Not_Exist
// Check chunk
output_path := os.name(r.output)
entry := r.indices[index]
// Find correct chunk
chunk_index := entry.offset / r.header.size_per_chunk
// Find offset within this chunk
chunk_offset := entry.offset % r.header.size_per_chunk
bytes = make([]u8, entry.size, allocator)
n_left: u64
n_read: u64
for entry.size > n_read {
// Chunk path
chunk_path := compute_chunk_path(
output_path,
cast(int)chunk_index,
allocator,
) or_return
defer delete(chunk_path, allocator)
// Open chunk
file := os.open(chunk_path, {.Read}) or_return
defer os.close(file)
// Calculate how much of the current chunk should be read
if entry.size > n_read + r.header.size_per_chunk - chunk_offset {
n_left = r.header.size_per_chunk - chunk_offset
} else {
n_left = entry.size - n_read
}
// Read
n_read += cast(u64)os.read_at(
file,
bytes[n_read:n_read + n_left],
cast(i64)chunk_offset,
) or_return
// Overflow to next chunk
chunk_index += 1
chunk_offset = 0
}
return bytes, nil
}
src/chunks/reader.odin
+11 -9
View File
@@ -12,20 +12,22 @@ Reader :: struct {
}
Read_Error :: enum u8 {
None = 0,
File_Not_A_Chunk = 1,
Asset_Not_Exist = 2,
Unimplemented = 127,
Okay = None,
None = 0,
File_Not_A_Chunk = 1,
Asset_Not_Exist = 2,
Asset_Index_Out_Of_Bounds = 3,
Unimplemented = 127,
Okay = None,
}
// -----------------------------------------
read_error_strings := #sparse[Read_Error]string { // enumerated array
.None = "",
.File_Not_A_Chunk = "file is not a chunk file",
.Asset_Not_Exist = "asset does not exist",
.Unimplemented = "feature is unimplemented",
.None = "",
.File_Not_A_Chunk = "file is not a chunk file",
.Asset_Not_Exist = "asset does not exist",
.Asset_Index_Out_Of_Bounds = "asset index out of bounds",
.Unimplemented = "feature is unimplemented",
}
// -----------------------------------------
src/chunks/write_assets.odin
+5
View File
@@ -62,6 +62,11 @@ write_assets :: proc(
// ----------------------------------------
// We cant open non-existing assets
if number_of_assets == 0 {
return nil
}
// Open asset
asset_stream, asset_size := open_asset(
w,
src/chunks/write_metadata.odin
+10 -4
View File
@@ -42,10 +42,16 @@ write_header :: proc(
size_per_chunk: u64 = SIZE_PER_CHUNK,
compression: u16 = COMPRESSION,
) -> Error {
last_offset :=
w.asset_table.asset_offsets[len(w.asset_table.asset_offsets) - 1]
last_size := w.asset_table.asset_sizes[len(w.asset_table.asset_sizes) - 1]
total_size := last_offset + last_size
// Get total_size
total_size: u64 = 0
offset_len := len(w.asset_table.asset_offsets)
size_len := len(w.asset_table.asset_sizes)
if offset_len > 0 && size_len > 0 {
last_offset := w.asset_table.asset_offsets[offset_len - 1]
last_size := w.asset_table.asset_sizes[size_len - 1]
total_size = last_offset + last_size
}
header := Header {
magic_string = MAGIC_STRING,
test.sh
+1
View File
@@ -3,6 +3,7 @@
mkdir -p build
odin test tests/ -collection:src=src -out:build/tests.bin \
-define:ODIN_TEST_CLIPBOARD=true \
-define:ODIN_TEST_THREADS=0 \
-define:ODIN_TEST_TRACK_MEMORY=false \
-define:ODIN_TEST_RANDOM_SEED=1 \
tests/helpers.odin
+11 -6
View File
@@ -131,7 +131,10 @@ read :: proc(
t: ^testing.T,
path: string,
allocator := context.allocator,
) -> string {
) -> (
bool,
string,
) {
relpath, err := os.join_path({ts.input_dir, path}, allocator)
testing.expect(t, err == nil, os.error_string(err))
defer delete(relpath, allocator)
@@ -144,12 +147,14 @@ read :: proc(
testing.expect(t, err3 == nil, chunks.format_error(err3))
defer chunks.reader_destroy(&r)
exists, i, err4 := chunks.file_exists(&r, relpath)
testing.expect(t, exists)
exists, i, err4 := chunks.asset_exists(&r, relpath)
testing.expect(t, err4 == nil, chunks.format_error(err4))
bytes, err5 := chunks.read_file(&r, relpath)
testing.expect(t, err5 == nil, chunks.format_error(err5))
if exists {
bytes, err5 := chunks.read_asset(&r, relpath)
testing.expect(t, err5 == nil, chunks.format_error(err5))
return true, string(bytes)
}
return string(bytes)
return false, {}
}
tests/tests.odin
+295 -51
View File
@@ -1,30 +1,10 @@
package tests
import "core:strings"
import "core:testing"
@(test)
single_tiny_file_read_from_chunk :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "single_tiny_file", "hello world!")
pack(&ts, t, 750)
delete_path(&ts, t, "")
// Act
read := read(&ts, t, "single_tiny_file")
// Assert
testing.expect_value(t, read, "hello world!")
}
@(test)
single_file_single_chunk_read_from_path :: proc(t: ^testing.T) {
single_asset_single_chunk_read_from_path :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
@@ -38,7 +18,7 @@ single_file_single_chunk_read_from_path :: proc(t: ^testing.T) {
// Act
read := read(&ts, t, "single_tiny_file")
_, read := read(&ts, t, "single_tiny_file")
// Assert
@@ -46,34 +26,53 @@ single_file_single_chunk_read_from_path :: proc(t: ^testing.T) {
}
@(test)
multiple_files_single_chunk_read_from_chunk :: proc(t: ^testing.T) {
single_asset_single_chunk_read_from_chunk :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "a", "hello a!")
create_file(&ts, t, "b", "hello b!")
create_file(&ts, t, "c", "hello c!")
pack(&ts, t, 1750)
create_file(&ts, t, "single_tiny_file", "hello world!")
pack(&ts, t, 750)
delete_path(&ts, t, "")
// Act
a := read(&ts, t, "a")
b := read(&ts, t, "b")
c := read(&ts, t, "c")
_, read := read(&ts, t, "single_tiny_file")
// Assert
testing.expect_value(t, a, "hello a!")
testing.expect_value(t, b, "hello b!")
testing.expect_value(t, c, "hello c!")
testing.expect_value(t, read, "hello world!")
}
@(test)
multiple_files_single_chunk_read_from_path :: proc(t: ^testing.T) {
single_asset_multiple_chunks_read_from_chunk :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
content := strings.repeat("abcde", 400)
defer delete(content)
create_file(&ts, t, "multi_chunk_file", content)
pack(&ts, t, 750)
delete_path(&ts, t, "")
// Act
_, read := read(&ts, t, "multi_chunk_file")
// Assert
testing.expect_value(t, read, content)
}
// -----------------------------------------
@(test)
multiple_assets_single_chunk_read_from_path :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
@@ -91,9 +90,9 @@ multiple_files_single_chunk_read_from_path :: proc(t: ^testing.T) {
// Act
a := read(&ts, t, "a")
b := read(&ts, t, "b")
c := read(&ts, t, "c")
_, a := read(&ts, t, "a")
_, b := read(&ts, t, "b")
_, c := read(&ts, t, "c")
// Assert
@@ -102,17 +101,262 @@ multiple_files_single_chunk_read_from_path :: proc(t: ^testing.T) {
testing.expect_value(t, c, "hello c from path!")
}
// @(test)
// blabla :: proc(t: ^testing.T) {
// // Arrange
//
// // Act
//
// // Assert
// }
@(test)
multiple_assets_single_chunk_read_from_chunk :: proc(t: ^testing.T) {
// Arrange
// TODO:
// v multiple files (path and chunk find)
// - nested file (path and chunk find)
// - not found in path
// - not found in path + chunk
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "a", "hello a!")
create_file(&ts, t, "b", "hello b!")
create_file(&ts, t, "c", "hello c!")
pack(&ts, t, 1750)
delete_path(&ts, t, "")
// Act
_, a := read(&ts, t, "a")
_, b := read(&ts, t, "b")
_, c := read(&ts, t, "c")
// Assert
testing.expect_value(t, a, "hello a!")
testing.expect_value(t, b, "hello b!")
testing.expect_value(t, c, "hello c!")
}
@(test)
multiple_assets_multiple_chunks_read_from_chunk :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
content_a := strings.repeat("a", 700)
defer delete(content_a)
content_b := strings.repeat("b", 700)
defer delete(content_b)
content_c := strings.repeat("c", 700)
defer delete(content_c)
create_file(&ts, t, "a", content_a)
create_file(&ts, t, "b", content_b)
create_file(&ts, t, "c", content_c)
pack(&ts, t, 1750)
delete_path(&ts, t, "")
// Act
_, a := read(&ts, t, "a")
_, b := read(&ts, t, "b")
_, c := read(&ts, t, "c")
// Assert
testing.expect_value(t, a, content_a)
testing.expect_value(t, b, content_b)
testing.expect_value(t, c, content_c)
}
// -----------------------------------------
@(test)
single_asset_single_chunk_nested_path_read_from_path :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "subdir/nested_file", "hello nested!")
pack(&ts, t, 750)
delete_path(&ts, t, "")
create_file(&ts, t, "subdir/nested_file", "hello nested from path!")
// Act
_, read := read(&ts, t, "subdir/nested_file")
// Assert
testing.expect_value(t, read, "hello nested from path!")
}
@(test)
single_asset_single_chunk_nested_path_read_from_chunk :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "subdir/nested_file", "hello nested!")
pack(&ts, t, 750)
delete_path(&ts, t, "")
// Act
_, read := read(&ts, t, "subdir/nested_file")
// Assert
testing.expect_value(t, read, "hello nested!")
}
@(test)
single_asset_multiple_chunks_nested_path_read_from_chunk :: proc(
t: ^testing.T,
) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
content := strings.repeat("abcde", 50)
defer delete(content)
create_file(&ts, t, "subdir/nested_file", content)
pack(&ts, t, 750)
delete_path(&ts, t, "")
// Act
_, read := read(&ts, t, "subdir/nested_file")
// Assert
testing.expect_value(t, read, content)
}
// -----------------------------------------
@(test)
asset_in_path_not_in_chunk_read_matching_file :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "chunk", "hello from chunk!")
pack(&ts, t, 750)
delete_path(&ts, t, "")
create_file(&ts, t, "target", "hello from path!")
// Act
_, read := read(&ts, t, "target")
// Assert
testing.expect_value(t, read, "hello from path!")
}
@(test)
asset_in_chunk_not_in_path_read_matching_file :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "target", "hello from chunk!")
pack(&ts, t, 750)
delete_path(&ts, t, "")
// Act
_, read := read(&ts, t, "target")
// Assert
testing.expect_value(t, read, "hello from chunk!")
}
@(test)
asset_in_chunk_and_in_path_read_matching_file :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "target", "hello from chunk!")
pack(&ts, t, 750)
delete_path(&ts, t, "")
create_file(&ts, t, "target", "hello from path!")
// Act
_, read := read(&ts, t, "target")
// Assert
testing.expect_value(t, read, "hello from path!")
}
@(test)
asset_in_path_not_in_chunk_read_non_matching_file :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
pack(&ts, t, 750)
delete_path(&ts, t, "")
create_file(&ts, t, "target", "hello from path!")
// Act
exists, _ := read(&ts, t, "non_matching")
// Assert
testing.expect(t, !exists)
}
@(test)
asset_in_chunk_not_in_path_read_non_matching_file :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "target", "hello from chunk!")
pack(&ts, t, 750)
delete_path(&ts, t, "")
// Act
exists, _ := read(&ts, t, "non_matching")
// Assert
testing.expect(t, !exists)
}
@(test)
asset_in_chunk_and_in_path_read_non_matching_file :: proc(t: ^testing.T) {
// Arrange
ts := test_state_init(t, #procedure)
defer test_state_destroy(&ts, t)
delete_path(&ts, t, "")
create_file(&ts, t, "target", "hello from chunk!")
pack(&ts, t, 750)
delete_path(&ts, t, "")
create_file(&ts, t, "target", "hello from path!")
// Act
exists, _ := read(&ts, t, "non_matching")
// Assert
testing.expect(t, !exists)
}