diff --git a/src/chunks/read_assets.odin b/src/chunks/read_assets.odin index 186729f..63a42f2 100644 --- a/src/chunks/read_assets.odin +++ b/src/chunks/read_assets.odin @@ -7,7 +7,7 @@ import "core:strings" // ----------------------------------------- -file_exists :: proc( +asset_exists :: proc( r: ^Reader, path: string, allocator := context.allocator, @@ -17,6 +17,13 @@ file_exists :: proc( error: Error, ) { clean_path := os.clean_path(path, allocator) or_return + + // Check filesystem first + if os.exists(clean_path) { + return true, max(u32), nil + } + + // Check chunk for i in 0 ..< r.header.number_of_assets { str := string(r.indices[i].relpath[:]) relpath := strings.trim_right_null(str) @@ -27,7 +34,7 @@ file_exists :: proc( } // Returning `[]u8` is owned by the caller. -read_file :: proc( +read_asset :: proc( r: ^Reader, path: string, allocator := context.allocator, @@ -37,8 +44,15 @@ read_file :: proc( ) { clean_path := os.clean_path(path, allocator) or_return + found, i := asset_exists(r, clean_path) or_return + + if !found { + fmt.eprintln("asset: ", clean_path) + return nil, .Asset_Not_Exist + } + // Check filesystem first, so chunks can be overriden - if os.exists(clean_path) { + if i == max(u32) { // Open file file := os.open(clean_path, {.Read}) or_return defer os.close(file) @@ -51,56 +65,69 @@ read_file :: proc( return bytes, nil } - // Check chunk - if found, i := file_exists(r, clean_path) or_return; found { - output_path := os.name(r.output) - entry := r.indices[i] + // Otherwise, read from chunk + return read_asset_from_chunk(r, i, allocator) +} - // Find correct chunk - chunk_index := entry.offset / r.header.size_per_chunk - - // Find offset within this chunk - chunk_offset := entry.offset % r.header.size_per_chunk - - bytes = make([]u8, entry.size, allocator) - n_left: u64 - n_read: u64 - - for entry.size > n_read { - // Chunk path - chunk_path := compute_chunk_path( - output_path, - cast(int)chunk_index, - allocator, - ) or_return - defer delete(chunk_path, allocator) - - // Open chunk - file := os.open(chunk_path, {.Read}) or_return - defer os.close(file) - - // Calculate how much of the current chunk should be read - if entry.size > n_read + r.header.size_per_chunk - chunk_offset { - n_left = r.header.size_per_chunk - chunk_offset - } else { - n_left = entry.size - n_read - } - - // Read - n_read += cast(u64)os.read_at( - file, - bytes[n_read:n_read + n_left], - cast(i64)chunk_offset, - ) or_return - - // Overflow to next chunk - chunk_index += 1 - chunk_offset = 0 - } - - return bytes, nil +// Returning `[]u8` is owned by the caller. +read_asset_from_chunk :: proc( + r: ^Reader, + index: u32, + allocator: runtime.Allocator, +) -> ( + bytes: []u8, + err: Error, +) { + if cast(u32)len(r.indices) < index { + fmt.eprintf("asset[{}]\n", index) + return nil, .Asset_Index_Out_Of_Bounds } - fmt.eprintln("asset: ", clean_path) - return nil, .Asset_Not_Exist + // Check chunk + output_path := os.name(r.output) + entry := r.indices[index] + + // Find correct chunk + chunk_index := entry.offset / r.header.size_per_chunk + + // Find offset within this chunk + chunk_offset := entry.offset % r.header.size_per_chunk + + bytes = make([]u8, entry.size, allocator) + n_left: u64 + n_read: u64 + + for entry.size > n_read { + // Chunk path + chunk_path := compute_chunk_path( + output_path, + cast(int)chunk_index, + allocator, + ) or_return + defer delete(chunk_path, allocator) + + // Open chunk + file := os.open(chunk_path, {.Read}) or_return + defer os.close(file) + + // Calculate how much of the current chunk should be read + if entry.size > n_read + r.header.size_per_chunk - chunk_offset { + n_left = r.header.size_per_chunk - chunk_offset + } else { + n_left = entry.size - n_read + } + + // Read + n_read += cast(u64)os.read_at( + file, + bytes[n_read:n_read + n_left], + cast(i64)chunk_offset, + ) or_return + + // Overflow to next chunk + chunk_index += 1 + chunk_offset = 0 + } + + return bytes, nil } diff --git a/src/chunks/reader.odin b/src/chunks/reader.odin index 7d40851..c1a496e 100644 --- a/src/chunks/reader.odin +++ b/src/chunks/reader.odin @@ -12,20 +12,22 @@ Reader :: struct { } Read_Error :: enum u8 { - None = 0, - File_Not_A_Chunk = 1, - Asset_Not_Exist = 2, - Unimplemented = 127, - Okay = None, + None = 0, + File_Not_A_Chunk = 1, + Asset_Not_Exist = 2, + Asset_Index_Out_Of_Bounds = 3, + Unimplemented = 127, + Okay = None, } // ----------------------------------------- read_error_strings := #sparse[Read_Error]string { // enumerated array - .None = "", - .File_Not_A_Chunk = "file is not a chunk file", - .Asset_Not_Exist = "asset does not exist", - .Unimplemented = "feature is unimplemented", + .None = "", + .File_Not_A_Chunk = "file is not a chunk file", + .Asset_Not_Exist = "asset does not exist", + .Asset_Index_Out_Of_Bounds = "asset index out of bounds", + .Unimplemented = "feature is unimplemented", } // ----------------------------------------- diff --git a/src/chunks/write_assets.odin b/src/chunks/write_assets.odin index bc1102c..87d7052 100644 --- a/src/chunks/write_assets.odin +++ b/src/chunks/write_assets.odin @@ -62,6 +62,11 @@ write_assets :: proc( // ---------------------------------------- + // We cant open non-existing assets + if number_of_assets == 0 { + return nil + } + // Open asset asset_stream, asset_size := open_asset( w, diff --git a/src/chunks/write_metadata.odin b/src/chunks/write_metadata.odin index 5a2bd82..fd8410b 100644 --- a/src/chunks/write_metadata.odin +++ b/src/chunks/write_metadata.odin @@ -42,10 +42,16 @@ write_header :: proc( size_per_chunk: u64 = SIZE_PER_CHUNK, compression: u16 = COMPRESSION, ) -> Error { - last_offset := - w.asset_table.asset_offsets[len(w.asset_table.asset_offsets) - 1] - last_size := w.asset_table.asset_sizes[len(w.asset_table.asset_sizes) - 1] - total_size := last_offset + last_size + + // Get total_size + total_size: u64 = 0 + offset_len := len(w.asset_table.asset_offsets) + size_len := len(w.asset_table.asset_sizes) + if offset_len > 0 && size_len > 0 { + last_offset := w.asset_table.asset_offsets[offset_len - 1] + last_size := w.asset_table.asset_sizes[size_len - 1] + total_size = last_offset + last_size + } header := Header { magic_string = MAGIC_STRING, diff --git a/test.sh b/test.sh index 2cfadd0..96ee415 100755 --- a/test.sh +++ b/test.sh @@ -3,6 +3,7 @@ mkdir -p build odin test tests/ -collection:src=src -out:build/tests.bin \ + -define:ODIN_TEST_CLIPBOARD=true \ -define:ODIN_TEST_THREADS=0 \ -define:ODIN_TEST_TRACK_MEMORY=false \ -define:ODIN_TEST_RANDOM_SEED=1 \ diff --git a/tests/helpers.odin b/tests/helpers.odin index fd04b8f..c766461 100644 --- a/tests/helpers.odin +++ b/tests/helpers.odin @@ -131,7 +131,10 @@ read :: proc( t: ^testing.T, path: string, allocator := context.allocator, -) -> string { +) -> ( + bool, + string, +) { relpath, err := os.join_path({ts.input_dir, path}, allocator) testing.expect(t, err == nil, os.error_string(err)) defer delete(relpath, allocator) @@ -144,12 +147,14 @@ read :: proc( testing.expect(t, err3 == nil, chunks.format_error(err3)) defer chunks.reader_destroy(&r) - exists, i, err4 := chunks.file_exists(&r, relpath) - testing.expect(t, exists) + exists, i, err4 := chunks.asset_exists(&r, relpath) testing.expect(t, err4 == nil, chunks.format_error(err4)) - bytes, err5 := chunks.read_file(&r, relpath) - testing.expect(t, err5 == nil, chunks.format_error(err5)) + if exists { + bytes, err5 := chunks.read_asset(&r, relpath) + testing.expect(t, err5 == nil, chunks.format_error(err5)) + return true, string(bytes) + } - return string(bytes) + return false, {} } diff --git a/tests/tests.odin b/tests/tests.odin index 85e09bc..2893d7e 100644 --- a/tests/tests.odin +++ b/tests/tests.odin @@ -1,30 +1,10 @@ package tests +import "core:strings" import "core:testing" @(test) -single_tiny_file_read_from_chunk :: proc(t: ^testing.T) { - // Arrange - - ts := test_state_init(t, #procedure) - defer test_state_destroy(&ts, t) - - delete_path(&ts, t, "") - create_file(&ts, t, "single_tiny_file", "hello world!") - pack(&ts, t, 750) - delete_path(&ts, t, "") - - // Act - - read := read(&ts, t, "single_tiny_file") - - // Assert - - testing.expect_value(t, read, "hello world!") -} - -@(test) -single_file_single_chunk_read_from_path :: proc(t: ^testing.T) { +single_asset_single_chunk_read_from_path :: proc(t: ^testing.T) { // Arrange ts := test_state_init(t, #procedure) @@ -38,7 +18,7 @@ single_file_single_chunk_read_from_path :: proc(t: ^testing.T) { // Act - read := read(&ts, t, "single_tiny_file") + _, read := read(&ts, t, "single_tiny_file") // Assert @@ -46,34 +26,53 @@ single_file_single_chunk_read_from_path :: proc(t: ^testing.T) { } @(test) -multiple_files_single_chunk_read_from_chunk :: proc(t: ^testing.T) { +single_asset_single_chunk_read_from_chunk :: proc(t: ^testing.T) { // Arrange ts := test_state_init(t, #procedure) defer test_state_destroy(&ts, t) delete_path(&ts, t, "") - create_file(&ts, t, "a", "hello a!") - create_file(&ts, t, "b", "hello b!") - create_file(&ts, t, "c", "hello c!") - pack(&ts, t, 1750) + create_file(&ts, t, "single_tiny_file", "hello world!") + pack(&ts, t, 750) delete_path(&ts, t, "") // Act - a := read(&ts, t, "a") - b := read(&ts, t, "b") - c := read(&ts, t, "c") + _, read := read(&ts, t, "single_tiny_file") // Assert - testing.expect_value(t, a, "hello a!") - testing.expect_value(t, b, "hello b!") - testing.expect_value(t, c, "hello c!") + testing.expect_value(t, read, "hello world!") } @(test) -multiple_files_single_chunk_read_from_path :: proc(t: ^testing.T) { +single_asset_multiple_chunks_read_from_chunk :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + content := strings.repeat("abcde", 400) + defer delete(content) + create_file(&ts, t, "multi_chunk_file", content) + pack(&ts, t, 750) + delete_path(&ts, t, "") + + // Act + + _, read := read(&ts, t, "multi_chunk_file") + + // Assert + + testing.expect_value(t, read, content) +} + +// ----------------------------------------- + +@(test) +multiple_assets_single_chunk_read_from_path :: proc(t: ^testing.T) { // Arrange ts := test_state_init(t, #procedure) @@ -91,9 +90,9 @@ multiple_files_single_chunk_read_from_path :: proc(t: ^testing.T) { // Act - a := read(&ts, t, "a") - b := read(&ts, t, "b") - c := read(&ts, t, "c") + _, a := read(&ts, t, "a") + _, b := read(&ts, t, "b") + _, c := read(&ts, t, "c") // Assert @@ -102,17 +101,262 @@ multiple_files_single_chunk_read_from_path :: proc(t: ^testing.T) { testing.expect_value(t, c, "hello c from path!") } -// @(test) -// blabla :: proc(t: ^testing.T) { -// // Arrange -// -// // Act -// -// // Assert -// } +@(test) +multiple_assets_single_chunk_read_from_chunk :: proc(t: ^testing.T) { + // Arrange -// TODO: -// v multiple files (path and chunk find) -// - nested file (path and chunk find) -// - not found in path -// - not found in path + chunk + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + create_file(&ts, t, "a", "hello a!") + create_file(&ts, t, "b", "hello b!") + create_file(&ts, t, "c", "hello c!") + pack(&ts, t, 1750) + delete_path(&ts, t, "") + + // Act + + _, a := read(&ts, t, "a") + _, b := read(&ts, t, "b") + _, c := read(&ts, t, "c") + + // Assert + + testing.expect_value(t, a, "hello a!") + testing.expect_value(t, b, "hello b!") + testing.expect_value(t, c, "hello c!") +} + +@(test) +multiple_assets_multiple_chunks_read_from_chunk :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + content_a := strings.repeat("a", 700) + defer delete(content_a) + content_b := strings.repeat("b", 700) + defer delete(content_b) + content_c := strings.repeat("c", 700) + defer delete(content_c) + create_file(&ts, t, "a", content_a) + create_file(&ts, t, "b", content_b) + create_file(&ts, t, "c", content_c) + pack(&ts, t, 1750) + delete_path(&ts, t, "") + + // Act + + _, a := read(&ts, t, "a") + _, b := read(&ts, t, "b") + _, c := read(&ts, t, "c") + + // Assert + + testing.expect_value(t, a, content_a) + testing.expect_value(t, b, content_b) + testing.expect_value(t, c, content_c) +} + +// ----------------------------------------- + +@(test) +single_asset_single_chunk_nested_path_read_from_path :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + create_file(&ts, t, "subdir/nested_file", "hello nested!") + pack(&ts, t, 750) + delete_path(&ts, t, "") + create_file(&ts, t, "subdir/nested_file", "hello nested from path!") + + // Act + + _, read := read(&ts, t, "subdir/nested_file") + + // Assert + + testing.expect_value(t, read, "hello nested from path!") +} + +@(test) +single_asset_single_chunk_nested_path_read_from_chunk :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + create_file(&ts, t, "subdir/nested_file", "hello nested!") + pack(&ts, t, 750) + delete_path(&ts, t, "") + + // Act + + _, read := read(&ts, t, "subdir/nested_file") + + // Assert + + testing.expect_value(t, read, "hello nested!") +} + +@(test) +single_asset_multiple_chunks_nested_path_read_from_chunk :: proc( + t: ^testing.T, +) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + content := strings.repeat("abcde", 50) + defer delete(content) + create_file(&ts, t, "subdir/nested_file", content) + pack(&ts, t, 750) + delete_path(&ts, t, "") + + // Act + + _, read := read(&ts, t, "subdir/nested_file") + + // Assert + + testing.expect_value(t, read, content) +} + +// ----------------------------------------- + +@(test) +asset_in_path_not_in_chunk_read_matching_file :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + create_file(&ts, t, "chunk", "hello from chunk!") + pack(&ts, t, 750) + delete_path(&ts, t, "") + create_file(&ts, t, "target", "hello from path!") + + // Act + + _, read := read(&ts, t, "target") + + // Assert + + testing.expect_value(t, read, "hello from path!") +} + +@(test) +asset_in_chunk_not_in_path_read_matching_file :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + create_file(&ts, t, "target", "hello from chunk!") + pack(&ts, t, 750) + delete_path(&ts, t, "") + + // Act + + _, read := read(&ts, t, "target") + + // Assert + + testing.expect_value(t, read, "hello from chunk!") +} + +@(test) +asset_in_chunk_and_in_path_read_matching_file :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + create_file(&ts, t, "target", "hello from chunk!") + pack(&ts, t, 750) + delete_path(&ts, t, "") + create_file(&ts, t, "target", "hello from path!") + + // Act + + _, read := read(&ts, t, "target") + + // Assert + + testing.expect_value(t, read, "hello from path!") +} + +@(test) +asset_in_path_not_in_chunk_read_non_matching_file :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + pack(&ts, t, 750) + delete_path(&ts, t, "") + create_file(&ts, t, "target", "hello from path!") + + // Act + + exists, _ := read(&ts, t, "non_matching") + + // Assert + + testing.expect(t, !exists) +} + +@(test) +asset_in_chunk_not_in_path_read_non_matching_file :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + create_file(&ts, t, "target", "hello from chunk!") + pack(&ts, t, 750) + delete_path(&ts, t, "") + + // Act + + exists, _ := read(&ts, t, "non_matching") + + // Assert + + testing.expect(t, !exists) +} + +@(test) +asset_in_chunk_and_in_path_read_non_matching_file :: proc(t: ^testing.T) { + // Arrange + + ts := test_state_init(t, #procedure) + defer test_state_destroy(&ts, t) + + delete_path(&ts, t, "") + create_file(&ts, t, "target", "hello from chunk!") + pack(&ts, t, 750) + delete_path(&ts, t, "") + create_file(&ts, t, "target", "hello from path!") + + // Act + + exists, _ := read(&ts, t, "non_matching") + + // Assert + + testing.expect(t, !exists) +}