diff --git a/src/chunks/chunks.odin b/src/chunks/chunks.odin index 6bce1d6..f26aeb5 100644 --- a/src/chunks/chunks.odin +++ b/src/chunks/chunks.odin @@ -12,6 +12,7 @@ VERSION :: 1 COMPRESSION :: 0 SIZE_PER_CHUNK :: 2 * (1 << 30) // 2GiB PATH_SIZE :: 512 +CHUNK :: "CHUNK" // ----------------------------------------- @@ -44,6 +45,7 @@ Writer :: struct { } Error :: union #shared_nil { + runtime.Allocator_Error, io.Error, os.Error, Read_Error, @@ -86,7 +88,7 @@ compute_chunk_path :: proc( @(static) buf: [20]u8 chunk_index_str := strconv.write_int(buf[:], cast(i64)chunk_index, 10) - chunk_name := strings.concatenate({"CHUNK", chunk_index_str}, allocator) + chunk_name := strings.concatenate({CHUNK, chunk_index_str}, allocator) defer delete(chunk_name, allocator) chunk_path, c_err := os.join_path({output_path, chunk_name}, allocator) diff --git a/src/chunks/read_metadata.odin b/src/chunks/read_metadata.odin index 3033df1..e608a04 100644 --- a/src/chunks/read_metadata.odin +++ b/src/chunks/read_metadata.odin @@ -1,9 +1,11 @@ package chunks import "base:runtime" +import "core:fmt" import "core:io" import "core:mem" import "core:os" +import "core:strings" // ----------------------------------------- @@ -12,16 +14,13 @@ Reader :: struct {} Read_Error :: enum u8 { None = 0, File_Not_A_Chunk = 1, + File_Not_Exist = 2, Unimplemented = 127, Okay = None, } // ----------------------------------------- -// read header -// file_exists() func -// read_file() -> check chunks -> check fs -> fail - read_header :: proc( chunk1: ^os.File, allocator := context.allocator, @@ -40,6 +39,7 @@ read_header_by_path :: proc( err: Error, ) { chunk1_file := os.open(chunk1, {.Read}) or_return + defer os.close(chunk1_file) return read_header_impl(chunk1_file, allocator) } @@ -67,3 +67,118 @@ read_header_impl :: proc( return header, nil } + +// Returning `[]Asset_Index` is owned by the caller. +read_asset_index :: proc( + chunk1: ^os.File, + header: Header, + allocator := context.allocator, +) -> ( + indices: []Asset_Index, + err: Error, +) { + chunk1_stream := os.to_stream(chunk1) + + indices = make([]Asset_Index, header.number_of_assets, allocator) or_return + index_bytes := mem.slice_to_bytes(indices) + io.read_at(chunk1_stream, index_bytes, size_of(Header)) or_return + + return indices, nil +} + +file_exists :: proc( + header: Header, + indices: []Asset_Index, + path: string, +) -> ( + bool, + u32, +) { + for i in 0 ..< header.number_of_assets { + str := string(indices[i].relpath[:]) + relpath := strings.trim_right_null(str) + if path == relpath do return true, i + } + + return false, 0 +} + +// Returning `[]u8` is owned by the caller. +read_file :: proc( + header: Header, + indices: []Asset_Index, + output: ^os.File, + path: string, + allocator := context.allocator, +) -> ( + bytes: []u8, + err: Error, +) { + + // Check filesystem first, so chunks can be overriden + if os.exists(path) { + // Open file + fmt.println("searching:", path) + file := os.open(path, {.Read}) or_return + defer os.close(file) + + // Read + file_size := os.file_size(file) or_return + bytes = make([]u8, file_size, allocator) + os.read(file, bytes) or_return + + return bytes, nil + } + + // Check chunk + if found, i := file_exists(header, indices, path); found { + output_path := os.name(output) + entry := indices[i] + + // Find correct chunk + chunk_index := entry.offset / header.size_per_chunk + + // Find offset within this chunk + chunk_offset := entry.offset % header.size_per_chunk + + bytes = make([]u8, entry.size, allocator) + n_left: u64 + n_read: u64 + + for entry.size > n_read { + // Chunk path + chunk_path := compute_chunk_path( + output_path, + cast(int)chunk_index, + allocator, + ) + defer delete(chunk_path, allocator) + + // Open chunk + file := os.open(chunk_path, {.Read}) or_return + defer os.close(file) + + // Calculate how much of the current chunk should be read + if entry.size > n_read + header.size_per_chunk - chunk_offset { + n_left = header.size_per_chunk - chunk_offset + } else { + n_left = entry.size - n_read + } + + // Read + n_read += cast(u64)os.read_at( + file, + bytes[n_read:n_read + n_left], + cast(i64)chunk_offset, + ) or_return + + // Overflow to next chunk + chunk_index += 1 + chunk_offset = 0 + } + + return bytes, nil + } + + return nil, .File_Not_Exist +} diff --git a/src/chunks/write_assets.odin b/src/chunks/write_assets.odin index d8273e9..51f2c81 100644 --- a/src/chunks/write_assets.odin +++ b/src/chunks/write_assets.odin @@ -136,7 +136,7 @@ create_new_chunk :: proc( ) -> io.Stream { // Remove old chunk chunk_path := compute_chunk_path(output_path, chunk_index, allocator) - defer delete(chunk_path) + defer delete(chunk_path, allocator) if os.exists(chunk_path) { rm_err := os.remove(chunk_path) if rm_err != nil { diff --git a/src/chunks/write_metadata.odin b/src/chunks/write_metadata.odin index 31c7a3a..642a269 100644 --- a/src/chunks/write_metadata.odin +++ b/src/chunks/write_metadata.odin @@ -23,7 +23,7 @@ write_metadata :: proc( // ---------------------------------------- chunk_path := compute_chunk_path(output_path, 0, allocator) - defer delete(chunk_path) + defer delete(chunk_path, allocator) chunk_file, err := os.open(chunk_path, {.Read, .Write}) if err != nil { fmt.eprintln("error: open chunk failed: ", err) diff --git a/src/file/list_dir.odin b/src/file/list_dir.odin index 382aef3..378bbb6 100644 --- a/src/file/list_dir.odin +++ b/src/file/list_dir.odin @@ -14,7 +14,7 @@ File_Entry :: struct #all_or_none { // ----------------------------------------- -// Returning `[dynamic]DirectoryEntry` and each relpath `string` inside it are owned by the caller. +// Returning `[dynamic]File_Entry` and each relpath `string` inside it are owned by the caller. list_dir_recursive :: proc( f: ^os.File, working_dir: string, @@ -24,7 +24,7 @@ list_dir_recursive :: proc( return list_dir_recursive_by_path_impl(path, working_dir, allocator) } -// Returning `[dynamic]DirectoryEntry` and each relpath `string` inside it are owned by the caller. +// Returning `[dynamic]File_Entry` and each relpath `string` inside it are owned by the caller. list_dir_recursive_by_path :: proc( path: string, working_dir: string, diff --git a/src/main.odin b/src/main.odin index 2c637e7..bced74a 100644 --- a/src/main.odin +++ b/src/main.odin @@ -1,5 +1,6 @@ package main +import "core:fmt" import "core:os" import "src:chunks" @@ -24,21 +25,38 @@ main :: proc() { w := chunks.writer_init(cast(u64)len(entries)) defer chunks.writer_destroy(&w) - chunks.write_assets( - &w, - opts.output, - entries[:], - opts.size, - opts.compression, + // chunks.write_assets( + // &w, + // opts.output, + // entries[:], + // opts.size, + // opts.compression, + // ) + // + // chunks.write_metadata( + // &w, + // opts.output, + // entries[:], + // opts.size, + // opts.compression, + // ) + + header, err := chunks.read_header_by_path("./build/CHUNK0") + chunk0, o_err := os.open("./build/CHUNK0") + asset_index, a_err := chunks.read_asset_index(chunk0, header) + fmt.println( + "found: ", + chunks.file_exists(header, asset_index, "not_exists"), + ) + fmt.println( + "found: ", + chunks.file_exists(header, asset_index, "./toby/engage"), ) - chunks.write_metadata( - &w, - opts.output, - entries[:], - opts.size, - opts.compression, + fmt.printf( + "file: %c\n", + chunks.read_file(header, asset_index, opts.output, "./toby/engage"), ) - - chunks.read_header_by_path("./build/CHUNK0") } + +// TODO: .chunkignore