diff --git a/doc/design.md b/doc/design.md index 8b721a1..ac17329 100644 --- a/doc/design.md +++ b/doc/design.md @@ -1,24 +1,24 @@ # Design The data layout design is very simple. It consists of data split into chunks -with the first chunk holding the metadata for the structure and files. +with the first chunk holding the metadata for the structure and assets. ## Spec ``` - METADATA DATA -|---------------------------------------------------------------------------||------| + METADATA DATA +|----------------------------------------------------------------------------------------||------| -[ MAGIC ][ VERSION ][ COMPRESSION ][ SIZE_PER_CHUNK ][ TOTAL_SIZE ][ #FILES ][ DATA ] # chunk 1 -[ DATA ] # chunk 2 -[ DATA ] # etc.. -[ DATA ] # last chunk +[ MAGIC ][ VERSION ][ COMPRESSION ][ SIZE_PER_CHUNK ][ TOTAL_SIZE ][ #ASSETS ][ []ASSETS ][ DATA ] # chunk 1 +[ DATA ] # chunk 2 +[ DATA ] # etc.. +[ DATA ] # last chunk ``` ### Magic string Magic string to detect if the read file is what we expect it is. -Represented in 3 bytes. +Represented in 4 bytes. Value is `0x514D21`, the closest approximation of "Sindri". @@ -32,7 +32,7 @@ Value is `1` currently. ### Compression Wether LZ4 compression is enabled/disabled. -Represented in `1 byte`. +Represented in `2 byte`. Value is configurable, defaulting to `0`. @@ -54,30 +54,30 @@ Value is configurable, defaulting to `2GiB`. The total size in bytes of all chunks combined. Represented in `8 bytes`. -### Number of files +### Number of assets -The total number of files in the pack. +The total number of assets in the pack. Represented in `4 bytes`. -### File +### []Asset -Piece of data stored, consisting of 3 parts: +Index to piece of stored data, consisting of 3 parts: - Path -- Offset - Size +- Offset #### Path -Path of the file as if it were on the filesystem. +Path of the asset as if it were on the filesystem. Represented in `512 bytes`. -#### Offset - -The offset to the file, starting from the true 0th byte of the first chunk. -Represented in `8 bytes`. - #### Size -Size of the file. +Size of the asset. +Represented in `8 bytes`. + +#### Offset + +The offset to the asset, starting from the true 0th byte of the first chunk. Represented in `8 bytes`. diff --git a/src/cli/cli.odin b/src/cli/cli.odin index 8e2d1f7..0ae01c7 100644 --- a/src/cli/cli.odin +++ b/src/cli/cli.odin @@ -7,8 +7,8 @@ import "core:strings" Options :: struct { verbose: bool `args:"name=verbose" usage:"Enable verbose output"`, - compression: u8 `args:"name=compression" usage:"Compression level, 0-12"`, - size: i64 `args:"name=size" usage:"Chunk size in bytes"`, + compression: u16 `args:"name=compression" usage:"Compression level, 0-12"`, + size: u64 `args:"name=size" usage:"Chunk size in bytes"`, input: ^os.File `args:"name=input,file=r,required" usage:"Path to input directory"`, output: ^os.File `args:"name=output,file=wc" usage:"Path to ouput directory"`, // overflow: [dynamic]string, diff --git a/src/file/header.odin b/src/file/header.odin new file mode 100644 index 0000000..5b71762 --- /dev/null +++ b/src/file/header.odin @@ -0,0 +1,89 @@ +package file + +import "core:fmt" +import "core:mem" +import "core:os" + +MAGIC_STRING :: 0x514D21 // SINDRI +VERSION :: 1 +COMPRESSION :: 0 +SIZE_PER_CHUNK :: 2 * (1 << 30) // 2GiB +PATH_SIZE :: 512 + +// ----------------------------------------- + +Header :: struct #packed #all_or_none { + magic_string: u32, + version: u16, + compression: u16, + size_per_chunk: u64, + total_size: u64, + number_of_assets: u32, + // asset_index: []AssetIndex, +} + +AssetIndex :: struct #packed #all_or_none { + relpath: [PATH_SIZE]u8, + size: u64, + offset: u64, +} + +// ----------------------------------------- + +// remove :: proc(name: string) -> Error {…} +// create :: proc(name: string) -> (^File, Error) {…} +// write_at :: proc(f: ^File, p: []u8, offset: i64) -> (n: int, err: Error) {…} + +// Returning `[]u8` is owned by the caller. +compute_header :: proc( + entries: [dynamic]FileEntry, + compression: u16 = COMPRESSION, + size_per_chunk: u64 = SIZE_PER_CHUNK, + allocator := context.allocator, +) -> ( + header: Header, + asset_index: []AssetIndex, +) { + number_of_assets := cast(u32)len(entries) + + index_size: u64 = size_of(AssetIndex) * cast(u64)number_of_assets + data_offset: u64 = size_of(Header) + index_size + total_size: u64 = data_offset + + bytes := make([]u8, index_size, allocator) + + for f, i in entries { + + path_length := len(f.relpath) + if path_length > 512 { + fmt.eprintln("error: path exeeds maximum of 512:", f.relpath) + os.exit(1) + } + + offset_in_bytes := size_of(AssetIndex) * cast(u64)i + offset_pointer := mem.ptr_offset( + &bytes[0], + offset_in_bytes, + ) + + ai := cast(^AssetIndex)offset_pointer + copy(ai.relpath[:PATH_SIZE], f.relpath) + ai.size = cast(u64)f.size + ai.offset = data_offset + data_offset += ai.size + + total_size += ai.size + } + + header = Header { + magic_string = MAGIC_STRING, + version = VERSION, + compression = compression, + size_per_chunk = size_per_chunk, + total_size = total_size, + number_of_assets = number_of_assets, + } + + assert(len(bytes) % size_of(AssetIndex) == 0) // clean multiple + return header, mem.slice_data_cast([]AssetIndex, bytes) // transmute slice with proper size +} diff --git a/src/file/list_dir.odin b/src/file/list_dir.odin index bee1539..e8e0195 100644 --- a/src/file/list_dir.odin +++ b/src/file/list_dir.odin @@ -7,7 +7,7 @@ import "core:strings" // ----------------------------------------- -DirectoryEntry :: struct #all_or_none { +FileEntry :: struct #all_or_none { relpath: string, // NOTE: owning string size: i64, } @@ -18,7 +18,7 @@ DirectoryEntry :: struct #all_or_none { list_dir_recursive :: proc( f: ^os.File, allocator := context.allocator, -) -> [dynamic]DirectoryEntry { +) -> [dynamic]FileEntry { path := os.name(f) return list_dir_recursive_by_path_impl(path, allocator) } @@ -27,7 +27,7 @@ list_dir_recursive :: proc( list_dir_recursive_by_path :: proc( path: string, allocator := context.allocator, -) -> [dynamic]DirectoryEntry { +) -> [dynamic]FileEntry { return list_dir_recursive_by_path_impl(path, allocator) } @@ -35,13 +35,13 @@ list_dir_recursive_by_path :: proc( list_dir_recursive_by_path_impl :: proc( path: string, allocator: runtime.Allocator, -) -> [dynamic]DirectoryEntry { +) -> [dynamic]FileEntry { if !os.is_dir(path) { fmt.eprintln("error: path is not a directory:", path) os.exit(1) } - result := make([dynamic]DirectoryEntry, allocator) + result := make([dynamic]FileEntry, allocator) queue := make([dynamic]string, 0, 1, allocator) // reserve 1 slot defer delete(queue) // dynamic array embeds allocator @@ -57,7 +57,7 @@ list_dir_recursive_by_path_impl :: proc( @(private) list_dir_queue :: proc( queue: ^[dynamic]string, - result: ^[dynamic]DirectoryEntry, + result: ^[dynamic]FileEntry, working_dir: string, allocator: runtime.Allocator, ) { @@ -99,10 +99,7 @@ list_dir_queue :: proc( if !was_allocation { relpath = strings.clone(relpath, result^.allocator) } - append( - result, - DirectoryEntry{relpath = relpath, size = entry.size}, - ) + append(result, FileEntry{relpath = relpath, size = entry.size}) } case: // skip other types } diff --git a/src/main.odin b/src/main.odin index 2818f1f..d1f126b 100644 --- a/src/main.odin +++ b/src/main.odin @@ -24,4 +24,8 @@ main :: proc() { entries := file.list_dir_recursive_by_path("./src") fmt.println("entries:", entries) + + header, asset_index_array := file.compute_header(entries) + fmt.println("header:", header) + fmt.println("aia:", asset_index_array) }