diff --git a/src/file/chunks.odin b/src/file/chunks.odin new file mode 100644 index 0000000..0a98313 --- /dev/null +++ b/src/file/chunks.odin @@ -0,0 +1,78 @@ +package file + +import "base:runtime" +import "core:fmt" +import "core:os" +import "core:strconv" +import "core:strings" + +MAGIC_STRING :: 0x514D21 // SINDRI +VERSION :: 1 +COMPRESSION :: 0 +SIZE_PER_CHUNK :: 2 * (1 << 30) // 2GiB +PATH_SIZE :: 512 + +// ----------------------------------------- + +Header :: struct #packed #all_or_none { + magic_string: u32, + version: u16, + compression: u16, + size_per_chunk: u64, + total_size: u64, + number_of_assets: u32, +} + +AssetIndex :: struct #packed #all_or_none { + relpath: [PATH_SIZE]u8, + size: u64, + offset: u64, +} + +Offsets :: struct { + asset_index_offset: u64, + data_offset: u64, + asset_offsets: [dynamic]u64, + asset_sizes: [dynamic]u64, +} + +// ----------------------------------------- + +@(private) +offsets: Offsets + +// ----------------------------------------- + +compute_metadata_offsets :: proc(number_of_assets: u64) { + // [ Header ][ []Asset Index ][ Data ] + offsets.asset_index_offset = size_of(Header) + offsets.data_offset = + size_of(Header) + (size_of(AssetIndex) * number_of_assets) +} + +@(private) +compute_chunk_path :: proc( + output_path: string, + chunk_index: int, + allocator: runtime.Allocator, +) -> string { + + @(static) buf: [20]u8 + + chunk_index_str := strconv.write_int(buf[:], cast(i64)chunk_index, 10) + chunk_name := strings.concatenate({"CHUNK", chunk_index_str}, allocator) + defer delete(chunk_name, allocator) + + chunk_path, c_err := os.join_path({output_path, chunk_name}, allocator) + if c_err != nil { + fmt.eprintln("error: chunk join path failed:", c_err) + os.exit(1) + } + + return chunk_path // owning string +} + +delete_offsets :: proc() { + delete(offsets.asset_offsets) + delete(offsets.asset_sizes) +} diff --git a/src/file/write_assets.odin b/src/file/write_assets.odin new file mode 100644 index 0000000..0c8b436 --- /dev/null +++ b/src/file/write_assets.odin @@ -0,0 +1,196 @@ +package file + +import "base:runtime" +import "core:fmt" +import "core:io" +import "core:os" + +// ----------------------------------------- + +@(private = "file") +chunk_file: ^os.File + +@(private = "file") +asset_file: ^os.File + +// ----------------------------------------- + +write_assets :: proc( + output: ^os.File, + entries: []FileEntry, + size_per_chunk: u64 = SIZE_PER_CHUNK, + compression: u16 = COMPRESSION, + allocator := context.allocator, +) { + output_path := os.name(output) + chunk_index := 0 + asset_index := 0 + chunk_offset: u64 = 0 + asset_offset: u64 = 0 + total_offset: u64 = offsets.data_offset + number_of_assets := len(entries) + resize(&offsets.asset_offsets, number_of_assets) + resize(&offsets.asset_sizes, number_of_assets) + + if size_per_chunk < offsets.data_offset { + fmt.eprintln( + "error: chunk size too small to hold asset index:", + offsets.data_offset, + ) + os.exit(1) + } + + // ---------------------------------------- + + // Create new chunk + chunk_stream := create_new_chunk(output_path, chunk_index, allocator) + + // Edge case of the chunk size being exactly the size of the data offset + if size_per_chunk == offsets.data_offset { + chunk_index += 1 + chunk_stream = create_new_chunk(output_path, chunk_index, allocator) + } + + // Chunk 0 starts after metadata + if (chunk_index == 0) { + chunk_offset = offsets.data_offset + } + + // ---------------------------------------- + + // Open asset + asset_stream, asset_size := open_asset(entries[asset_index].relpath) + offsets.asset_offsets[asset_index] = total_offset + offsets.asset_sizes[asset_index] = asset_size + + // ---------------------------------------- + + for { + chunk_remaining := size_per_chunk - chunk_offset + asset_remaining := asset_size - asset_offset + + io.seek(chunk_stream, cast(i64)chunk_offset, .Start) + io.seek(asset_stream, cast(i64)asset_offset, .Start) + + if asset_remaining < chunk_remaining { + + // Write asset to chunk + write_chunk(chunk_stream, asset_stream, asset_remaining) + chunk_offset += asset_remaining + total_offset += asset_remaining + + // Open next asset + asset_index += 1 + asset_offset = 0 + if asset_index == number_of_assets do break + asset_stream, asset_size = open_asset(entries[asset_index].relpath) + offsets.asset_offsets[asset_index] = total_offset + offsets.asset_sizes[asset_index] = asset_size + + } else if asset_remaining == chunk_remaining { + + // Write asset to chunk + write_chunk(chunk_stream, asset_stream, asset_remaining) + total_offset += asset_remaining + + // Open next asset + asset_index += 1 + asset_offset = 0 + if asset_index == number_of_assets do break + asset_stream, asset_size = open_asset(entries[asset_index].relpath) + offsets.asset_offsets[asset_index] = total_offset + offsets.asset_sizes[asset_index] = asset_size + + // Create new chunk + chunk_index += 1 + chunk_offset = 0 + chunk_stream = create_new_chunk( + output_path, + chunk_index, + allocator, + ) + } else { // spillover to next chunk + write_chunk(chunk_stream, asset_stream, chunk_remaining) + asset_offset += chunk_remaining + total_offset += chunk_remaining + + // Create new chunk + chunk_index += 1 + chunk_offset = 0 + chunk_stream = create_new_chunk( + output_path, + chunk_index, + allocator, + ) + } + } + + // ---------------------------------------- + + // Cleaup leftover + os.close(chunk_file) + os.close(asset_file) +} + +@(private) +create_new_chunk :: proc( + output_path: string, + chunk_index: int, + allocator: runtime.Allocator, +) -> io.Stream { + // Remove old chunk + chunk_path := compute_chunk_path(output_path, chunk_index, allocator) + defer delete(chunk_path) + if os.exists(chunk_path) { + rm_err := os.remove(chunk_path) + if rm_err != nil { + fmt.eprintln("error: remove old chunk failed:", rm_err) + os.exit(1) + } + } + + if chunk_file != nil do os.close(chunk_file) // close previous + + // Create new chunk + cf_err: os.Error + chunk_file, cf_err = os.create(chunk_path) + if cf_err != nil { + fmt.eprintln("error: create new chunk failed:", cf_err) + os.exit(1) + } + + chunk_stream := os.to_stream(chunk_file) + + return chunk_stream +} + +@(private) +open_asset :: proc(path: string) -> (io.Stream, u64) { + if asset_file != nil do os.close(asset_file) // close previous + + // Open asset + asset, open_err := os.open(path, {.Read}) + if open_err != nil { + fmt.eprintln("error: asset open failed:", open_err) + os.exit(1) + } + + // Asset size + asset_stream := os.to_stream(asset) + asset_size, size_err := io.size(asset_stream) + if size_err != nil { + fmt.eprintln("error: asset size failed:", size_err) + os.exit(1) + } + + return asset_stream, cast(u64)asset_size +} + +@(private) +write_chunk :: proc(dst: io.Stream, src: io.Stream, n: u64) { + w, err := io.copy_n(dst, src, cast(i64)n) + if err != nil { + fmt.eprintln("error: chunk write error:", err) + os.exit(1) + } +} diff --git a/src/file/write_chunks.odin b/src/file/write_chunks.odin deleted file mode 100644 index 90904ee..0000000 --- a/src/file/write_chunks.odin +++ /dev/null @@ -1,328 +0,0 @@ -package file - -import "base:runtime" -import "core:fmt" -import "core:io" -import "core:mem" -import "core:os" -import "core:strconv" -import "core:strings" - -MAGIC_STRING :: 0x514D21 // SINDRI -VERSION :: 1 -COMPRESSION :: 0 -SIZE_PER_CHUNK :: 2 * (1 << 30) // 2GiB -PATH_SIZE :: 512 - -// ----------------------------------------- - -Header :: struct #packed #all_or_none { - magic_string: u32, - version: u16, - compression: u16, - size_per_chunk: u64, - total_size: u64, - number_of_assets: u32, -} - -AssetIndex :: struct #packed #all_or_none { - relpath: [PATH_SIZE]u8, - size: u64, - offset: u64, -} - -Offsets :: struct { - asset_index_offset: u64, - data_offset: u64, - asset_offsets: [dynamic]u64, -} - -// ----------------------------------------- - -@(private) -offsets: Offsets - -@(private = "file") -chunk_file: ^os.File - -@(private = "file") -asset_file: ^os.File - -// ----------------------------------------- - -compute_metadata_offsets :: proc(number_of_assets: u64) { - // [ Header ][ []Asset Index ][ Data ] - offsets.asset_index_offset = size_of(Header) - offsets.data_offset = - size_of(Header) + (size_of(AssetIndex) * number_of_assets) -} - -// Returning `[]u8` is owned by the caller. -compute_header :: proc( - entries: []FileEntry, - compression: u16 = COMPRESSION, - size_per_chunk: u64 = SIZE_PER_CHUNK, - allocator := context.allocator, -) -> ( - header: Header, - asset_index: []AssetIndex, -) { - number_of_assets := cast(u32)len(entries) - - index_size: u64 = size_of(AssetIndex) * cast(u64)number_of_assets - data_offset: u64 = size_of(Header) + index_size - total_size: u64 = data_offset - - if data_offset > size_per_chunk { - fmt.eprintln( - "error: chunk size too small to hold metadata:", - data_offset, - ) - os.exit(1) - } - - bytes := make([]u8, index_size, allocator) - - for f, i in entries { - - path_length := len(f.relpath) - if path_length > 512 { - fmt.eprintln("error: path exeeds maximum of 512:", f.relpath) - os.exit(1) - } - - offset_in_bytes := size_of(AssetIndex) * cast(u64)i - offset_pointer := mem.ptr_offset(&bytes[0], offset_in_bytes) - - ai := cast(^AssetIndex)offset_pointer - copy(ai.relpath[:PATH_SIZE], f.relpath) - ai.size = cast(u64)f.size - ai.offset = data_offset - data_offset += ai.size - - total_size += ai.size - } - - header = Header { - magic_string = MAGIC_STRING, - version = VERSION, - compression = compression, - size_per_chunk = size_per_chunk, - total_size = total_size, - number_of_assets = number_of_assets, - } - - assert(len(bytes) % size_of(AssetIndex) == 0) // clean multiple - return header, mem.slice_data_cast([]AssetIndex, bytes) // transmute slice with proper size -} - -write_header :: proc( - output: ^os.File, - header: Header, - asset_index: []AssetIndex, - allocator := context.allocator, -) { - output_path := os.name(output) - - chunk_1, c1_err := os.join_path({output_path, "CHUNK1"}, allocator) - if c1_err != nil { - fmt.eprintln("error: failed to allocate path join:", c1_err) - os.exit(1) - } - - // Remove old chunk 1 - if os.exists(chunk_1) { - rm_err := os.remove(chunk_1) - if rm_err != nil { - fmt.eprintln("error: failed to remove old chunk:", rm_err) - os.exit(1) - } - } - - // Create chunk 1 - chunk_file_1, cf1_err := os.create(chunk_1) - if cf1_err != nil { - fmt.eprintln("error: failed to create chunk:", cf1_err) - os.exit(1) - } - - // Write header - header := header // shadow parameter - header_bytes := mem.ptr_to_bytes(&header) - os.write_at(chunk_file_1, header_bytes, 0) - - // Write asset index - asset_index := asset_index // shadow parameter - index_bytes := mem.slice_data_cast([]u8, asset_index) // transmute slice with proper size - os.write_at(chunk_file_1, index_bytes, size_of(Header)) -} - -write_chunks :: proc( - output: ^os.File, - size_per_chunk: u64 = SIZE_PER_CHUNK, - entries: []FileEntry, - allocator := context.allocator, -) { - output_path := os.name(output) - chunk_index := 0 - asset_index := 0 - chunk_offset: u64 = 0 - asset_offset: u64 = 0 - total_offset: u64 = 0 - number_of_assets := len(entries) - reserve(&offsets.asset_offsets, number_of_assets) - - buf: [20]u8 - - // ---------------------------------------- - - chunk_stream := create_new_chunk( - output_path, - chunk_index, - buf[:], - allocator, - ) - - if (chunk_index == 0) { - // Chunk 0 starts after metadata - chunk_offset = offsets.data_offset - total_offset = offsets.data_offset - } - - // ---------------------------------------- - - asset_stream, asset_size := open_asset(entries[asset_index].relpath) - offsets.asset_offsets[asset_index] = total_offset - - // ---------------------------------------- - - for { - chunk_remaining := size_per_chunk - chunk_offset - asset_remaining := asset_size - asset_offset - - io.seek(chunk_stream, cast(i64)chunk_offset, .Start) - io.seek(asset_stream, cast(i64)asset_offset, .Start) - - if asset_remaining < chunk_remaining { - - // Write asset to chunk - io.copy_n(chunk_stream, asset_stream, cast(i64)asset_remaining) - chunk_offset += asset_remaining - total_offset += asset_remaining - - // Open new asset - asset_index += 1 - asset_offset = 0 - if asset_index == number_of_assets do break - asset_stream, asset_size = open_asset(entries[asset_index].relpath) - offsets.asset_offsets[asset_index] = total_offset - - } else if asset_remaining == chunk_remaining { - - // Write asset to chunk - io.copy_n(chunk_stream, asset_stream, cast(i64)asset_remaining) - total_offset += asset_remaining - - // Open new asset - asset_index += 1 - asset_offset = 0 - if asset_index == number_of_assets do break - asset_stream, asset_size = open_asset(entries[asset_index].relpath) - offsets.asset_offsets[asset_index] = total_offset - - // Create new chunk - chunk_index += 1 - chunk_offset = 0 - chunk_stream = create_new_chunk( - output_path, - chunk_index, - buf[:], - allocator, - ) - } else { - io.copy_n(chunk_stream, asset_stream, cast(i64)chunk_remaining) - asset_offset += chunk_remaining - total_offset += chunk_remaining - - // Create new chunk - chunk_index += 1 - chunk_offset = 0 - chunk_stream = create_new_chunk( - output_path, - chunk_index, - buf[:], - allocator, - ) - } - } - - // ---------------------------------------- - - // Cleaup leftover - os.close(chunk_file) - os.close(asset_file) -} - -@(private) -create_new_chunk :: proc( - output_path: string, - chunk_index: int, - buf: []u8, - allocator: runtime.Allocator, -) -> io.Stream { - // Get chunk path - chunk_index_str := strconv.write_int(buf[:], cast(i64)chunk_index, 10) - chunk_name := strings.concatenate({"CHUNK", chunk_index_str}, allocator) - defer delete(chunk_name, allocator) - chunk, c_err := os.join_path({output_path, chunk_name}, allocator) - if c_err != nil { - fmt.eprintln("error: chunk join path failed:", c_err) - os.exit(1) - } - defer delete(chunk, allocator) - - // Remove old chunk - if os.exists(chunk) { - rm_err := os.remove(chunk) - if rm_err != nil { - fmt.eprintln("error: remove old chunk failed:", rm_err) - os.exit(1) - } - } - - if chunk_file != nil do os.close(chunk_file) // close previous - - // Create new chunk - cf_err: os.Error - chunk_file, cf_err = os.create(chunk) - if cf_err != nil { - fmt.eprintln("error: create new chunk failed:", cf_err) - os.exit(1) - } - - chunk_stream := os.to_stream(chunk_file) - - return chunk_stream -} - -@(private) -open_asset :: proc(path: string) -> (io.Stream, u64) { - if asset_file != nil do os.close(asset_file) // close previous - - // Open asset - asset, open_err := os.open(path, {.Read}) - if open_err != nil { - fmt.eprintln("error: asset open failed:", open_err) - os.exit(1) - } - - // Asset size - asset_stream := os.to_stream(asset) - asset_size, size_err := io.size(asset_stream) - if size_err != nil { - fmt.eprintln("error: asset size failed:", size_err) - os.exit(1) - } - - return asset_stream, cast(u64)asset_size -} diff --git a/src/file/write_metadata.odin b/src/file/write_metadata.odin new file mode 100644 index 0000000..bb9ac34 --- /dev/null +++ b/src/file/write_metadata.odin @@ -0,0 +1,108 @@ +package file + +import "base:runtime" +import "core:fmt" +import "core:mem" +import "core:os" + +// ----------------------------------------- + +write_metadata :: proc( + output: ^os.File, + entries: []FileEntry, + size_per_chunk: u64 = SIZE_PER_CHUNK, + compression: u16 = COMPRESSION, + allocator := context.allocator, +) { + output_path := os.name(output) + number_of_assets := cast(u32)len(entries) + + // ---------------------------------------- + + chunk_path := compute_chunk_path(output_path, 0, allocator) + defer delete(chunk_path) + chunk_file, err := os.open(chunk_path, {.Read, .Write}) + if err != nil { + fmt.eprintln("error: open chunk failed: ", err) + os.exit(1) + } + defer os.close(chunk_file) + + write_header(chunk_file, number_of_assets, size_per_chunk, compression) + + write_asset_index(chunk_file, entries, number_of_assets, allocator) +} + +@(private) +write_header :: proc( + chunk_file: ^os.File, + number_of_assets: u32, + size_per_chunk: u64 = SIZE_PER_CHUNK, + compression: u16 = COMPRESSION, +) { + last_offset := offsets.asset_offsets[len(offsets.asset_offsets) - 1] + last_size := offsets.asset_sizes[len(offsets.asset_sizes) - 1] + total_size := last_offset + last_size + + header := Header { + magic_string = MAGIC_STRING, + version = VERSION, + compression = compression, + size_per_chunk = size_per_chunk, + total_size = total_size, + number_of_assets = number_of_assets, + } + + // Write header + header_bytes := mem.ptr_to_bytes(&header) + n, err := os.write_at(chunk_file, header_bytes, 0) + if err != nil { + fmt.eprintln("error: chunk write error:", err) + os.exit(1) + } +} + +@(private) +write_asset_index :: proc( + chunk_file: ^os.File, + entries: []FileEntry, + number_of_assets: u32, + allocator: runtime.Allocator, +) { + index_size: u64 = size_of(AssetIndex) * cast(u64)number_of_assets + + // ---------------------------------------- + + index_bytes := make([]u8, index_size, allocator) + + for f, i in entries { + + // TODO: Move to cli validation so it happens sooner + path_length := len(f.relpath) + if path_length > 512 { + fmt.eprintln("error: path exeeds maximum of 512:", f.relpath) + os.exit(1) + } + + offset_in_bytes := size_of(AssetIndex) * cast(u64)i + offset_pointer := mem.ptr_offset(&index_bytes[0], offset_in_bytes) + + ai := cast(^AssetIndex)offset_pointer + copy(ai.relpath[:PATH_SIZE], f.relpath) + ai.size = offsets.asset_sizes[i] + ai.offset = offsets.asset_offsets[i] + } + + assert(len(index_bytes) % size_of(AssetIndex) == 0) // clean multiple + + // Write asset index + n, err := os.write_at( + chunk_file, + index_bytes, + cast(i64)offsets.asset_index_offset, + ) + if err != nil { + fmt.eprintln("error: chunk write error:", err) + os.exit(1) + } +} diff --git a/src/main.odin b/src/main.odin index 3ec6bd1..09afdf9 100644 --- a/src/main.odin +++ b/src/main.odin @@ -1,6 +1,6 @@ package main -import "core:fmt" +import "core:os" import "src:cli" import "src:file" @@ -17,22 +17,10 @@ main :: proc() { // fmt.println("compression:", opts.compression) // fmt.println("size:", opts.size) - // asset := file.get_asset_by_path("./odinfmt.json") - // fmt.println("path:", asset.relpath) - // fmt.println("size:", asset.size) - // fmt.println("data:", asset.data) - - entries := file.list_dir_recursive_by_path("./src") - // fmt.println("entries:", entries) - + entries := file.list_dir_recursive(opts.input) file.compute_metadata_offsets(cast(u64)len(entries)) - file.write_chunks(opts.output, opts.size, entries[:]) - - - // header, asset_index := file.compute_header(entries[:]) - // fmt.println("header:", header) - // fmt.println("aia:", asset_index) - // - // file.write_header(opts.output, header, asset_index) + file.write_assets(opts.output, entries[:], opts.size, opts.compression) + file.write_metadata(opts.output, entries[:], opts.size, opts.compression) + file.delete_offsets() }