riyyi/sindri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add chunk writing code

Browse Source
This commit is contained in:
Riyyi
2026-08-03 17:55:12 +02:00
parent 362f0dc048
commit d1c9c1be05
4 changed files with 347 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2
View File
@@ -22,3 +22,5 @@ Asset packer.
./build.sh ./build.sh
./build.sh debug ./build.sh debug
``` ```
<!-- TODO: git hook with fmt setup -->
src/file/header.odin
-130
View File
@@ -1,130 +0,0 @@
package file
import "core:fmt"
import "core:mem"
import "core:os"
MAGIC_STRING :: 0x514D21 // SINDRI
VERSION :: 1
COMPRESSION :: 0
SIZE_PER_CHUNK :: 2 * (1 << 30) // 2GiB
PATH_SIZE :: 512
// -----------------------------------------
Header :: struct #packed #all_or_none {
magic_string: u32,
version: u16,
compression: u16,
size_per_chunk: u64,
total_size: u64,
number_of_assets: u32,
}
AssetIndex :: struct #packed #all_or_none {
relpath: [PATH_SIZE]u8,
size: u64,
offset: u64,
}
// -----------------------------------------
// remove :: proc(name: string) -> Error {…}
// create :: proc(name: string) -> (^File, Error) {…}
// write_at :: proc(f: ^File, p: []u8, offset: i64) -> (n: int, err: Error) {…}
// Returning `[]u8` is owned by the caller.
compute_header :: proc(
entries: [dynamic]FileEntry,
compression: u16 = COMPRESSION,
size_per_chunk: u64 = SIZE_PER_CHUNK,
allocator := context.allocator,
) -> (
header: Header,
asset_index: []AssetIndex,
) {
number_of_assets := cast(u32)len(entries)
index_size: u64 = size_of(AssetIndex) * cast(u64)number_of_assets
data_offset: u64 = size_of(Header) + index_size
total_size: u64 = data_offset
bytes := make([]u8, index_size, allocator)
for f, i in entries {
path_length := len(f.relpath)
if path_length > 512 {
fmt.eprintln("error: path exeeds maximum of 512:", f.relpath)
os.exit(1)
}
offset_in_bytes := size_of(AssetIndex) * cast(u64)i
offset_pointer := mem.ptr_offset(&bytes[0], offset_in_bytes)
ai := cast(^AssetIndex)offset_pointer
copy(ai.relpath[:PATH_SIZE], f.relpath)
ai.size = cast(u64)f.size
ai.offset = data_offset
data_offset += ai.size
total_size += ai.size
}
header = Header {
magic_string = MAGIC_STRING,
version = VERSION,
compression = compression,
size_per_chunk = size_per_chunk,
total_size = total_size,
number_of_assets = number_of_assets,
}
assert(len(bytes) % size_of(AssetIndex) == 0) // clean multiple
return header, mem.slice_data_cast([]AssetIndex, bytes) // transmute slice with proper size
}
write_header :: proc(
output: ^os.File,
header: Header,
asset_index: []AssetIndex,
allocator := context.allocator,
) {
// remove :: proc(name: string) -> Error {…}
// create :: proc(name: string) -> (^File, Error) {…}
// write_at :: proc(f: ^File, p: []u8, offset: i64) -> (n: int, err: Error) {…}
output_path := os.name(output)
chunk_1, c1_err := os.join_path({output_path, "CHUNK1"}, allocator)
if c1_err != nil {
fmt.eprintln("error: failed to allocate path join:", c1_err)
os.exit(1)
}
// Remove old chunk 1
if os.exists(chunk_1) {
rm_err := os.remove(chunk_1)
if rm_err != nil {
fmt.eprintln("error: failed to remove old chunk:", rm_err)
os.exit(1)
}
}
// Create chunk 1
chunk_file_1, cf1_err := os.create(chunk_1)
if cf1_err != nil {
fmt.eprintln("error: failed to create chunk:", cf1_err)
os.exit(1)
}
// Write header
header := header // shadow parameter
header_bytes := mem.ptr_to_bytes(&header)
os.write_at(chunk_file_1, header_bytes, 0)
// Write asset index
asset_index := asset_index // shadow parameter
index_bytes := mem.slice_data_cast([]u8, asset_index) // transmute slice with proper size
os.write_at(chunk_file_1, index_bytes, size_of(Header))
}
src/file/write_chunks.odin
+328
View File
@@ -0,0 +1,328 @@
package file
import "base:runtime"
import "core:fmt"
import "core:io"
import "core:mem"
import "core:os"
import "core:strconv"
import "core:strings"
MAGIC_STRING :: 0x514D21 // SINDRI
VERSION :: 1
COMPRESSION :: 0
SIZE_PER_CHUNK :: 2 * (1 << 30) // 2GiB
PATH_SIZE :: 512
// -----------------------------------------
Header :: struct #packed #all_or_none {
magic_string: u32,
version: u16,
compression: u16,
size_per_chunk: u64,
total_size: u64,
number_of_assets: u32,
}
AssetIndex :: struct #packed #all_or_none {
relpath: [PATH_SIZE]u8,
size: u64,
offset: u64,
}
Offsets :: struct {
asset_index_offset: u64,
data_offset: u64,
asset_offsets: [dynamic]u64,
}
// -----------------------------------------
@(private)
offsets: Offsets
@(private = "file")
chunk_file: ^os.File
@(private = "file")
asset_file: ^os.File
// -----------------------------------------
compute_metadata_offsets :: proc(number_of_assets: u64) {
// [ Header ][ []Asset Index ][ Data ]
offsets.asset_index_offset = size_of(Header)
offsets.data_offset =
size_of(Header) + (size_of(AssetIndex) * number_of_assets)
}
// Returning `[]u8` is owned by the caller.
compute_header :: proc(
entries: []FileEntry,
compression: u16 = COMPRESSION,
size_per_chunk: u64 = SIZE_PER_CHUNK,
allocator := context.allocator,
) -> (
header: Header,
asset_index: []AssetIndex,
) {
number_of_assets := cast(u32)len(entries)
index_size: u64 = size_of(AssetIndex) * cast(u64)number_of_assets
data_offset: u64 = size_of(Header) + index_size
total_size: u64 = data_offset
if data_offset > size_per_chunk {
fmt.eprintln(
"error: chunk size too small to hold metadata:",
data_offset,
)
os.exit(1)
}
bytes := make([]u8, index_size, allocator)
for f, i in entries {
path_length := len(f.relpath)
if path_length > 512 {
fmt.eprintln("error: path exeeds maximum of 512:", f.relpath)
os.exit(1)
}
offset_in_bytes := size_of(AssetIndex) * cast(u64)i
offset_pointer := mem.ptr_offset(&bytes[0], offset_in_bytes)
ai := cast(^AssetIndex)offset_pointer
copy(ai.relpath[:PATH_SIZE], f.relpath)
ai.size = cast(u64)f.size
ai.offset = data_offset
data_offset += ai.size
total_size += ai.size
}
header = Header {
magic_string = MAGIC_STRING,
version = VERSION,
compression = compression,
size_per_chunk = size_per_chunk,
total_size = total_size,
number_of_assets = number_of_assets,
}
assert(len(bytes) % size_of(AssetIndex) == 0) // clean multiple
return header, mem.slice_data_cast([]AssetIndex, bytes) // transmute slice with proper size
}
write_header :: proc(
output: ^os.File,
header: Header,
asset_index: []AssetIndex,
allocator := context.allocator,
) {
output_path := os.name(output)
chunk_1, c1_err := os.join_path({output_path, "CHUNK1"}, allocator)
if c1_err != nil {
fmt.eprintln("error: failed to allocate path join:", c1_err)
os.exit(1)
}
// Remove old chunk 1
if os.exists(chunk_1) {
rm_err := os.remove(chunk_1)
if rm_err != nil {
fmt.eprintln("error: failed to remove old chunk:", rm_err)
os.exit(1)
}
}
// Create chunk 1
chunk_file_1, cf1_err := os.create(chunk_1)
if cf1_err != nil {
fmt.eprintln("error: failed to create chunk:", cf1_err)
os.exit(1)
}
// Write header
header := header // shadow parameter
header_bytes := mem.ptr_to_bytes(&header)
os.write_at(chunk_file_1, header_bytes, 0)
// Write asset index
asset_index := asset_index // shadow parameter
index_bytes := mem.slice_data_cast([]u8, asset_index) // transmute slice with proper size
os.write_at(chunk_file_1, index_bytes, size_of(Header))
}
write_chunks :: proc(
output: ^os.File,
size_per_chunk: u64 = SIZE_PER_CHUNK,
entries: []FileEntry,
allocator := context.allocator,
) {
output_path := os.name(output)
chunk_index := 0
asset_index := 0
chunk_offset: u64 = 0
asset_offset: u64 = 0
total_offset: u64 = 0
number_of_assets := len(entries)
reserve(&offsets.asset_offsets, number_of_assets)
buf: [20]u8
// ----------------------------------------
chunk_stream := create_new_chunk(
output_path,
chunk_index,
buf[:],
allocator,
)
if (chunk_index == 0) {
// Chunk 0 starts after metadata
chunk_offset = offsets.data_offset
total_offset = offsets.data_offset
}
// ----------------------------------------
asset_stream, asset_size := open_asset(entries[asset_index].relpath)
offsets.asset_offsets[asset_index] = total_offset
// ----------------------------------------
for {
chunk_remaining := size_per_chunk - chunk_offset
asset_remaining := asset_size - asset_offset
io.seek(chunk_stream, cast(i64)chunk_offset, .Start)
io.seek(asset_stream, cast(i64)asset_offset, .Start)
if asset_remaining < chunk_remaining {
// Write asset to chunk
io.copy_n(chunk_stream, asset_stream, cast(i64)asset_remaining)
chunk_offset += asset_remaining
total_offset += asset_remaining
// Open new asset
asset_index += 1
asset_offset = 0
if asset_index == number_of_assets do break
asset_stream, asset_size = open_asset(entries[asset_index].relpath)
offsets.asset_offsets[asset_index] = total_offset
} else if asset_remaining == chunk_remaining {
// Write asset to chunk
io.copy_n(chunk_stream, asset_stream, cast(i64)asset_remaining)
total_offset += asset_remaining
// Open new asset
asset_index += 1
asset_offset = 0
if asset_index == number_of_assets do break
asset_stream, asset_size = open_asset(entries[asset_index].relpath)
offsets.asset_offsets[asset_index] = total_offset
// Create new chunk
chunk_index += 1
chunk_offset = 0
chunk_stream = create_new_chunk(
output_path,
chunk_index,
buf[:],
allocator,
)
} else {
io.copy_n(chunk_stream, asset_stream, cast(i64)chunk_remaining)
asset_offset += chunk_remaining
total_offset += chunk_remaining
// Create new chunk
chunk_index += 1
chunk_offset = 0
chunk_stream = create_new_chunk(
output_path,
chunk_index,
buf[:],
allocator,
)
}
}
// ----------------------------------------
// Cleaup leftover
os.close(chunk_file)
os.close(asset_file)
}
@(private)
create_new_chunk :: proc(
output_path: string,
chunk_index: int,
buf: []u8,
allocator: runtime.Allocator,
) -> io.Stream {
// Get chunk path
chunk_index_str := strconv.write_int(buf[:], cast(i64)chunk_index, 10)
chunk_name := strings.concatenate({"CHUNK", chunk_index_str}, allocator)
defer delete(chunk_name, allocator)
chunk, c_err := os.join_path({output_path, chunk_name}, allocator)
if c_err != nil {
fmt.eprintln("error: chunk join path failed:", c_err)
os.exit(1)
}
defer delete(chunk, allocator)
// Remove old chunk
if os.exists(chunk) {
rm_err := os.remove(chunk)
if rm_err != nil {
fmt.eprintln("error: remove old chunk failed:", rm_err)
os.exit(1)
}
}
if chunk_file != nil do os.close(chunk_file) // close previous
// Create new chunk
cf_err: os.Error
chunk_file, cf_err = os.create(chunk)
if cf_err != nil {
fmt.eprintln("error: create new chunk failed:", cf_err)
os.exit(1)
}
chunk_stream := os.to_stream(chunk_file)
return chunk_stream
}
@(private)
open_asset :: proc(path: string) -> (io.Stream, u64) {
if asset_file != nil do os.close(asset_file) // close previous
// Open asset
asset, open_err := os.open(path, {.Read})
if open_err != nil {
fmt.eprintln("error: asset open failed:", open_err)
os.exit(1)
}
// Asset size
asset_stream := os.to_stream(asset)
asset_size, size_err := io.size(asset_stream)
if size_err != nil {
fmt.eprintln("error: asset size failed:", size_err)
os.exit(1)
}
return asset_stream, cast(u64)asset_size
}
src/main.odin
+17 -12
View File
@@ -13,21 +13,26 @@ main :: proc() {
opts := cli.parse(file.working_dir) opts := cli.parse(file.working_dir)
fmt.println("verbose:", opts.verbose) // fmt.println("verbose:", opts.verbose)
fmt.println("compression:", opts.compression) // fmt.println("compression:", opts.compression)
fmt.println("size:", opts.size) // fmt.println("size:", opts.size)
asset := file.get_asset_by_path("./odinfmt.json") // asset := file.get_asset_by_path("./odinfmt.json")
fmt.println("path:", asset.relpath) // fmt.println("path:", asset.relpath)
fmt.println("size:", asset.size) // fmt.println("size:", asset.size)
fmt.println("data:", asset.data) // fmt.println("data:", asset.data)
entries := file.list_dir_recursive_by_path("./src") entries := file.list_dir_recursive_by_path("./src")
fmt.println("entries:", entries) // fmt.println("entries:", entries)
header, asset_index := file.compute_header(entries)
fmt.println("header:", header)
fmt.println("aia:", asset_index)
file.write_header(opts.output, header, asset_index) file.compute_metadata_offsets(cast(u64)len(entries))
file.write_chunks(opts.output, opts.size, entries[:])
// header, asset_index := file.compute_header(entries[:])
// fmt.println("header:", header)
// fmt.println("aia:", asset_index)
//
// file.write_header(opts.output, header, asset_index)
} }