Move chunk impl to chunk package
This commit is contained in:
Riyyi
@@ -0,0 +1,92 @@
|
||||
package chunks
|
||||
|
||||
import "base:runtime"
|
||||
import "core:fmt"
|
||||
import "core:os"
|
||||
import "core:strconv"
|
||||
import "core:strings"
|
||||
|
||||
MAGIC_STRING :: 0x514D21 // SINDRI
|
||||
VERSION :: 1
|
||||
COMPRESSION :: 0
|
||||
SIZE_PER_CHUNK :: 2 * (1 << 30) // 2GiB
|
||||
PATH_SIZE :: 512
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
Header :: struct #packed #all_or_none {
|
||||
magic_string: u32,
|
||||
version: u16,
|
||||
compression: u16,
|
||||
size_per_chunk: u64,
|
||||
total_size: u64,
|
||||
number_of_assets: u32,
|
||||
}
|
||||
|
||||
AssetIndex :: struct #packed #all_or_none {
|
||||
relpath: [PATH_SIZE]u8,
|
||||
size: u64,
|
||||
offset: u64,
|
||||
}
|
||||
|
||||
AssetTable :: struct {
|
||||
asset_index_offset: u64,
|
||||
data_offset: u64,
|
||||
asset_offsets: [dynamic]u64,
|
||||
asset_sizes: [dynamic]u64,
|
||||
}
|
||||
|
||||
Writer :: struct {
|
||||
chunk_file: ^os.File,
|
||||
asset_file: ^os.File,
|
||||
asset_table: AssetTable,
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
writer_init :: proc(
|
||||
number_of_assets: u64,
|
||||
allocator := context.allocator,
|
||||
) -> (
|
||||
w: Writer,
|
||||
err: runtime.Allocator_Error,
|
||||
) #optional_allocator_error {
|
||||
// [ Header ][ []Asset Index ][ Data ]
|
||||
w.asset_table.asset_index_offset = size_of(Header)
|
||||
w.asset_table.data_offset =
|
||||
size_of(Header) + (size_of(AssetIndex) * number_of_assets)
|
||||
w.asset_table.asset_offsets = make([dynamic]u64, allocator) or_return
|
||||
w.asset_table.asset_sizes = make([dynamic]u64, allocator) or_return
|
||||
resize(&w.asset_table.asset_offsets, number_of_assets)
|
||||
resize(&w.asset_table.asset_sizes, number_of_assets)
|
||||
return w, nil
|
||||
}
|
||||
|
||||
writer_destroy :: proc(w: ^Writer) {
|
||||
if w.chunk_file != nil do os.close(w.chunk_file)
|
||||
if w.asset_file != nil do os.close(w.asset_file)
|
||||
delete(w.asset_table.asset_offsets)
|
||||
delete(w.asset_table.asset_sizes)
|
||||
}
|
||||
|
||||
@(private)
|
||||
compute_chunk_path :: proc(
|
||||
output_path: string,
|
||||
chunk_index: int,
|
||||
allocator: runtime.Allocator,
|
||||
) -> string {
|
||||
|
||||
@(static) buf: [20]u8
|
||||
|
||||
chunk_index_str := strconv.write_int(buf[:], cast(i64)chunk_index, 10)
|
||||
chunk_name := strings.concatenate({"CHUNK", chunk_index_str}, allocator)
|
||||
defer delete(chunk_name, allocator)
|
||||
|
||||
chunk_path, c_err := os.join_path({output_path, chunk_name}, allocator)
|
||||
if c_err != nil {
|
||||
fmt.eprintln("error: chunk join path failed:", c_err)
|
||||
os.exit(1)
|
||||
}
|
||||
|
||||
return chunk_path // owning string
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package chunks
|
||||
@@ -0,0 +1,192 @@
|
||||
package chunks
|
||||
|
||||
import "base:runtime"
|
||||
import "core:fmt"
|
||||
import "core:io"
|
||||
import "core:os"
|
||||
|
||||
import "src:file"
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
write_assets :: proc(
|
||||
w: ^Writer,
|
||||
output: ^os.File,
|
||||
entries: []file.FileEntry,
|
||||
size_per_chunk: u64 = SIZE_PER_CHUNK,
|
||||
compression: u16 = COMPRESSION,
|
||||
allocator := context.allocator,
|
||||
) {
|
||||
output_path := os.name(output)
|
||||
chunk_index := 0
|
||||
asset_index := 0
|
||||
chunk_offset: u64 = 0
|
||||
asset_offset: u64 = 0
|
||||
total_offset: u64 = w.asset_table.data_offset
|
||||
number_of_assets := len(entries)
|
||||
|
||||
if size_per_chunk < w.asset_table.data_offset {
|
||||
fmt.eprintln(
|
||||
"error: chunk size too small to hold asset index:",
|
||||
w.asset_table.data_offset,
|
||||
)
|
||||
os.exit(1)
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
// Create new chunk
|
||||
chunk_stream := create_new_chunk(w, output_path, chunk_index, allocator)
|
||||
|
||||
// Edge case of the chunk size being exactly the size of the data offset
|
||||
if size_per_chunk == w.asset_table.data_offset {
|
||||
chunk_index += 1
|
||||
chunk_stream = create_new_chunk(w, output_path, chunk_index, allocator)
|
||||
}
|
||||
|
||||
// Chunk 0 starts after metadata
|
||||
if (chunk_index == 0) {
|
||||
chunk_offset = w.asset_table.data_offset
|
||||
}
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
// Open asset
|
||||
asset_stream, asset_size := open_asset(w, entries[asset_index].relpath)
|
||||
w.asset_table.asset_offsets[asset_index] = total_offset
|
||||
w.asset_table.asset_sizes[asset_index] = asset_size
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
for {
|
||||
chunk_remaining := size_per_chunk - chunk_offset
|
||||
asset_remaining := asset_size - asset_offset
|
||||
|
||||
io.seek(chunk_stream, cast(i64)chunk_offset, .Start)
|
||||
io.seek(asset_stream, cast(i64)asset_offset, .Start)
|
||||
|
||||
if asset_remaining < chunk_remaining {
|
||||
|
||||
// Write asset to chunk
|
||||
write_chunk(chunk_stream, asset_stream, asset_remaining)
|
||||
chunk_offset += asset_remaining
|
||||
total_offset += asset_remaining
|
||||
|
||||
// Open next asset
|
||||
asset_index += 1
|
||||
asset_offset = 0
|
||||
if asset_index == number_of_assets do break
|
||||
asset_stream, asset_size = open_asset(
|
||||
w,
|
||||
entries[asset_index].relpath,
|
||||
)
|
||||
w.asset_table.asset_offsets[asset_index] = total_offset
|
||||
w.asset_table.asset_sizes[asset_index] = asset_size
|
||||
|
||||
} else if asset_remaining == chunk_remaining {
|
||||
|
||||
// Write asset to chunk
|
||||
write_chunk(chunk_stream, asset_stream, asset_remaining)
|
||||
total_offset += asset_remaining
|
||||
|
||||
// Open next asset
|
||||
asset_index += 1
|
||||
asset_offset = 0
|
||||
if asset_index == number_of_assets do break
|
||||
asset_stream, asset_size = open_asset(
|
||||
w,
|
||||
entries[asset_index].relpath,
|
||||
)
|
||||
w.asset_table.asset_offsets[asset_index] = total_offset
|
||||
w.asset_table.asset_sizes[asset_index] = asset_size
|
||||
|
||||
// Create new chunk
|
||||
chunk_index += 1
|
||||
chunk_offset = 0
|
||||
chunk_stream = create_new_chunk(
|
||||
w,
|
||||
output_path,
|
||||
chunk_index,
|
||||
allocator,
|
||||
)
|
||||
} else { // spillover to next chunk
|
||||
write_chunk(chunk_stream, asset_stream, chunk_remaining)
|
||||
asset_offset += chunk_remaining
|
||||
total_offset += chunk_remaining
|
||||
|
||||
// Create new chunk
|
||||
chunk_index += 1
|
||||
chunk_offset = 0
|
||||
chunk_stream = create_new_chunk(
|
||||
w,
|
||||
output_path,
|
||||
chunk_index,
|
||||
allocator,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@(private)
|
||||
create_new_chunk :: proc(
|
||||
w: ^Writer,
|
||||
output_path: string,
|
||||
chunk_index: int,
|
||||
allocator: runtime.Allocator,
|
||||
) -> io.Stream {
|
||||
// Remove old chunk
|
||||
chunk_path := compute_chunk_path(output_path, chunk_index, allocator)
|
||||
defer delete(chunk_path)
|
||||
if os.exists(chunk_path) {
|
||||
rm_err := os.remove(chunk_path)
|
||||
if rm_err != nil {
|
||||
fmt.eprintln("error: remove old chunk failed:", rm_err)
|
||||
os.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
if w.chunk_file != nil do os.close(w.chunk_file) // close previous
|
||||
|
||||
// Create new chunk
|
||||
cf_err: os.Error
|
||||
w.chunk_file, cf_err = os.create(chunk_path)
|
||||
if cf_err != nil {
|
||||
fmt.eprintln("error: create new chunk failed:", cf_err)
|
||||
os.exit(1)
|
||||
}
|
||||
|
||||
chunk_stream := os.to_stream(w.chunk_file)
|
||||
|
||||
return chunk_stream
|
||||
}
|
||||
|
||||
@(private)
|
||||
open_asset :: proc(w: ^Writer, path: string) -> (io.Stream, u64) {
|
||||
if w.asset_file != nil do os.close(w.asset_file) // close previous
|
||||
|
||||
// Open asset
|
||||
asset, open_err := os.open(path, {.Read})
|
||||
if open_err != nil {
|
||||
fmt.eprintln("error: asset open failed:", open_err)
|
||||
os.exit(1)
|
||||
}
|
||||
|
||||
// Asset size
|
||||
asset_stream := os.to_stream(asset)
|
||||
asset_size, size_err := io.size(asset_stream)
|
||||
if size_err != nil {
|
||||
fmt.eprintln("error: asset size failed:", size_err)
|
||||
os.exit(1)
|
||||
}
|
||||
|
||||
return asset_stream, cast(u64)asset_size
|
||||
}
|
||||
|
||||
@(private)
|
||||
write_chunk :: proc(dst: io.Stream, src: io.Stream, n: u64) {
|
||||
w, err := io.copy_n(dst, src, cast(i64)n)
|
||||
if err != nil {
|
||||
fmt.eprintln("error: chunk write error:", err)
|
||||
os.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package chunks
|
||||
|
||||
import "base:runtime"
|
||||
import "core:fmt"
|
||||
import "core:mem"
|
||||
import "core:os"
|
||||
|
||||
import "src:file"
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
write_metadata :: proc(
|
||||
w: ^Writer,
|
||||
output: ^os.File,
|
||||
entries: []file.FileEntry,
|
||||
size_per_chunk: u64 = SIZE_PER_CHUNK,
|
||||
compression: u16 = COMPRESSION,
|
||||
allocator := context.allocator,
|
||||
) {
|
||||
output_path := os.name(output)
|
||||
number_of_assets := cast(u32)len(entries)
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
chunk_path := compute_chunk_path(output_path, 0, allocator)
|
||||
defer delete(chunk_path)
|
||||
chunk_file, err := os.open(chunk_path, {.Read, .Write})
|
||||
if err != nil {
|
||||
fmt.eprintln("error: open chunk failed: ", err)
|
||||
os.exit(1)
|
||||
}
|
||||
defer os.close(chunk_file)
|
||||
|
||||
write_header(w, chunk_file, number_of_assets, size_per_chunk, compression)
|
||||
|
||||
write_asset_index(w, chunk_file, entries, number_of_assets, allocator)
|
||||
}
|
||||
|
||||
@(private)
|
||||
write_header :: proc(
|
||||
w: ^Writer,
|
||||
chunk_file: ^os.File,
|
||||
number_of_assets: u32,
|
||||
size_per_chunk: u64 = SIZE_PER_CHUNK,
|
||||
compression: u16 = COMPRESSION,
|
||||
) {
|
||||
last_offset :=
|
||||
w.asset_table.asset_offsets[len(w.asset_table.asset_offsets) - 1]
|
||||
last_size := w.asset_table.asset_sizes[len(w.asset_table.asset_sizes) - 1]
|
||||
total_size := last_offset + last_size
|
||||
|
||||
header := Header {
|
||||
magic_string = MAGIC_STRING,
|
||||
version = VERSION,
|
||||
compression = compression,
|
||||
size_per_chunk = size_per_chunk,
|
||||
total_size = total_size,
|
||||
number_of_assets = number_of_assets,
|
||||
}
|
||||
|
||||
// Write header
|
||||
header_bytes := mem.ptr_to_bytes(&header)
|
||||
n, err := os.write_at(chunk_file, header_bytes, 0)
|
||||
if err != nil {
|
||||
fmt.eprintln("error: chunk write error:", err)
|
||||
os.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@(private)
|
||||
write_asset_index :: proc(
|
||||
w: ^Writer,
|
||||
chunk_file: ^os.File,
|
||||
entries: []file.FileEntry,
|
||||
number_of_assets: u32,
|
||||
allocator: runtime.Allocator,
|
||||
) {
|
||||
index_size: u64 = size_of(AssetIndex) * cast(u64)number_of_assets
|
||||
|
||||
// ----------------------------------------
|
||||
|
||||
index_bytes := make([]u8, index_size, allocator)
|
||||
defer delete(index_bytes)
|
||||
|
||||
for f, i in entries {
|
||||
|
||||
// TODO: Move to cli validation so it happens sooner
|
||||
path_length := len(f.relpath)
|
||||
if path_length > 512 {
|
||||
fmt.eprintln("error: path exeeds maximum of 512:", f.relpath)
|
||||
os.exit(1)
|
||||
}
|
||||
|
||||
offset_in_bytes := size_of(AssetIndex) * cast(u64)i
|
||||
offset_pointer := mem.ptr_offset(&index_bytes[0], offset_in_bytes)
|
||||
|
||||
ai := cast(^AssetIndex)offset_pointer
|
||||
copy(ai.relpath[:PATH_SIZE], f.relpath)
|
||||
ai.size = w.asset_table.asset_sizes[i]
|
||||
ai.offset = w.asset_table.asset_offsets[i]
|
||||
}
|
||||
|
||||
assert(len(index_bytes) % size_of(AssetIndex) == 0) // clean multiple
|
||||
|
||||
// Write asset index
|
||||
n, err := os.write_at(
|
||||
chunk_file,
|
||||
index_bytes,
|
||||
cast(i64)w.asset_table.asset_index_offset,
|
||||
)
|
||||
if err != nil {
|
||||
fmt.eprintln("error: chunk write error:", err)
|
||||
os.exit(1)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user