gram/src/file/write_chunks.odin

package file

import "base:runtime"
import "core:fmt"
import "core:io"
import "core:mem"
import "core:os"
import "core:strconv"
import "core:strings"

MAGIC_STRING :: 0x514D21 // SINDRI
VERSION :: 1
COMPRESSION :: 0
SIZE_PER_CHUNK :: 2 * (1 << 30) // 2GiB
PATH_SIZE :: 512

// -----------------------------------------

Header :: struct #packed #all_or_none {
	magic_string:     u32,
	version:          u16,
	compression:      u16,
	size_per_chunk:   u64,
	total_size:       u64,
	number_of_assets: u32,
}

AssetIndex :: struct #packed #all_or_none {
	relpath: [PATH_SIZE]u8,
	size:    u64,
	offset:  u64,
}

Offsets :: struct {
	asset_index_offset: u64,
	data_offset:        u64,
	asset_offsets:      [dynamic]u64,
}

// -----------------------------------------

@(private)
offsets: Offsets

@(private = "file")
chunk_file: ^os.File

@(private = "file")
asset_file: ^os.File

// -----------------------------------------

compute_metadata_offsets :: proc(number_of_assets: u64) {
	// [ Header ][ []Asset Index ][ Data ]
	offsets.asset_index_offset = size_of(Header)
	offsets.data_offset =
		size_of(Header) + (size_of(AssetIndex) * number_of_assets)
}

// Returning `[]u8` is owned by the caller.
compute_header :: proc(
	entries: []FileEntry,
	compression: u16 = COMPRESSION,
	size_per_chunk: u64 = SIZE_PER_CHUNK,
	allocator := context.allocator,
) -> (
	header: Header,
	asset_index: []AssetIndex,
) {
	number_of_assets := cast(u32)len(entries)

	index_size: u64 = size_of(AssetIndex) * cast(u64)number_of_assets
	data_offset: u64 = size_of(Header) + index_size
	total_size: u64 = data_offset

	if data_offset > size_per_chunk {
		fmt.eprintln(
			"error: chunk size too small to hold metadata:",
			data_offset,
		)
		os.exit(1)
	}

	bytes := make([]u8, index_size, allocator)

	for f, i in entries {

		path_length := len(f.relpath)
		if path_length > 512 {
			fmt.eprintln("error: path exeeds maximum of 512:", f.relpath)
			os.exit(1)
		}

		offset_in_bytes := size_of(AssetIndex) * cast(u64)i
		offset_pointer := mem.ptr_offset(&bytes[0], offset_in_bytes)

		ai := cast(^AssetIndex)offset_pointer
		copy(ai.relpath[:PATH_SIZE], f.relpath)
		ai.size = cast(u64)f.size
		ai.offset = data_offset
		data_offset += ai.size

		total_size += ai.size
	}

	header = Header {
		magic_string     = MAGIC_STRING,
		version          = VERSION,
		compression      = compression,
		size_per_chunk   = size_per_chunk,
		total_size       = total_size,
		number_of_assets = number_of_assets,
	}

	assert(len(bytes) % size_of(AssetIndex) == 0) // clean multiple
	return header, mem.slice_data_cast([]AssetIndex, bytes) // transmute slice with proper size
}

write_header :: proc(
	output: ^os.File,
	header: Header,
	asset_index: []AssetIndex,
	allocator := context.allocator,
) {
	output_path := os.name(output)

	chunk_1, c1_err := os.join_path({output_path, "CHUNK1"}, allocator)
	if c1_err != nil {
		fmt.eprintln("error: failed to allocate path join:", c1_err)
		os.exit(1)
	}

	// Remove old chunk 1
	if os.exists(chunk_1) {
		rm_err := os.remove(chunk_1)
		if rm_err != nil {
			fmt.eprintln("error: failed to remove old chunk:", rm_err)
			os.exit(1)
		}
	}

	// Create chunk 1
	chunk_file_1, cf1_err := os.create(chunk_1)
	if cf1_err != nil {
		fmt.eprintln("error: failed to create chunk:", cf1_err)
		os.exit(1)
	}

	// Write header
	header := header // shadow parameter
	header_bytes := mem.ptr_to_bytes(&header)
	os.write_at(chunk_file_1, header_bytes, 0)

	// Write asset index
	asset_index := asset_index // shadow parameter
	index_bytes := mem.slice_data_cast([]u8, asset_index) // transmute slice with proper size
	os.write_at(chunk_file_1, index_bytes, size_of(Header))
}

write_chunks :: proc(
	output: ^os.File,
	size_per_chunk: u64 = SIZE_PER_CHUNK,
	entries: []FileEntry,
	allocator := context.allocator,
) {
	output_path := os.name(output)
	chunk_index := 0
	asset_index := 0
	chunk_offset: u64 = 0
	asset_offset: u64 = 0
	total_offset: u64 = 0
	number_of_assets := len(entries)
	reserve(&offsets.asset_offsets, number_of_assets)

	buf: [20]u8

	// ----------------------------------------

	chunk_stream := create_new_chunk(
		output_path,
		chunk_index,
		buf[:],
		allocator,
	)

	if (chunk_index == 0) {
		// Chunk 0 starts after metadata
		chunk_offset = offsets.data_offset
		total_offset = offsets.data_offset
	}

	// ----------------------------------------

	asset_stream, asset_size := open_asset(entries[asset_index].relpath)
	offsets.asset_offsets[asset_index] = total_offset

	// ----------------------------------------

	for {
		chunk_remaining := size_per_chunk - chunk_offset
		asset_remaining := asset_size - asset_offset

		io.seek(chunk_stream, cast(i64)chunk_offset, .Start)
		io.seek(asset_stream, cast(i64)asset_offset, .Start)

		if asset_remaining < chunk_remaining {

			// Write asset to chunk
			io.copy_n(chunk_stream, asset_stream, cast(i64)asset_remaining)
			chunk_offset += asset_remaining
			total_offset += asset_remaining

			// Open new asset
			asset_index += 1
			asset_offset = 0
			if asset_index == number_of_assets do break
			asset_stream, asset_size = open_asset(entries[asset_index].relpath)
			offsets.asset_offsets[asset_index] = total_offset

		} else if asset_remaining == chunk_remaining {

			// Write asset to chunk
			io.copy_n(chunk_stream, asset_stream, cast(i64)asset_remaining)
			total_offset += asset_remaining

			// Open new asset
			asset_index += 1
			asset_offset = 0
			if asset_index == number_of_assets do break
			asset_stream, asset_size = open_asset(entries[asset_index].relpath)
			offsets.asset_offsets[asset_index] = total_offset

			// Create new chunk
			chunk_index += 1
			chunk_offset = 0
			chunk_stream = create_new_chunk(
				output_path,
				chunk_index,
				buf[:],
				allocator,
			)
		} else {
			io.copy_n(chunk_stream, asset_stream, cast(i64)chunk_remaining)
			asset_offset += chunk_remaining
			total_offset += chunk_remaining

			// Create new chunk
			chunk_index += 1
			chunk_offset = 0
			chunk_stream = create_new_chunk(
				output_path,
				chunk_index,
				buf[:],
				allocator,
			)
		}
	}

	// ----------------------------------------

	// Cleaup leftover
	os.close(chunk_file)
	os.close(asset_file)
}

@(private)
create_new_chunk :: proc(
	output_path: string,
	chunk_index: int,
	buf: []u8,
	allocator: runtime.Allocator,
) -> io.Stream {
	// Get chunk path
	chunk_index_str := strconv.write_int(buf[:], cast(i64)chunk_index, 10)
	chunk_name := strings.concatenate({"CHUNK", chunk_index_str}, allocator)
	defer delete(chunk_name, allocator)
	chunk, c_err := os.join_path({output_path, chunk_name}, allocator)
	if c_err != nil {
		fmt.eprintln("error: chunk join path failed:", c_err)
		os.exit(1)
	}
	defer delete(chunk, allocator)

	// Remove old chunk
	if os.exists(chunk) {
		rm_err := os.remove(chunk)
		if rm_err != nil {
			fmt.eprintln("error: remove old chunk failed:", rm_err)
			os.exit(1)
		}
	}

	if chunk_file != nil do os.close(chunk_file) // close previous

	// Create new chunk
	cf_err: os.Error
	chunk_file, cf_err = os.create(chunk)
	if cf_err != nil {
		fmt.eprintln("error: create new chunk failed:", cf_err)
		os.exit(1)
	}

	chunk_stream := os.to_stream(chunk_file)

	return chunk_stream
}

@(private)
open_asset :: proc(path: string) -> (io.Stream, u64) {
	if asset_file != nil do os.close(asset_file) // close previous

	// Open asset
	asset, open_err := os.open(path, {.Read})
	if open_err != nil {
		fmt.eprintln("error: asset open failed:", open_err)
		os.exit(1)
	}

	// Asset size
	asset_stream := os.to_stream(asset)
	asset_size, size_err := io.size(asset_stream)
	if size_err != nil {
		fmt.eprintln("error: asset size failed:", size_err)
		os.exit(1)
	}

	return asset_stream, cast(u64)asset_size
}