riyyi/sindri
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add header computation function

Browse Source
This commit is contained in:
Riyyi
2026-08-03 09:13:34 +02:00
parent 87d20f2ae5
commit 34f7b70039
5 changed files with 123 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/design.md
+21 -21
View File
@@ -1,24 +1,24 @@
# Design
The data layout design is very simple. It consists of data split into chunks
with the first chunk holding the metadata for the structure and files.
with the first chunk holding the metadata for the structure and assets.
## Spec
```
METADATA DATA
|---------------------------------------------------------------------------||------|
METADATA DATA
|----------------------------------------------------------------------------------------||------|
[ MAGIC ][ VERSION ][ COMPRESSION ][ SIZE_PER_CHUNK ][ TOTAL_SIZE ][ #FILES ][ DATA ] # chunk 1
[ DATA ] # chunk 2
[ DATA ] # etc..
[ DATA ] # last chunk
[ MAGIC ][ VERSION ][ COMPRESSION ][ SIZE_PER_CHUNK ][ TOTAL_SIZE ][ #ASSETS ][ []ASSETS ][ DATA ] # chunk 1
[ DATA ] # chunk 2
[ DATA ] # etc..
[ DATA ] # last chunk
```
### Magic string
Magic string to detect if the read file is what we expect it is.
Represented in 3 bytes.
Represented in 4 bytes.
Value is `0x514D21`, the closest approximation of "Sindri".
@@ -32,7 +32,7 @@ Value is `1` currently.
### Compression
Wether LZ4 compression is enabled/disabled.
Represented in `1 byte`.
Represented in `2 byte`.
Value is configurable, defaulting to `0`.
@@ -54,30 +54,30 @@ Value is configurable, defaulting to `2GiB`.
The total size in bytes of all chunks combined.
Represented in `8 bytes`.
### Number of files
### Number of assets
The total number of files in the pack.
The total number of assets in the pack.
Represented in `4 bytes`.
### File
### []Asset
Piece of data stored, consisting of 3 parts:
Index to piece of stored data, consisting of 3 parts:
- Path
- Offset
- Size
- Offset
#### Path
Path of the file as if it were on the filesystem.
Path of the asset as if it were on the filesystem.
Represented in `512 bytes`.
#### Offset
The offset to the file, starting from the true 0th byte of the first chunk.
Represented in `8 bytes`.
#### Size
Size of the file.
Size of the asset.
Represented in `8 bytes`.
#### Offset
The offset to the asset, starting from the true 0th byte of the first chunk.
Represented in `8 bytes`.
src/cli/cli.odin
+2 -2
View File
@@ -7,8 +7,8 @@ import "core:strings"
Options :: struct {
verbose: bool `args:"name=verbose" usage:"Enable verbose output"`,
compression: u8 `args:"name=compression" usage:"Compression level, 0-12"`,
size: i64 `args:"name=size" usage:"Chunk size in bytes"`,
compression: u16 `args:"name=compression" usage:"Compression level, 0-12"`,
size: u64 `args:"name=size" usage:"Chunk size in bytes"`,
input: ^os.File `args:"name=input,file=r,required" usage:"Path to input directory"`,
output: ^os.File `args:"name=output,file=wc" usage:"Path to ouput directory"`,
// overflow: [dynamic]string,
src/file/header.odin
+89
View File
@@ -0,0 +1,89 @@
package file
import "core:fmt"
import "core:mem"
import "core:os"
MAGIC_STRING :: 0x514D21 // SINDRI
VERSION :: 1
COMPRESSION :: 0
SIZE_PER_CHUNK :: 2 * (1 << 30) // 2GiB
PATH_SIZE :: 512
// -----------------------------------------
Header :: struct #packed #all_or_none {
magic_string: u32,
version: u16,
compression: u16,
size_per_chunk: u64,
total_size: u64,
number_of_assets: u32,
// asset_index: []AssetIndex,
}
AssetIndex :: struct #packed #all_or_none {
relpath: [PATH_SIZE]u8,
size: u64,
offset: u64,
}
// -----------------------------------------
// remove :: proc(name: string) -> Error {…}
// create :: proc(name: string) -> (^File, Error) {…}
// write_at :: proc(f: ^File, p: []u8, offset: i64) -> (n: int, err: Error) {…}
// Returning `[]u8` is owned by the caller.
compute_header :: proc(
entries: [dynamic]FileEntry,
compression: u16 = COMPRESSION,
size_per_chunk: u64 = SIZE_PER_CHUNK,
allocator := context.allocator,
) -> (
header: Header,
asset_index: []AssetIndex,
) {
number_of_assets := cast(u32)len(entries)
index_size: u64 = size_of(AssetIndex) * cast(u64)number_of_assets
data_offset: u64 = size_of(Header) + index_size
total_size: u64 = data_offset
bytes := make([]u8, index_size, allocator)
for f, i in entries {
path_length := len(f.relpath)
if path_length > 512 {
fmt.eprintln("error: path exeeds maximum of 512:", f.relpath)
os.exit(1)
}
offset_in_bytes := size_of(AssetIndex) * cast(u64)i
offset_pointer := mem.ptr_offset(
&bytes[0],
offset_in_bytes,
)
ai := cast(^AssetIndex)offset_pointer
copy(ai.relpath[:PATH_SIZE], f.relpath)
ai.size = cast(u64)f.size
ai.offset = data_offset
data_offset += ai.size
total_size += ai.size
}
header = Header {
magic_string = MAGIC_STRING,
version = VERSION,
compression = compression,
size_per_chunk = size_per_chunk,
total_size = total_size,
number_of_assets = number_of_assets,
}
assert(len(bytes) % size_of(AssetIndex) == 0) // clean multiple
return header, mem.slice_data_cast([]AssetIndex, bytes) // transmute slice with proper size
}
src/file/list_dir.odin
+7 -10
View File
@@ -7,7 +7,7 @@ import "core:strings"
// -----------------------------------------
DirectoryEntry :: struct #all_or_none {
FileEntry :: struct #all_or_none {
relpath: string, // NOTE: owning string
size: i64,
}
@@ -18,7 +18,7 @@ DirectoryEntry :: struct #all_or_none {
list_dir_recursive :: proc(
f: ^os.File,
allocator := context.allocator,
) -> [dynamic]DirectoryEntry {
) -> [dynamic]FileEntry {
path := os.name(f)
return list_dir_recursive_by_path_impl(path, allocator)
}
@@ -27,7 +27,7 @@ list_dir_recursive :: proc(
list_dir_recursive_by_path :: proc(
path: string,
allocator := context.allocator,
) -> [dynamic]DirectoryEntry {
) -> [dynamic]FileEntry {
return list_dir_recursive_by_path_impl(path, allocator)
}
@@ -35,13 +35,13 @@ list_dir_recursive_by_path :: proc(
list_dir_recursive_by_path_impl :: proc(
path: string,
allocator: runtime.Allocator,
) -> [dynamic]DirectoryEntry {
) -> [dynamic]FileEntry {
if !os.is_dir(path) {
fmt.eprintln("error: path is not a directory:", path)
os.exit(1)
}
result := make([dynamic]DirectoryEntry, allocator)
result := make([dynamic]FileEntry, allocator)
queue := make([dynamic]string, 0, 1, allocator) // reserve 1 slot
defer delete(queue) // dynamic array embeds allocator
@@ -57,7 +57,7 @@ list_dir_recursive_by_path_impl :: proc(
@(private)
list_dir_queue :: proc(
queue: ^[dynamic]string,
result: ^[dynamic]DirectoryEntry,
result: ^[dynamic]FileEntry,
working_dir: string,
allocator: runtime.Allocator,
) {
@@ -99,10 +99,7 @@ list_dir_queue :: proc(
if !was_allocation {
relpath = strings.clone(relpath, result^.allocator)
}
append(
result,
DirectoryEntry{relpath = relpath, size = entry.size},
)
append(result, FileEntry{relpath = relpath, size = entry.size})
}
case: // skip other types
}
src/main.odin
+4
View File
@@ -24,4 +24,8 @@ main :: proc() {
entries := file.list_dir_recursive_by_path("./src")
fmt.println("entries:", entries)
header, asset_index_array := file.compute_header(entries)
fmt.println("header:", header)
fmt.println("aia:", asset_index_array)
}