Split reader into metadata/assets
This commit is contained in:
Riyyi
@@ -0,0 +1,96 @@
|
||||
package chunks
|
||||
|
||||
import "base:runtime"
|
||||
import "core:fmt"
|
||||
import "core:os"
|
||||
import "core:strings"
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
file_exists :: proc(r: ^Reader, path: string) -> (bool, u32) {
|
||||
for i in 0 ..< r.header.number_of_assets {
|
||||
str := string(r.indices[i].relpath[:])
|
||||
relpath := strings.trim_right_null(str)
|
||||
if path == relpath do return true, i
|
||||
}
|
||||
|
||||
return false, 0
|
||||
}
|
||||
|
||||
// Returning `[]u8` is owned by the caller.
|
||||
read_file :: proc(
|
||||
r: ^Reader,
|
||||
path: string,
|
||||
allocator := context.allocator,
|
||||
) -> (
|
||||
bytes: []u8,
|
||||
err: Error,
|
||||
) {
|
||||
|
||||
// Check filesystem first, so chunks can be overriden
|
||||
if os.exists(path) {
|
||||
// Open file
|
||||
fmt.println("searching:", path)
|
||||
file := os.open(path, {.Read}) or_return
|
||||
defer os.close(file)
|
||||
|
||||
// Read
|
||||
file_size := os.file_size(file) or_return
|
||||
bytes = make([]u8, file_size, allocator)
|
||||
os.read(file, bytes) or_return
|
||||
|
||||
return bytes, nil
|
||||
}
|
||||
|
||||
// Check chunk
|
||||
if found, i := file_exists(r, path); found {
|
||||
output_path := os.name(r.output)
|
||||
entry := r.indices[i]
|
||||
|
||||
// Find correct chunk
|
||||
chunk_index := entry.offset / r.header.size_per_chunk
|
||||
|
||||
// Find offset within this chunk
|
||||
chunk_offset := entry.offset % r.header.size_per_chunk
|
||||
|
||||
bytes = make([]u8, entry.size, allocator)
|
||||
n_left: u64
|
||||
n_read: u64
|
||||
|
||||
for entry.size > n_read {
|
||||
// Chunk path
|
||||
chunk_path := compute_chunk_path(
|
||||
output_path,
|
||||
cast(int)chunk_index,
|
||||
allocator,
|
||||
)
|
||||
defer delete(chunk_path, allocator)
|
||||
|
||||
// Open chunk
|
||||
file := os.open(chunk_path, {.Read}) or_return
|
||||
defer os.close(file)
|
||||
|
||||
// Calculate how much of the current chunk should be read
|
||||
if entry.size > n_read + r.header.size_per_chunk - chunk_offset {
|
||||
n_left = r.header.size_per_chunk - chunk_offset
|
||||
} else {
|
||||
n_left = entry.size - n_read
|
||||
}
|
||||
|
||||
// Read
|
||||
n_read += cast(u64)os.read_at(
|
||||
file,
|
||||
bytes[n_read:n_read + n_left],
|
||||
cast(i64)chunk_offset,
|
||||
) or_return
|
||||
|
||||
// Overflow to next chunk
|
||||
chunk_index += 1
|
||||
chunk_offset = 0
|
||||
}
|
||||
|
||||
return bytes, nil
|
||||
}
|
||||
|
||||
return nil, .File_Not_Exist
|
||||
}
|
||||
+28
-157
@@ -1,184 +1,55 @@
|
||||
package chunks
|
||||
|
||||
import "base:runtime"
|
||||
import "core:fmt"
|
||||
import "core:io"
|
||||
import "core:mem"
|
||||
import "core:os"
|
||||
import "core:strings"
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
Reader :: struct {}
|
||||
|
||||
Read_Error :: enum u8 {
|
||||
None = 0,
|
||||
File_Not_A_Chunk = 1,
|
||||
File_Not_Exist = 2,
|
||||
Unimplemented = 127,
|
||||
Okay = None,
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
@(private)
|
||||
read_header :: proc(
|
||||
chunk1: ^os.File,
|
||||
allocator := context.allocator,
|
||||
r: ^Reader,
|
||||
chunk: ^os.File,
|
||||
allocator: runtime.Allocator,
|
||||
) -> (
|
||||
Header,
|
||||
Error,
|
||||
) {
|
||||
return read_header_impl(chunk1, allocator)
|
||||
}
|
||||
|
||||
read_header_by_path :: proc(
|
||||
chunk1: string,
|
||||
allocator := context.allocator,
|
||||
) -> (
|
||||
header: Header,
|
||||
err: Error,
|
||||
) {
|
||||
chunk1_file := os.open(chunk1, {.Read}) or_return
|
||||
defer os.close(chunk1_file)
|
||||
chunk1_stream := os.to_stream(chunk)
|
||||
|
||||
return read_header_impl(chunk1_file, allocator)
|
||||
header_bytes := mem.ptr_to_bytes(&r.header)
|
||||
io.read_at(chunk1_stream, header_bytes, 0) or_return
|
||||
|
||||
if r.header.magic_string != MAGIC_STRING || r.header.version != 1 {
|
||||
return .File_Not_A_Chunk
|
||||
}
|
||||
|
||||
if r.header.compression != 0 {
|
||||
return .Unimplemented
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@(private)
|
||||
read_header_impl :: proc(
|
||||
chunk1: ^os.File,
|
||||
allocator: runtime.Allocator,
|
||||
) -> (
|
||||
header: Header,
|
||||
err: Error,
|
||||
) {
|
||||
chunk1_stream := os.to_stream(chunk1)
|
||||
|
||||
header_bytes := mem.ptr_to_bytes(&header)
|
||||
io.read_at(chunk1_stream, header_bytes, 0) or_return
|
||||
|
||||
if header.magic_string != MAGIC_STRING || header.version != 1 {
|
||||
return {}, .File_Not_A_Chunk
|
||||
}
|
||||
|
||||
if header.compression != 0 {
|
||||
return {}, .Unimplemented
|
||||
}
|
||||
|
||||
return header, nil
|
||||
}
|
||||
|
||||
// Returning `[]Asset_Index` is owned by the caller.
|
||||
read_asset_index :: proc(
|
||||
chunk1: ^os.File,
|
||||
header: Header,
|
||||
r: ^Reader,
|
||||
chunk: ^os.File,
|
||||
allocator := context.allocator,
|
||||
) -> (
|
||||
indices: []Asset_Index,
|
||||
err: Error,
|
||||
) {
|
||||
chunk1_stream := os.to_stream(chunk1)
|
||||
chunk1_stream := os.to_stream(chunk)
|
||||
|
||||
indices = make([]Asset_Index, header.number_of_assets, allocator) or_return
|
||||
indices := make(
|
||||
[]Asset_Index,
|
||||
r.header.number_of_assets,
|
||||
allocator,
|
||||
) or_return
|
||||
index_bytes := mem.slice_to_bytes(indices)
|
||||
io.read_at(chunk1_stream, index_bytes, size_of(Header)) or_return
|
||||
|
||||
return indices, nil
|
||||
}
|
||||
|
||||
file_exists :: proc(
|
||||
header: Header,
|
||||
indices: []Asset_Index,
|
||||
path: string,
|
||||
) -> (
|
||||
bool,
|
||||
u32,
|
||||
) {
|
||||
for i in 0 ..< header.number_of_assets {
|
||||
str := string(indices[i].relpath[:])
|
||||
relpath := strings.trim_right_null(str)
|
||||
if path == relpath do return true, i
|
||||
}
|
||||
|
||||
return false, 0
|
||||
}
|
||||
|
||||
// Returning `[]u8` is owned by the caller.
|
||||
read_file :: proc(
|
||||
header: Header,
|
||||
indices: []Asset_Index,
|
||||
output: ^os.File,
|
||||
path: string,
|
||||
allocator := context.allocator,
|
||||
) -> (
|
||||
bytes: []u8,
|
||||
err: Error,
|
||||
) {
|
||||
|
||||
// Check filesystem first, so chunks can be overriden
|
||||
if os.exists(path) {
|
||||
// Open file
|
||||
fmt.println("searching:", path)
|
||||
file := os.open(path, {.Read}) or_return
|
||||
defer os.close(file)
|
||||
|
||||
// Read
|
||||
file_size := os.file_size(file) or_return
|
||||
bytes = make([]u8, file_size, allocator)
|
||||
os.read(file, bytes) or_return
|
||||
|
||||
return bytes, nil
|
||||
}
|
||||
|
||||
// Check chunk
|
||||
if found, i := file_exists(header, indices, path); found {
|
||||
output_path := os.name(output)
|
||||
entry := indices[i]
|
||||
|
||||
// Find correct chunk
|
||||
chunk_index := entry.offset / header.size_per_chunk
|
||||
|
||||
// Find offset within this chunk
|
||||
chunk_offset := entry.offset % header.size_per_chunk
|
||||
|
||||
bytes = make([]u8, entry.size, allocator)
|
||||
n_left: u64
|
||||
n_read: u64
|
||||
|
||||
for entry.size > n_read {
|
||||
// Chunk path
|
||||
chunk_path := compute_chunk_path(
|
||||
output_path,
|
||||
cast(int)chunk_index,
|
||||
allocator,
|
||||
)
|
||||
defer delete(chunk_path, allocator)
|
||||
|
||||
// Open chunk
|
||||
file := os.open(chunk_path, {.Read}) or_return
|
||||
defer os.close(file)
|
||||
|
||||
// Calculate how much of the current chunk should be read
|
||||
if entry.size > n_read + header.size_per_chunk - chunk_offset {
|
||||
n_left = header.size_per_chunk - chunk_offset
|
||||
} else {
|
||||
n_left = entry.size - n_read
|
||||
}
|
||||
|
||||
// Read
|
||||
n_read += cast(u64)os.read_at(
|
||||
file,
|
||||
bytes[n_read:n_read + n_left],
|
||||
cast(i64)chunk_offset,
|
||||
) or_return
|
||||
|
||||
// Overflow to next chunk
|
||||
chunk_index += 1
|
||||
chunk_offset = 0
|
||||
}
|
||||
|
||||
return bytes, nil
|
||||
}
|
||||
|
||||
return nil, .File_Not_Exist
|
||||
r.indices = indices
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package chunks
|
||||
|
||||
import "base:runtime"
|
||||
import "core:os"
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
Reader :: struct {
|
||||
output: ^os.File,
|
||||
header: Header,
|
||||
indices: []Asset_Index,
|
||||
}
|
||||
|
||||
Read_Error :: enum u8 {
|
||||
None = 0,
|
||||
File_Not_A_Chunk = 1,
|
||||
File_Not_Exist = 2,
|
||||
Unimplemented = 127,
|
||||
Okay = None,
|
||||
}
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
reader_init :: proc(
|
||||
output: ^os.File,
|
||||
allocator := context.allocator,
|
||||
) -> (
|
||||
r: Reader,
|
||||
err: Error,
|
||||
) {
|
||||
output_path := os.name(output)
|
||||
|
||||
// Chunk path
|
||||
chunk_path := compute_chunk_path(output_path, 0, allocator)
|
||||
defer delete(chunk_path, allocator)
|
||||
|
||||
// Chunk open
|
||||
chunk_file := os.open(chunk_path, {.Read}) or_return
|
||||
defer os.close(chunk_file)
|
||||
|
||||
// Chunk read
|
||||
read_header(&r, chunk_file, allocator) or_return
|
||||
read_asset_index(&r, chunk_file, allocator) or_return
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
reader_destroy :: proc(r: ^Reader) {
|
||||
delete(r.indices)
|
||||
}
|
||||
+12
-29
@@ -25,37 +25,20 @@ main :: proc() {
|
||||
w := chunks.writer_init(cast(u64)len(entries))
|
||||
defer chunks.writer_destroy(&w)
|
||||
|
||||
// chunks.write_assets(
|
||||
// &w,
|
||||
// opts.output,
|
||||
// entries[:],
|
||||
// opts.size,
|
||||
// opts.compression,
|
||||
// )
|
||||
//
|
||||
// chunks.write_metadata(
|
||||
// &w,
|
||||
// opts.output,
|
||||
// entries[:],
|
||||
// opts.size,
|
||||
// opts.compression,
|
||||
// )
|
||||
|
||||
header, err := chunks.read_header_by_path("./build/CHUNK0")
|
||||
chunk0, o_err := os.open("./build/CHUNK0")
|
||||
asset_index, a_err := chunks.read_asset_index(chunk0, header)
|
||||
fmt.println(
|
||||
"found: ",
|
||||
chunks.file_exists(header, asset_index, "not_exists"),
|
||||
)
|
||||
fmt.println(
|
||||
"found: ",
|
||||
chunks.file_exists(header, asset_index, "./toby/engage"),
|
||||
chunks.write_assets(
|
||||
&w,
|
||||
opts.output,
|
||||
entries[:],
|
||||
opts.size,
|
||||
opts.compression,
|
||||
)
|
||||
|
||||
fmt.printf(
|
||||
"file: %c\n",
|
||||
chunks.read_file(header, asset_index, opts.output, "./toby/engage"),
|
||||
chunks.write_metadata(
|
||||
&w,
|
||||
opts.output,
|
||||
entries[:],
|
||||
opts.size,
|
||||
opts.compression,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user