Skip to content

Commit eb6824f

Browse files
thesprockeeclaude
andauthored
Add lossless codec pipeline for extract/pack round-trips (#16)
* Add lossless codec pipeline for extract/pack round-trips Extraction now decodes package payloads to source-friendly files; packing re-encodes them back to the original wire format. The round-trip is lossless: no data is recompressed or reformatted beyond adding/stripping headers. Decode (extract): TypeRawBCTexture → .dds (DDS header prepended from paired TextureMetadata) TypeDDSTexture → .dds (verbatim) TypeTextureMetadata → .tmeta (renamed from .meta to avoid sidecar collision) Unknown types → sniff magic bytes → .ogg/.wav/.dds/.png/.json/etc Sidecars (.meta) always written for every file — no name table required Encode (pack/build): .dds with TypeRawBCTexture sidecar → strip DDS header → raw BC payload All other files → verbatim pkg/manifest/decode.go — sniffExtension, decodeRawBCTexture, parseTextureMetadata pkg/manifest/encode.go — encodeFile, stripDDSHeader pkg/manifest/package.go — two-pass extraction (pre-collect TextureMetadata), per-file decode, always-write sidecars pkg/manifest/builder.go — encodeFile call before frame packing pkg/naming/type_mapper — TypeRawBCTexture → .dds, TypeTextureMetadata → .tmeta Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * Add comprehensive test coverage for scanner, sidecar, patch, codec, and CLI Tests cover: - Scanner: sidecar layout, hex layout, .meta skipping, negative symbols, bug tests for base-10 hex parsing and file extension handling - Sidecar: round-trip, hex formatting, negative symbols, malformed JSON, .meta extension collision documentation - Patch: deep copy, basic patch with decompression verification, insert before terminator, header updates, not-found errors, FrameIndex shift corruption bug test - Decode: sniffExtension all formats, DDS header prepend, nil meta, size patching - Encode: DDS header stripping (standard + DX10), non-texture passthrough, lossless encode/decode round-trip - CLI: detectMagic with duplicate RIFF bug test, Shannon entropy, formatBytes, parseHex, parseHexSymbol, copyFile, countFiles, countKnownTypes 106 test cases total (103 pass, 3 skip confirming known bugs). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: resolve all shared bugs across codec pipeline - fix(manifest): use base-16 parsing for hex symbol filenames in scanner - fix(manifest): strip file extensions before parsing symbols in scanner - fix(manifest): change sidecar extension from .meta to .evrmeta to avoid collision with TypeTextureMetadata files - fix(manifest): shift FrameIndex for all affected entries when inserting frames in PatchFile, preventing data corruption - fix(naming): use uint64 cast in TypeName to correctly format negative symbol values as 16-digit hex - fix(evrtools): replace duplicate WAV/RIFF magic entries with single RIFF container signature in analyze mode - fix(hash): remove dead test assertion block in SNSMessageHash test - feat(evrtools): add search and patch modes to CLI Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent ffd570d commit eb6824f

26 files changed

Lines changed: 3256 additions & 61 deletions

cmd/evrtools/analyze.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ var magicSignatures = []struct {
2020
}{
2121
{"DDS texture", 0, []byte{0x44, 0x44, 0x53, 0x20}}, // "DDS "
2222
{"OGG audio", 0, []byte{0x4F, 0x67, 0x67, 0x53}}, // "OggS"
23-
{"WAV audio", 0, []byte{0x52, 0x49, 0x46, 0x46}}, // "RIFF"
2423
{"PNG image", 0, []byte{0x89, 0x50, 0x4E, 0x47}}, // "\x89PNG"
2524
{"JPEG image", 0, []byte{0xFF, 0xD8, 0xFF}}, // JPEG SOI
2625
{"JSON text", 0, []byte{0x7B}}, // "{"
@@ -30,7 +29,7 @@ var magicSignatures = []struct {
3029
{"Protobuf", 0, []byte{0x0A}}, // common protobuf field tag
3130
{"EXE/DLL", 0, []byte{0x4D, 0x5A}}, // "MZ"
3231
{"RAD video", 0, []byte{0x42, 0x49, 0x4B, 0x69}}, // "BIKi" (Bink)
33-
{"RIFF generic", 0, []byte{0x52, 0x49, 0x46, 0x46}}, // "RIFF"
32+
{"RIFF container", 0, []byte{0x52, 0x49, 0x46, 0x46}}, // "RIFF"
3433
{"FLAC audio", 0, []byte{0x66, 0x4C, 0x61, 0x43}}, // "fLaC"
3534
{"MP3 audio", 0, []byte{0xFF, 0xFB}}, // MP3 sync
3635
{"Null-padded", 0, []byte{0x00, 0x00, 0x00, 0x00}}, // all zeros

cmd/evrtools/analyze_test.go

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,10 @@ func TestDetectMagic_AllSignatures(t *testing.T) {
3838
}
3939
}
4040

41-
// TestDetectMagic_DuplicateRIFF demonstrates a bug: any RIFF-based format
42-
// (e.g., AVI with bytes "RIFF....AVI ") is misclassified as "WAV audio"
43-
// because the magic table checks only the first 4 bytes ("RIFF") and the
44-
// "WAV audio" entry appears before the "RIFF generic" entry. The table
45-
// does not inspect the RIFF sub-format bytes at offset 8.
46-
func TestDetectMagic_DuplicateRIFF(t *testing.T) {
41+
// TestDetectMagic_RIFFContainer verifies that all RIFF-based formats
42+
// (WAV, AVI, etc.) are classified as "RIFF container" since sub-format
43+
// detection is handled by sniffExtension in decode.go.
44+
func TestDetectMagic_RIFFContainer(t *testing.T) {
4745
// Construct a RIFF/AVI header: "RIFF" + 4-byte size + "AVI "
4846
avi := []byte{
4947
0x52, 0x49, 0x46, 0x46, // "RIFF"
@@ -52,12 +50,8 @@ func TestDetectMagic_DuplicateRIFF(t *testing.T) {
5250
}
5351

5452
got := detectMagic(avi)
55-
56-
// The correct classification would be something like "RIFF generic" or
57-
// "AVI video", but because the WAV entry matches first on the shared
58-
// 4-byte prefix, we get "WAV audio" for non-WAV RIFF data.
59-
if got != "WAV audio" {
60-
t.Errorf("expected bug: detectMagic(AVI data) = %q, want %q (bug: RIFF sub-format not checked)", got, "WAV audio")
53+
if got != "RIFF container" {
54+
t.Errorf("detectMagic(AVI data) = %q, want %q", got, "RIFF container")
6155
}
6256
}
6357

cmd/evrtools/main.go

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,17 @@ var (
2323
verbose bool
2424
diffManifestA string
2525
diffManifestB string
26+
wordlistPath string
27+
searchHash string
28+
searchType string
29+
searchName string
30+
patchInput string
31+
patchType string
32+
patchFile string
2633
)
2734

2835
func init() {
29-
flag.StringVar(&mode, "mode", "", "Operation mode: extract, build, inventory, analyze, diff")
36+
flag.StringVar(&mode, "mode", "", "Operation mode: extract, build, inventory, analyze, diff, search, patch")
3037
flag.StringVar(&packageName, "package", "", "Package name (e.g., 48037dc70b0ecab2)")
3138
flag.StringVar(&dataDir, "data", "", "Path to _data directory containing manifests/packages")
3239
flag.StringVar(&inputDir, "input", "", "Input directory (inventory/analyze/build mode)")
@@ -37,6 +44,13 @@ func init() {
3744
flag.BoolVar(&verbose, "verbose", false, "Print detailed file list (diff mode)")
3845
flag.StringVar(&diffManifestA, "manifest-a", "", "First manifest path (diff mode)")
3946
flag.StringVar(&diffManifestB, "manifest-b", "", "Second manifest path (diff mode)")
47+
flag.StringVar(&wordlistPath, "wordlist", "", "Path to wordlist file for named extraction")
48+
flag.StringVar(&searchHash, "search-hash", "", "Search by file/type symbol hash (search mode)")
49+
flag.StringVar(&searchType, "search-type", "", "Filter by type symbol hash (search mode)")
50+
flag.StringVar(&searchName, "search-name", "", "Filter by filename glob pattern (search mode)")
51+
flag.StringVar(&patchInput, "patch-input", "", "Replacement file path (patch mode)")
52+
flag.StringVar(&patchType, "patch-type", "", "Type symbol hex to patch (patch mode)")
53+
flag.StringVar(&patchFile, "patch-file", "", "File symbol hex to patch (patch mode)")
4054
}
4155

4256
func main() {
@@ -75,6 +89,10 @@ func run() error {
7589
return runAnalyze()
7690
case "diff":
7791
return runDiff()
92+
case "search":
93+
return runSearch()
94+
case "patch":
95+
return runPatch()
7896
default:
7997
return fmt.Errorf("unknown mode: %s", mode)
8098
}
@@ -97,16 +115,20 @@ func validateFlags() error {
97115
if packageName == "" {
98116
packageName = "package"
99117
}
100-
case "inventory", "analyze":
118+
case "inventory", "analyze", "search":
101119
if inputDir == "" {
102120
return fmt.Errorf("%s mode requires -input", mode)
103121
}
122+
case "patch":
123+
if dataDir == "" || packageName == "" || patchInput == "" || patchType == "" || patchFile == "" {
124+
return fmt.Errorf("patch mode requires -data, -package, -patch-input, -patch-type, -patch-file")
125+
}
104126
case "diff":
105127
if diffManifestA == "" || diffManifestB == "" {
106128
return fmt.Errorf("diff mode requires -manifest-a and -manifest-b")
107129
}
108130
default:
109-
return fmt.Errorf("mode must be one of: extract, build, inventory, analyze, diff")
131+
return fmt.Errorf("mode must be one of: extract, build, inventory, analyze, diff, search, patch")
110132
}
111133

112134
return nil

cmd/evrtools/patch.go

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"io"
6+
"os"
7+
"path/filepath"
8+
"strconv"
9+
10+
"github.com/EchoTools/evrFileTools/pkg/manifest"
11+
)
12+
13+
func runPatch() error {
14+
// 1. Read manifest from dataDir/manifests/packageName.
15+
manifestPath := filepath.Join(dataDir, "manifests", packageName)
16+
m, err := manifest.ReadFile(manifestPath)
17+
if err != nil {
18+
return fmt.Errorf("read manifest: %w", err)
19+
}
20+
fmt.Printf("Manifest loaded: %d files in %d packages\n", m.FileCount(), m.PackageCount())
21+
22+
// 2. Read replacement file from patchInput.
23+
data, err := os.ReadFile(patchInput)
24+
if err != nil {
25+
return fmt.Errorf("read patch input file %s: %w", patchInput, err)
26+
}
27+
fmt.Printf("Replacement file read: %d bytes\n", len(data))
28+
29+
// 3. Parse patchType and patchFile hex strings.
30+
typeSymbol, err := parseHexSymbol(patchType)
31+
if err != nil {
32+
return fmt.Errorf("parse -patch-type %q: %w", patchType, err)
33+
}
34+
fileSymbol, err := parseHexSymbol(patchFile)
35+
if err != nil {
36+
return fmt.Errorf("parse -patch-file %q: %w", patchFile, err)
37+
}
38+
39+
// 4. Copy original package files to outputDir/packages/.
40+
srcPkgDir := filepath.Join(dataDir, "packages")
41+
dstPkgDir := filepath.Join(outputDir, "packages")
42+
if err := os.MkdirAll(dstPkgDir, 0755); err != nil {
43+
return fmt.Errorf("create output packages dir: %w", err)
44+
}
45+
46+
if err := copyPackageFiles(srcPkgDir, dstPkgDir, packageName, m.PackageCount()); err != nil {
47+
return fmt.Errorf("copy package files: %w", err)
48+
}
49+
fmt.Printf("Copied %d package file(s) to %s\n", m.PackageCount(), dstPkgDir)
50+
51+
// 5. Call manifest.PatchFile with the copied package base path.
52+
pkgBasePath := filepath.Join(dstPkgDir, packageName)
53+
updated, err := manifest.PatchFile(m, pkgBasePath, typeSymbol, fileSymbol, data)
54+
if err != nil {
55+
return fmt.Errorf("patch file: %w", err)
56+
}
57+
58+
// 6. Write updated manifest to outputDir/manifests/packageName.
59+
manifestsDir := filepath.Join(outputDir, "manifests")
60+
if err := os.MkdirAll(manifestsDir, 0755); err != nil {
61+
return fmt.Errorf("create output manifests dir: %w", err)
62+
}
63+
outManifestPath := filepath.Join(manifestsDir, packageName)
64+
if err := manifest.WriteFile(outManifestPath, updated); err != nil {
65+
return fmt.Errorf("write updated manifest: %w", err)
66+
}
67+
68+
fmt.Printf("Patch complete. Updated manifest written to %s\n", outManifestPath)
69+
return nil
70+
}
71+
72+
// parseHexSymbol parses a hex string (with or without 0x prefix) into int64.
73+
func parseHexSymbol(s string) (int64, error) {
74+
// Strip optional 0x prefix.
75+
trimmed := s
76+
if len(s) >= 2 && (s[:2] == "0x" || s[:2] == "0X") {
77+
trimmed = s[2:]
78+
}
79+
v, err := strconv.ParseUint(trimmed, 16, 64)
80+
if err != nil {
81+
return 0, fmt.Errorf("invalid hex value %q: %w", s, err)
82+
}
83+
return int64(v), nil
84+
}
85+
86+
// copyPackageFiles copies all package files (<name>_0, <name>_1, ...) from src to dst dir.
87+
func copyPackageFiles(srcDir, dstDir, name string, count int) error {
88+
for i := 0; i < count; i++ {
89+
filename := fmt.Sprintf("%s_%d", name, i)
90+
src := filepath.Join(srcDir, filename)
91+
dst := filepath.Join(dstDir, filename)
92+
if err := copyFile(src, dst); err != nil {
93+
return fmt.Errorf("copy %s: %w", filename, err)
94+
}
95+
}
96+
return nil
97+
}
98+
99+
// copyFile copies a single file from src to dst, creating dst if necessary.
100+
func copyFile(src, dst string) error {
101+
in, err := os.Open(src)
102+
if err != nil {
103+
return err
104+
}
105+
defer in.Close()
106+
107+
out, err := os.Create(dst)
108+
if err != nil {
109+
return err
110+
}
111+
defer out.Close()
112+
113+
if _, err := io.Copy(out, in); err != nil {
114+
return err
115+
}
116+
return out.Sync()
117+
}

cmd/evrtools/patch_cli_test.go

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
package main
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"testing"
7+
)
8+
9+
func TestParseHexSymbol(t *testing.T) {
10+
wantVal := int64(int64(-4707359568332879775))
11+
tests := []struct {
12+
name string
13+
input string
14+
want int64
15+
wantErr bool
16+
}{
17+
{"with 0x prefix", "0xbeac1969cb7b8861", wantVal, false},
18+
{"without prefix", "beac1969cb7b8861", wantVal, false},
19+
{"empty string", "", 0, true},
20+
}
21+
for _, tc := range tests {
22+
t.Run(tc.name, func(t *testing.T) {
23+
got, err := parseHexSymbol(tc.input)
24+
if tc.wantErr {
25+
if err == nil {
26+
t.Errorf("parseHexSymbol(%q) expected error, got %d", tc.input, got)
27+
}
28+
return
29+
}
30+
if err != nil {
31+
t.Fatalf("parseHexSymbol(%q) unexpected error: %v", tc.input, err)
32+
}
33+
if got != tc.want {
34+
t.Errorf("parseHexSymbol(%q) = %d, want %d", tc.input, got, tc.want)
35+
}
36+
})
37+
}
38+
}
39+
40+
func TestCopyFile(t *testing.T) {
41+
dir := t.TempDir()
42+
srcPath := filepath.Join(dir, "source.bin")
43+
dstPath := filepath.Join(dir, "dest.bin")
44+
45+
content := []byte("test file contents for copy")
46+
if err := os.WriteFile(srcPath, content, 0644); err != nil {
47+
t.Fatal(err)
48+
}
49+
50+
if err := copyFile(srcPath, dstPath); err != nil {
51+
t.Fatalf("copyFile() error: %v", err)
52+
}
53+
54+
got, err := os.ReadFile(dstPath)
55+
if err != nil {
56+
t.Fatalf("reading dest: %v", err)
57+
}
58+
if string(got) != string(content) {
59+
t.Errorf("copyFile() dest contents = %q, want %q", got, content)
60+
}
61+
}
62+
63+
func TestCopyFile_SourceMissing(t *testing.T) {
64+
dir := t.TempDir()
65+
srcPath := filepath.Join(dir, "nonexistent.bin")
66+
dstPath := filepath.Join(dir, "dest.bin")
67+
68+
err := copyFile(srcPath, dstPath)
69+
if err == nil {
70+
t.Error("copyFile() with missing source expected error, got nil")
71+
}
72+
}

0 commit comments

Comments
 (0)