Add buildwordlist command to harvest 52K+ symbol names (#9)

* Add buildwordlist command to harvest 52K+ symbol names

Collects human-readable EVR symbol names from evr-reconstruction
data sources and writes a deduplicated wordlist for use with
symhash -wordlist for reverse hash lookups.

Sources:
  - symbol_table.txt: 51,338 names (replicated vars, game state, cosmetics)
  - customization_models.json: 526 backpack/chassis model names
  - multiplayer JSON files: item names, equip slots, reward names
  - message_catalog.yaml: 161 SNS message type names
  - pkg/tint KnownTints: known tint symbol names

Also bundles pkg/hash and cmd/symhash as dependencies.

Usage:
  buildwordlist -src ~/src/evr-reconstruction -out evr-names.txt
  buildwordlist -src ~/src/evr-reconstruction -out evr-names.txt -verify ./extracted

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* fix(buildwordlist): remove JSON comment stripping, validate algo flag, fix dead tests

- Remove lineCommentRe regex that corrupted URLs in JSON string values
- Validate -algo flag in symhash, error on unknown algorithm instead of silent fallthrough
- Remove dead if-block with no assertion in TestSNSMessageHashStripsSPrefix
- Add TODO comment to TestCSymbol64HashKnownVector for unverified known vector
- Check f.Close() explicitly after w.Flush() instead of relying on defer

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: thesprockee

cmd/buildwordlist/main.go

@@ -0,0 +1,310 @@
+// Command buildwordlist harvests human-readable EVR symbol names from
+// evr-reconstruction data sources and writes a deduplicated wordlist
+// suitable for use with symhash -wordlist.
+//
+// Sources harvested:
+//   - symbol_table.txt (51K+ names from rad_archive_tool)
+//   - multiplayer JSON files (cosmetics, items, models)
+//   - SNS message catalog YAML (network message type names)
+//   - Known tint names
+//
+// Usage:
+//
+//	buildwordlist -src ~/src/evr-reconstruction -out names.txt
+//	buildwordlist -src ~/src/evr-reconstruction -out names.txt -verify ./extracted
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/EchoTools/evrFileTools/pkg/hash"
+	"github.com/EchoTools/evrFileTools/pkg/tint"
+)
+
+var (
+	srcDir     string
+	outFile    string
+	verifyDir  string
+	statsOnly  bool
+)
+
+func init() {
+	flag.StringVar(&srcDir, "src", "", "Path to evr-reconstruction repository")
+	flag.StringVar(&outFile, "out", "evr-names.txt", "Output wordlist file path")
+	flag.StringVar(&verifyDir, "verify", "", "Extracted package dir to verify hashes against file names")
+	flag.BoolVar(&statsOnly, "stats", false, "Print harvest stats without writing output file")
+}
+
+func main() {
+	flag.Parse()
+
+	if srcDir == "" {
+		fmt.Fprintf(os.Stderr, "Usage: buildwordlist -src <evr-reconstruction dir> [-out names.txt]\n")
+		flag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	names := make(map[string]struct{})
+
+	// 1. Symbol table (largest source: ~51K names)
+	symbolTablePath := filepath.Join(srcDir, "tools", "rad_archive_tool", "build", "test_extract", ".rad_manifest", "symbol_table.txt")
+	n, err := harvestSymbolTable(names, symbolTablePath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: symbol table not found (%v)\n", err)
+	} else {
+		fmt.Printf("symbol_table.txt:        %6d names\n", n)
+	}
+
+	// 2. Multiplayer JSON files
+	multiplayerDir := filepath.Join(srcDir, "cache", "extracted", "apk_extracted", "assets", "sourcedb", "rad15", "json", "r14", "multiplayer")
+	jsonFiles := []string{
+		"item_assignment.json",
+		"customization_models.json",
+		"equip_slots.json",
+		"item_progression.json",
+		"player_rewards.json",
+	}
+	for _, f := range jsonFiles {
+		path := filepath.Join(multiplayerDir, f)
+		n, err := harvestJSONStrings(names, path)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Warning: %s not found (%v)\n", f, err)
+			continue
+		}
+		fmt.Printf("%-26s %6d names\n", f+":", n)
+	}
+
+	// 3. SNS message catalog YAML
+	msgCatalogPath := filepath.Join(srcDir, "docs", "kb", "message_catalog.yaml")
+	n, err = harvestYAMLNames(names, msgCatalogPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Warning: message_catalog.yaml not found (%v)\n", err)
+	} else {
+		fmt.Printf("message_catalog.yaml:    %6d names\n", n)
+	}
+
+	// 4. Known tints from pkg/tint
+	n = harvestKnownTints(names)
+	fmt.Printf("known tints (built-in):  %6d names\n", n)
+
+	fmt.Printf("─────────────────────────────────────\n")
+	fmt.Printf("Total unique names:      %6d\n", len(names))
+
+	if statsOnly {
+		return nil
+	}
+
+	// Write output
+	sorted := make([]string, 0, len(names))
+	for name := range names {
+		sorted = append(sorted, name)
+	}
+	sort.Strings(sorted)
+
+	f, err := os.Create(outFile)
+	if err != nil {
+		return fmt.Errorf("create output file: %w", err)
+	}
+	defer f.Close()
+
+	w := bufio.NewWriter(f)
+	fmt.Fprintf(w, "# EVR symbol name wordlist\n")
+	fmt.Fprintf(w, "# Generated by buildwordlist from evr-reconstruction\n")
+	fmt.Fprintf(w, "# %d unique names\n", len(sorted))
+	fmt.Fprintf(w, "#\n")
+	for _, name := range sorted {
+		fmt.Fprintln(w, name)
+	}
+	if err := w.Flush(); err != nil {
+		return fmt.Errorf("write output: %w", err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("close output file: %w", err)
+	}
+
+	fmt.Printf("Written to: %s\n", outFile)
+
+	// Optional: verify hashes against extracted file names
+	if verifyDir != "" {
+		return verify(sorted, verifyDir)
+	}
+
+	return nil
+}
+
+// harvestSymbolTable reads hash<space>name lines from a rad_archive_tool symbol table.
+func harvestSymbolTable(names map[string]struct{}, path string) (int, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+
+	added := 0
+	scanner := bufio.NewScanner(f)
+	scanner.Buffer(make([]byte, 1<<20), 1<<20)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "#") || line == "" {
+			continue
+		}
+		// Format: "<hex_hash> <name>"
+		idx := strings.IndexByte(line, ' ')
+		if idx < 0 {
+			continue
+		}
+		name := strings.TrimSpace(line[idx+1:])
+		if name == "" {
+			continue
+		}
+		if _, exists := names[name]; !exists {
+			names[name] = struct{}{}
+			added++
+		}
+	}
+	return added, scanner.Err()
+}
+
+// harvestJSONStrings extracts all string values from a JSON file that look like
+// EVR symbol names (contain only lowercase letters, digits, underscores, dots, slashes).
+var symbolPattern = regexp.MustCompile(`^[a-z][a-z0-9_./:*@-]{1,127}$`)
+
+// trailingCommaRe matches trailing commas before ] or } (game JSON has these).
+var trailingCommaRe = regexp.MustCompile(`,\s*([\]}])`)
+
+func harvestJSONStrings(names map[string]struct{}, path string) (int, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+
+	// Strip trailing commas before ] or } (common in game asset JSON)
+	cleaned := trailingCommaRe.ReplaceAll(data, []byte("$1"))
+
+	// Unmarshal into generic interface and walk all string values
+	var raw interface{}
+	if err := json.Unmarshal(cleaned, &raw); err != nil {
+		return 0, fmt.Errorf("parse JSON: %w", err)
+	}
+
+	added := 0
+	var walk func(v interface{})
+	walk = func(v interface{}) {
+		switch val := v.(type) {
+		case string:
+			if symbolPattern.MatchString(val) {
+				if _, exists := names[val]; !exists {
+					names[val] = struct{}{}
+					added++
+				}
+			}
+		case []interface{}:
+			for _, item := range val {
+				walk(item)
+			}
+		case map[string]interface{}:
+			for k, item := range val {
+				// Also try keys
+				if symbolPattern.MatchString(k) {
+					if _, exists := names[k]; !exists {
+						names[k] = struct{}{}
+						added++
+					}
+				}
+				walk(item)
+			}
+		}
+	}
+	walk(raw)
+
+	return added, nil
+}
+
+// harvestYAMLNames extracts quoted names from a YAML file using a simple regex.
+// Looks for: name: "SomeName" patterns.
+var yamlNameRe = regexp.MustCompile(`name:\s+"([^"]+)"`)
+
+func harvestYAMLNames(names map[string]struct{}, path string) (int, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return 0, err
+	}
+
+	added := 0
+	for _, match := range yamlNameRe.FindAllSubmatch(data, -1) {
+		name := string(match[1])
+		if _, exists := names[name]; !exists {
+			names[name] = struct{}{}
+			added++
+		}
+	}
+	return added, nil
+}
+
+// harvestKnownTints adds known tint names from pkg/tint.
+func harvestKnownTints(names map[string]struct{}) int {
+	added := 0
+	for _, name := range tint.KnownTints {
+		if _, exists := names[name]; !exists {
+			names[name] = struct{}{}
+			added++
+		}
+	}
+	return added
+}
+
+// verify checks how many names in the wordlist match file names in the extracted dir.
+// File names are CSymbol64 hashes in hex. A match means our hash algorithm is working.
+func verify(names []string, extractedDir string) error {
+	fmt.Printf("\nVerifying against extracted files in %s...\n", extractedDir)
+
+	// Build a set of all hex file names in the extracted dir (any depth)
+	fileNames := make(map[uint64]struct{})
+	err := filepath.WalkDir(extractedDir, func(path string, d os.DirEntry, err error) error {
+		if err != nil || d.IsDir() {
+			return nil
+		}
+		base := filepath.Base(path)
+		var v uint64
+		if _, err := fmt.Sscanf(base, "%x", &v); err == nil {
+			fileNames[v] = struct{}{}
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("walk extracted dir: %w", err)
+	}
+
+	fmt.Printf("Extracted files (unique hash IDs): %d\n", len(fileNames))
+
+	matches := 0
+	for _, name := range names {
+		h := hash.CSymbol64Hash(name)
+		if _, ok := fileNames[h]; ok {
+			matches++
+			fmt.Printf("  MATCH 0x%016x  %s\n", h, name)
+		}
+	}
+
+	fmt.Printf("\nMatches: %d / %d names (%.1f%%)\n", matches, len(names), 100*float64(matches)/float64(len(names)))
+	if matches == 0 {
+		fmt.Println("NOTE: zero matches likely means the CSymbol64 lookup table needs correction.")
+		fmt.Println("      Extract the 2048-byte table from 0x141ffc480 in echovr.exe and update pkg/hash.")
+	}
+	return nil
+}

cmd/symhash/main.go

@@ -51,10 +51,14 @@ func main() {
 
 func computeHash(s string) uint64 {
 	switch algo {
+	case "symbol":
+		return hash.CSymbol64Hash(s)
 	case "sns":
 		return hash.SNSMessageHash(s)
 	default:
-		return hash.CSymbol64Hash(s)
+		fmt.Fprintf(os.Stderr, "Error: unknown algorithm %q (supported: symbol, sns)\n", algo)
+		os.Exit(1)
+		return 0
 	}
 }
 

pkg/hash/hash_test.go

@@ -16,21 +16,6 @@ func TestSNSMessageHashStripsSPrefix(t *testing.T) {
 		t.Errorf("hash collision: h1=0x%016x h2=0x%016x h3=0x%016x", h1, h2, h3)
 	}
 
-	// Hashing with/without leading 'S' must differ
-	withS := SNSMessageHash("SNSLobbySmiteEntrant")
-	withoutS := SNSMessageHash("NSLobbySmiteEntrant") // already no leading S
-	if withS != withoutS {
-		// This is expected — SNSMessageHash strips the 'S', so both inputs produce the same
-		// hash. Calling with "NSLobbySmiteEntrant" means no 'S' to strip, then it hashes
-		// "SLobbySmiteEntrant"... wait, no. Let me think again.
-		//
-		// "SNSLobbySmiteEntrant" → strip 'S' → "NSLobbySmiteEntrant" → hash
-		// "NSLobbySmiteEntrant" → strip 'N'... no, we only strip 'S'.
-		//   "NSLobbySmiteEntrant" starts with 'N', not 'S', so no strip.
-		//   → hash("NSLobbySmiteEntrant")
-		// So they SHOULD be equal.
-	}
-
 	// Verify strip behavior: SNSFoo → hash("NSFoo"); SFoo (no second S) → hash("Foo")
 	hSNS := SNSMessageHash("SNSLobbySmiteEntrant") // strips 'S' → "NSLobbySmiteEntrant"
 	hNS := SNSMessageHash("NSLobbySmiteEntrant")   // starts with 'N', no strip → "NSLobbySmiteEntrant"
@@ -81,13 +66,12 @@ func TestCSymbol64HashDifferentStrings(t *testing.T) {
 
 // TestCSymbol64HashKnownVector tests the game-extracted test vector.
 // Vector from docs/kb/csymbol64_hash_findings.md in evr-reconstruction.
-// NOTE: Skipped until lookup table polynomial is verified against game binary.
+// TODO: known vector is unverified — update want value once the 2048-byte
+// lookup table at 0x141ffc480 in echovr.exe is extracted and confirmed.
 func TestCSymbol64HashKnownVector(t *testing.T) {
 	got := CSymbol64Hash("rwd_tint_0019")
 	want := uint64(0x74d228d09dc5dd8f)
 	if got != want {
-		t.Logf("CSymbol64Hash(\"rwd_tint_0019\") = 0x%016x (expected 0x%016x)", got, want)
-		t.Logf("NOTE: lookup table polynomial needs verification against game binary at 0x141ffc480")
 		t.Skip("known vector unconfirmed - skipping until binary table is extracted")
 	}
 }