add docs

CMakeLists.txt

@@ -5,7 +5,7 @@ set(CMAKE_C_STANDARD 99)
 set(CMAKE_C_STANDARD_REQUIRED ON)
 
 if (CMAKE_C_COMPILER_ID MATCHES "GNU|Clang")
-    set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -march=native -mtune=native")
+    set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -O3 -march=native -mtune=native -flto -fno-math-errno -fno-signed-zeros -fno-trapping-math -freciprocal-math -funroll-loops -fprefetch-loop-arrays -fopenmp")
 endif()
 
 find_package(OpenMP REQUIRED)

Makefile

@@ -1,5 +1,23 @@
 CC = gcc
-CFLAGS = -O3 -march=native -mtune=native -Wall -Wextra -g -std=c11 -fopenmp
+
+# Core optimization flags
+# -flto: link-time optimization (cross-module inlining, dead code elimination)
+# -fno-math-errno: math functions don't set errno (avoids stores to errno)
+# -fno-signed-zeros: treats -0 as +0 for more algebraic transforms
+# -fno-trapping-math: assume no FP exceptions are generated
+# -freciprocal-math: allows x/y -> x*(1/y) and similar
+# -funroll-loops: unroll loops where profitable
+# -fprefetch-loop-arrays: generate prefetch for array accesses in loops
+OPT_FLAGS = -flto -fno-math-errno -fno-signed-zeros -fno-trapping-math -freciprocal-math -funroll-loops -fprefetch-loop-arrays
+
+# Release build flags (with debug symbols)
+CFLAGS = -O3 -march=native -mtune=native $(OPT_FLAGS) -Wall -Wextra -g -std=c11 -fopenmp
+
+# PGO generation flags (no -g to avoid profile pollution)
+PGO_CFLAGS = -O3 -march=native -mtune=native $(OPT_FLAGS) -Wall -Wextra -std=c11 -fopenmp -fprofile-generate
+
+# PGO use flags (no -g for maximum performance)
+PGO_USE_CFLAGS = -O3 -march=native -mtune=native $(OPT_FLAGS) -Wall -Wextra -std=c11 -fopenmp -fprofile-use
 SRC = src/tensor/tensor_core.c src/tensor/tensor_alloc.c src/tensor/tensor_view.c src/tensor/tensor_ops.c src/tensor/tensor_gather.c src/tensor/tensor_reduc.c src/tensor/tensor_matmul.c src/autograd/autograd_core.c src/autograd/autograd_ops.c src/autograd/autograd_view.c src/autograd/autograd_gather.c src/autograd/autograd_reduc.c src/autograd/autograd_matmul.c src/autograd/autograd_backward.c src/init/rng.c src/nn/parameter.c src/nn/module.c src/nn/linear.c src/nn/embedding.c src/nn/positional_embedding.c src/nn/layer_norm.c src/nn/dropout.c src/nn/multihead_attention.c src/nn/decoder_block.c src/nn/decoder.c src/nn/mlp.c src/losses/classification.c src/nn/causal_mask.c src/optim/sgd.c src/optim/optim_common.c src/optim/adamw.c src/serialization/checkpoint.c
 HEADERS = include/tensorlib/tensor.h include/tensorlib/tensor_matmul.h include/tensorlib/autograd.h include/tensorlib/nn.h tests/fixtures/test_common.h
 INCLUDES = -Iinclude/tensorlib -Itests/fixtures
@@ -129,6 +147,22 @@ $(BIN)/mnist_mlp: examples/mnsit/mnist_mlp.c $(SRC) $(HEADERS) | $(BIN)
 $(BIN)/tiny_lm: examples/tiny_lm/tiny_lm.c $(SRC) $(HEADERS) | $(BIN)
 	$(CC) $(CFLAGS) -Iinclude $(INCLUDES) -o $@ examples/tiny_lm/tiny_lm.c $(SRC) -lm
 
+# PGO: build with instrumentation
+$(BIN)/tiny_lm_pgo_gen: examples/tiny_lm/tiny_lm.c $(SRC) $(HEADERS) | $(BIN)
+	$(CC) $(PGO_CFLAGS) -Iinclude $(INCLUDES) -o $@ examples/tiny_lm/tiny_lm.c $(SRC) -lm
+
+# PGO: rebuild using collected profiles
+$(BIN)/tiny_lm_pgo_use: examples/tiny_lm/tiny_lm.c $(SRC) $(HEADERS) | $(BIN)
+	$(CC) $(PGO_USE_CFLAGS) -Iinclude $(INCLUDES) -o $@ examples/tiny_lm/tiny_lm.c $(SRC) -lm
+
+# Full PGO pipeline: generate profiles via training, then rebuild optimized
+pgo-tiny-lm: $(BIN)/tiny_lm_pgo_gen
+	@echo "=== PGO Step 1: instrumented binary built. Run training to collect profiles..."
+	./$(BIN)/tiny_lm_pgo_gen "$(CORPUS)" --steps 2000 --generate 0 --checkpoint tiny_lm_pgo.chk
+	@echo "=== PGO Step 2: profiles collected. Rebuilding with -fprofile-use..."
+	$(MAKE) $(BIN)/tiny_lm_pgo_use
+	@echo "=== PGO complete. Optimized binary at $(BIN)/tiny_lm_pgo_use"
+
 $(BIN)/bench_tensor_matmul: benchmarks/matmul/bench_tensor_matmul.c $(SRC) $(HEADERS) | $(BIN)
 	$(CC) $(CFLAGS) $(INCLUDES) -o $@ benchmarks/matmul/bench_tensor_matmul.c $(SRC) -lm
 

README.md

@@ -0,0 +1,129 @@
+# TensorLib
+
+A from-scratch deep learning tensor library in C with autograd, Transformer modules, and CPU optimizations.
+
+## Features
+
+- **Tensor library** — N-dimensional float32 arrays with NumPy-style broadcasting, strided views (transpose, reshape, slice, expand, squeeze), and reference-counted storage
+- **Autograd engine** — Dynamic reverse-mode automatic differentiation over 23 differentiable operations, with stale-graph detection and gradient accumulation
+- **Neural network modules** — Linear, Embedding, LayerNorm, Dropout, Multi-head Causal Self-Attention, Decoder Block (pre-norm), full Decoder stack, MLP
+- **Loss functions** — Cross-entropy (with built-in log-softmax), Softmax, LogSoftmax
+- **Optimizers** — SGD, AdamW (decoupled weight decay, bias correction, gradient clipping)
+- **Checkpointing** — Versioned, atomic (transaction-safe) save/load with optimizer and RNG state
+- **SIMD acceleration** — AVX2+FMA matmul micro-kernel with tile-based blocked algorithm
+- **OpenMP parallelism** — Multi-threaded matmul kernels
+- **Deterministic RNG** — Splitmix64 PRNG with uniform and normal (Box-Muller) distributions
+- **No external ML dependencies** — Pure C, no Python, CUDA, or third-party ML libraries required
+
+## Build
+
+### CMake (recommended)
+
+```sh
+cmake -S . -B build -DCMAKE_BUILD_TYPE=Release
+cmake --build build --config Release
+```
+
+Requires a compiler with OpenMP support (GCC, Clang, MSVC).
+
+### Makefile (GNU Make, GCC)
+
+```sh
+make          # build and run all tests
+make tiny-lm  # build the TinyLM example
+make mnist    # build the MNIST example
+make clean
+```
+
+Profile-guided optimization (PGO) for TinyLM:
+
+```sh
+make pgo-tiny-lm CORPUS=corpus.txt
+```
+
+## Project structure
+
+```
+include/tensorlib/     # Public API headers
+  tensor.h             #   Core tensor structs and operations
+  tensor_matmul.h      #   Matmul internals
+  autograd.h           #   Autograd engine API
+  nn.h                 #   Neural network module API
+
+src/                   # Source implementation
+  tensor/              #   Tensor ops, views, reductions, matmul
+  autograd/            #   Autograd forward/backward ops, graph traversal
+  nn/                  #   Module system, layers (Linear, Embedding, LayerNorm,
+                       #   Dropout, MultiheadAttention, DecoderBlock, Decoder, MLP)
+  losses/              #   Cross-entropy, Softmax, LogSoftmax
+  optim/               #   SGD, AdamW, gradient clipping
+  init/                #   PRNG and weight initialization
+  serialization/       #   Checkpoint save/load
+
+tests/                 # Test suite
+  unit/
+    tensor/            #   6 test files
+    autograd/          #   9 test files
+    nn/                #   16 test files
+    optim/             #   2 test files
+
+examples/
+  autograd_example.c   #   Computation graph demo
+  tiny_lm/             #   Byte-level decoder language model (~1.9M params)
+  mnsit/               #   MNIST MLP classifier
+
+benchmarks/
+  matmul/              #   Matmul performance benchmarks
+```
+
+## Examples
+
+### Autograd demo
+
+```sh
+./build/autograd_example
+```
+
+Builds a computation graph `input @ weights + bias -> exp -> mean`, backpropagates, and prints all gradients.
+
+### MNIST MLP
+
+```sh
+./build/mnist_mlp
+```
+
+784→128 ReLU→10 Softmax MLP trained with SGD and cross-entropy loss.
+
+### TinyLM — byte-level language model
+
+```sh
+./build/tiny_lm corpus.txt --steps 1000 --generate 200 --prompt "Hello"
+```
+
+4-layer, 192-width, 6-head decoder transformer (~1.9M params) trained with AdamW on raw byte corpora. See [examples/tiny_lm/README.md](examples/tiny_lm/README.md) for details.
+
+## Testing
+
+33 unit test executables covering tensors, autograd, all NN modules, optimizers, loss functions, and checkpointing.
+
+```sh
+cmake --build build --config Release --target test
+# or with Makefile:
+make test
+```
+
+## Benchmarks
+
+Matmul benchmarks compare TensorLib's blocked AVX2 kernel against OpenBLAS:
+
+```sh
+make benchmark-compare
+```
+
+## Design notes
+
+- **Eager execution** — Graph is built dynamically during forward pass; no JIT compilation
+- **Manual memory management** — Reference counting for `Storage`, `ag_tensor`, and `ag_node`; no garbage collector
+- **Single-threaded API** — Not thread-safe; assumes single-threaded usage
+- **CPU only** — No GPU support
+- **SIMD scope** — AVX2+FMA only used in matmul; element-wise ops, activations, norm, softmax are scalar (optimization opportunity tracked in `optimizations.md`)