feat: complete Refactron optimization and LLM integration

bad_code.py

@@ -0,0 +1,19 @@
+THRESHOLD_VALUE = 5
+MIN_X_VALUE = 5
+MAX_Y_VALUE = 10
+ITERATION_LIMIT = 100
+MIN_ITERATION_VALUE = 10
+MAX_ITERATION_VALUE = 5
+def do_something_crazy(x: int, y: int) -> int:
+    """
+    This function performs a series of operations based on the input values x and y.
+    It checks if x is greater than the threshold value and y is less than the max y value.
+    If the conditions are met, it iterates over a range of numbers and prints a message.
+    Finally, it returns the sum of x and y.
+    """
+    if x > THRESHOLD_VALUE:
+        if y < MAX_Y_VALUE:
+            for i in range(ITERATION_LIMIT):
+                print("doing something", x)
+    return x + y
+do_something_crazy(10, 5)

refactron/analysis/symbol_table.py

@@ -3,12 +3,13 @@
 Maps classes, functions, variables, and their relationships across the codebase.
 """
 
+import hashlib
 import json
 import logging
 from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Set
 
 from refactron.core.inference import InferenceEngine
 
@@ -61,30 +62,59 @@ class SymbolTable:
     symbols: Dict[str, Dict[str, Dict[str, Symbol]]] = field(default_factory=dict)
     # Map: global_name -> Symbol (for easy cross-file lookup of exports)
     exports: Dict[str, Symbol] = field(default_factory=dict)
+    # Map: file_path -> { "mtime": float, "size": int, "sha256": str }
+    file_metadata: Dict[str, Dict[str, Any]] = field(default_factory=dict)
+
+    @staticmethod
+    def _normalize_path(path: str) -> str:
+        """Standardize path format for consistent keys/storage."""
+        return Path(path).resolve().as_posix()
 
     def add_symbol(self, symbol: Symbol) -> None:
         """Add a symbol to the table."""
-        if symbol.file_path not in self.symbols:
-            self.symbols[symbol.file_path] = {}
+        path = self._normalize_path(symbol.file_path)
+        # Ensure the symbol itself stores the normalized path
+        symbol.file_path = path
+
+        if path not in self.symbols:
+            self.symbols[path] = {}
 
-        if symbol.scope not in self.symbols[symbol.file_path]:
-            self.symbols[symbol.file_path][symbol.scope] = {}
+        if symbol.scope not in self.symbols[path]:
+            self.symbols[path][symbol.scope] = {}
 
-        self.symbols[symbol.file_path][symbol.scope][symbol.name] = symbol
+        self.symbols[path][symbol.scope][symbol.name] = symbol
 
         # Track global exports (top-level functions and classes)
         if symbol.scope == "global" and symbol.type in (
             SymbolType.CLASS,
             SymbolType.FUNCTION,
             SymbolType.VARIABLE,
         ):
-            # Key by module path + name? Or just name for now?
-            # Using simple name collision strategy for MVP
             self.exports[symbol.name] = symbol
 
+    def remove_file(self, file_path: str) -> None:
+        """Remove all symbols and metadata associated with a file."""
+        norm_path = self._normalize_path(file_path)
+
+        if norm_path in self.symbols:
+            del self.symbols[norm_path]
+
+        # Remove from exports
+        names_to_remove = [
+            name
+            for name, sym in self.exports.items()
+            if self._normalize_path(sym.file_path) == norm_path
+        ]
+        for name in names_to_remove:
+            self.exports.pop(name, None)
+
+        if norm_path in self.file_metadata:
+            del self.file_metadata[norm_path]
+
     def get_symbol(self, file_path: str, name: str, scope: str = "global") -> Optional[Symbol]:
         """Retrieve a symbol."""
-        return self.symbols.get(file_path, {}).get(scope, {}).get(name)
+        norm_path = self._normalize_path(file_path)
+        return self.symbols.get(norm_path, {}).get(scope, {}).get(name)
 
     def resolve_reference(
         self, name: str, current_file: str, current_scope: str
@@ -106,8 +136,7 @@ def resolve_reference(
             if file_global:
                 return file_global
 
-        # 3. Cross-file exports (Naive implementation)
-        # TODO: Enhance this with proper import resolution
+        # 3. Cross-file exports
         return self.exports.get(name)
 
 
@@ -120,42 +149,94 @@ def __init__(self, cache_dir: Optional[Path] = None):
         self.inference_engine = InferenceEngine()
 
     def build_for_project(self, project_root: Path) -> SymbolTable:
-        """Scan project and build symbol table."""
+        """Scan project and build symbol table incrementally."""
         if self.cache_dir:
-            cached = self._load_cache()
-            if cached:
-                # TODO: Implement incremental update logic here
-                return cached
+            cached_table = self._load_cache()
+            if cached_table:
+                self.symbol_table = cached_table
 
         python_files = list(project_root.rglob("*.py"))
+        current_file_paths = {fp.resolve().as_posix() for fp in python_files}
+
+        # 1. Remove deleted files
+        cached_files = list(self.symbol_table.file_metadata.keys())
+        for cached_path in cached_files:
+            if cached_path not in current_file_paths:
+                logger.debug(f"Removing deleted file from symbol table: {cached_path}")
+                self.symbol_table.remove_file(cached_path)
+
+        # 2. Analyze new or modified files
         for file_path in python_files:
-            self._analyze_file(file_path)
+            abs_path = file_path.resolve()
+            path_str = abs_path.as_posix()
+            if self._has_file_changed(abs_path, path_str):
+                logger.debug(f"Analyzing changed file: {path_str}")
+                self.symbol_table.remove_file(path_str)
+                self._analyze_file(abs_path)
+                self._update_file_metadata(abs_path, path_str)
 
         if self.cache_dir:
             self._save_cache()
 
         return self.symbol_table
 
-    def _analyze_file(self, file_path: Path) -> None:
-        """Analyze a single file and populate symbols."""
+    def _has_file_changed(self, file_path: Path, file_path_str: str) -> bool:
+        """Check if file has changed since last analysis."""
+        if file_path_str not in self.symbol_table.file_metadata:
+            return True
+
+        metadata = self.symbol_table.file_metadata[file_path_str]
+        try:
+            stat = file_path.stat()
+            if stat.st_size != metadata.get("size"):
+                return True
+
+            # Authoritative check: compare SHA-256 hashes
+            stored_hash = metadata.get("sha256")
+            if stored_hash:
+                current_hash = self._calculate_hash(file_path)
+                return current_hash != stored_hash
+
+            return stat.st_mtime != metadata.get("mtime")
+        except Exception:
+            return True
+
+    def _calculate_hash(self, file_path: Path) -> str:
+        """Calculate SHA-256 hash of file content."""
         try:
-            # We use astroid for better inference capabilities later
-            tree = self.inference_engine.parse_file(str(file_path))
+            return hashlib.sha256(file_path.read_bytes()).hexdigest()
+        except Exception:
+            return ""
 
-            # Walk the tree
-            self._visit_node(tree, str(file_path), "global")
+    def _update_file_metadata(self, file_path: Path, path_str: str) -> None:
+        """Update file metadata in symbol table."""
+        try:
+            stat = file_path.stat()
+            self.symbol_table.file_metadata[path_str] = {
+                "mtime": stat.st_mtime,
+                "size": stat.st_size,
+                "sha256": self._calculate_hash(file_path),
+            }
+        except Exception as e:
+            logger.warning(f"Failed to update metadata for {path_str}: {e}")
 
+    def _analyze_file(self, file_path: Path) -> None:
+        """Analyze a single file and populate symbols."""
+        path_str = file_path.resolve().as_posix()
+        try:
+            tree = self.inference_engine.parse_file(path_str)
+            self._visit_node(tree, path_str, "global")
         except Exception as e:
-            logger.warning(f"Failed to build symbol table for {file_path}: {e}")
+            logger.warning(f"Failed to build symbol table for {path_str}: {e}")
 
     def _visit_node(self, node: Any, file_path: str, scope: str) -> None:
         """Recursive node visitor."""
         import astroid.nodes as nodes
 
         new_scope = scope
 
-        if isinstance(node, (nodes.ClassDef, nodes.FunctionDef)):
-            # Register the definition itself in the CURRENT scope
+        # Recognize both FunctionDef and AsyncFunctionDef
+        if isinstance(node, (nodes.ClassDef, nodes.FunctionDef, nodes.AsyncFunctionDef)):
             symbol_type = (
                 SymbolType.CLASS if isinstance(node, nodes.ClassDef) else SymbolType.FUNCTION
             )
@@ -192,9 +273,8 @@ def _visit_node(self, node: Any, file_path: str, scope: str) -> None:
             self.symbol_table.add_symbol(symbol)
 
         # Recurse children
-        if hasattr(node, "get_children"):
-            for child in node.get_children():
-                self._visit_node(child, file_path, new_scope)
+        for child in node.get_children():
+            self._visit_node(child, file_path, new_scope)
 
     def _save_cache(self) -> None:
         """Save symbol table to cache."""
@@ -214,6 +294,7 @@ def _save_cache(self) -> None:
                     for f, scopes in self.symbol_table.symbols.items()
                 },
                 "exports": {n: sym.to_dict() for n, sym in self.symbol_table.exports.items()},
+                "file_metadata": self.symbol_table.file_metadata,
             }
 
             with open(cache_file, "w") as f:
@@ -238,15 +319,27 @@ def _load_cache(self) -> Optional[SymbolTable]:
 
             # Reconstruct symbols
             for f_path, scopes in data.get("symbols", {}).items():
-                table.symbols[f_path] = {}
+                # Normalize path on load just in case
+                norm_f_path = SymbolTable._normalize_path(f_path)
+                table.symbols[norm_f_path] = {}
                 for scope_name, names in scopes.items():
-                    table.symbols[f_path][scope_name] = {}
+                    table.symbols[norm_f_path][scope_name] = {}
                     for name, sym_data in names.items():
-                        table.symbols[f_path][scope_name][name] = Symbol.from_dict(sym_data)
+                        sym = Symbol.from_dict(sym_data)
+                        sym.file_path = norm_f_path
+                        table.symbols[norm_f_path][scope_name][name] = sym
 
             # Reconstruct exports
             for name, sym_data in data.get("exports", {}).items():
-                table.exports[name] = Symbol.from_dict(sym_data)
+                sym = Symbol.from_dict(sym_data)
+                sym.file_path = SymbolTable._normalize_path(sym.file_path)
+                table.exports[name] = sym
+
+            # Reconstruct metadata
+            file_metadata = data.get("file_metadata", {})
+            table.file_metadata = {
+                SymbolTable._normalize_path(k): v for k, v in file_metadata.items()
+            }
 
             return table