Improve charset selection when parsing Delphi source files

CHANGELOG.md

@@ -22,12 +22,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- The ANSI encoding configured by `sonar.delphi.codePage` or inferred from `DCC_CodePage` is now
+  preferred over the system encoding when parsing source files.
 - Metrics are no longer reported on test sources.
 - Duplications are no longer reported on test sources.
 - Coverage data is no longer reported on test sources.
 
 ### Fixed
 
+- The configured `sonar.sourceEncoding` was never used for search path units.
 - Quick fixes removing too much surrounding code in `RedundantInherited`.
 
 ## [1.18.3] - 2025-11-11

delphi-checks-testkit/src/main/java/au/com/integradev/delphi/builders/AbstractDelphiTestFile.java

@@ -85,7 +85,7 @@ private static DelphiFileConfig mockConfig() {
         new TypeFactoryImpl(
             DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT, DelphiProperties.COMPILER_VERSION_DEFAULT);
     DelphiFileConfig mock = mock(DelphiFileConfig.class);
-    when(mock.getEncoding()).thenReturn(UTF_8.name());
+    when(mock.getCharset()).thenReturn(UTF_8);
     when(mock.getTypeFactory()).thenReturn(typeFactory);
     when(mock.getSearchPath()).thenReturn(SearchPath.create(Collections.emptyList()));
     when(mock.getDefinitions()).thenReturn(Collections.emptySet());

delphi-checks-testkit/src/main/java/au/com/integradev/delphi/checks/verifier/CheckVerifierImpl.java

@@ -323,7 +323,7 @@ private static Supplier<DelphiFileStream> newFileStreamSupplier(DelphiFile delph
           try {
             return new DelphiFileStream(
                 delphiFile.getSourceCodeFile().getAbsolutePath(),
-                delphiFile.getSourceCodeFileEncoding());
+                delphiFile.getSourceCodeFileCharset());
           } catch (IOException e) {
             throw new UncheckedIOException(e);
           }

delphi-frontend/src/main/java/au/com/integradev/delphi/antlr/DelphiFileStream.java

@@ -29,43 +29,39 @@
 
 public class DelphiFileStream extends ANTLRStringStream {
   private final String fileName;
-  private final String encoding;
+  private final Charset charset;
 
-  public DelphiFileStream(String fileName, String encoding) throws IOException {
+  public DelphiFileStream(String fileName, Charset charset) throws IOException {
     this.fileName = fileName;
-    this.encoding = this.load(fileName, encoding);
+    this.charset = this.load(fileName, charset);
   }
 
-  private String load(String fileName, String encoding) throws IOException {
+  private Charset load(String fileName, Charset charset) throws IOException {
     if (fileName != null) {
       File f = new File(fileName);
       int size = (int) f.length();
       try (BOMInputStream input = bomInputStream(fileName).get()) {
         ByteOrderMark bom = input.getBOM();
         if (bom != null) {
-          encoding = bom.getCharsetName();
+          charset = Charset.forName(bom.getCharsetName());
         }
 
-        if (encoding == null) {
-          encoding = Charset.defaultCharset().name();
-        }
-
-        try (InputStreamReader reader = new InputStreamReader(input, encoding)) {
+        try (InputStreamReader reader = new InputStreamReader(input, charset)) {
           this.data = new char[size];
           super.n = reader.read(this.data);
         }
       }
     }
-    return encoding;
+    return charset;
   }
 
   @Override
   public String getSourceName() {
     return this.fileName;
   }
 
-  public String getEncoding() {
-    return this.encoding;
+  public Charset getCharset() {
+    return this.charset;
   }
 
   private static BOMInputStream.Builder bomInputStream(String fileName) throws IOException {

delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFile.java

@@ -36,7 +36,7 @@ class DefaultDelphiFile implements DelphiFile {
   private CompilerSwitchRegistry switchRegistry;
   private TextBlockLineEndingModeRegistry textBlockLineEndingModeRegistry;
   private TypeFactory typeFactory;
-  private String encoding;
+  private Charset charset;
   private Charset ansiCharset;
 
   DefaultDelphiFile() {
@@ -54,8 +54,8 @@ public List<String> getSourceCodeFileLines() {
   }
 
   @Override
-  public String getSourceCodeFileEncoding() {
-    return encoding;
+  public Charset getSourceCodeFileCharset() {
+    return charset;
   }
 
   @Override
@@ -101,8 +101,8 @@ void setSourceCodeLines(List<String> sourceCodeLines) {
     this.sourceCodeLines = List.copyOf(sourceCodeLines);
   }
 
-  void setSourceCodeEncoding(String encoding) {
-    this.encoding = encoding;
+  void setSourceCodeCharset(Charset charset) {
+    this.charset = charset;
   }
 
   void setAnsiCharset(Charset ansiCharset) {

delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFileConfig.java

@@ -22,11 +22,10 @@
 import au.com.integradev.delphi.preprocessor.search.SearchPath;
 import java.nio.charset.Charset;
 import java.util.Set;
-import javax.annotation.Nullable;
 import org.sonar.plugins.communitydelphi.api.type.TypeFactory;
 
 public class DefaultDelphiFileConfig implements DelphiFileConfig {
-  private final String encoding;
+  private final Charset charset;
   private final Charset ansiCharset;
   private final DelphiPreprocessorFactory preprocessorFactory;
   private final TypeFactory typeFactory;
@@ -35,14 +34,14 @@ public class DefaultDelphiFileConfig implements DelphiFileConfig {
   private final boolean skipImplementation;
 
   DefaultDelphiFileConfig(
-      String encoding,
+      Charset charset,
       Charset ansiCharset,
       DelphiPreprocessorFactory preprocessorFactory,
       TypeFactory typeFactory,
       SearchPath searchPath,
       Set<String> definitions,
       boolean skipImplementation) {
-    this.encoding = encoding;
+    this.charset = charset;
     this.ansiCharset = ansiCharset;
     this.preprocessorFactory = preprocessorFactory;
     this.typeFactory = typeFactory;
@@ -51,10 +50,9 @@ public class DefaultDelphiFileConfig implements DelphiFileConfig {
     this.skipImplementation = skipImplementation;
   }
 
-  @Nullable
   @Override
-  public String getEncoding() {
-    return encoding;
+  public Charset getCharset() {
+    return charset;
   }
 
   @Override

delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFile.java

@@ -39,7 +39,6 @@
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
-import javax.annotation.Nullable;
 import org.antlr.runtime.BufferedTokenStream;
 import org.antlr.runtime.CommonToken;
 import org.antlr.runtime.RecognitionException;
@@ -55,7 +54,7 @@ public interface DelphiFile {
 
   List<String> getSourceCodeFileLines();
 
-  String getSourceCodeFileEncoding();
+  Charset getSourceCodeFileCharset();
 
   DelphiAst getAst();
 
@@ -84,10 +83,10 @@ static DelphiInputFile from(InputFile inputFile, DelphiFileConfig config) {
 
     private static DelphiFileConfig useInputFileEncoding(
         InputFile inputFile, DelphiFileConfig config) {
-      if (inputFile.charset() != null && !inputFile.charset().name().equals(config.getEncoding())) {
+      if (inputFile.charset() != null && !inputFile.charset().equals(config.getCharset())) {
         config =
             DelphiFile.createConfig(
-                inputFile.charset().name(),
+                inputFile.charset(),
                 config.getAnsiCharset(),
                 config.getPreprocessorFactory(),
                 config.getTypeFactory(),
@@ -112,26 +111,26 @@ class EmptyDelphiFileException extends RuntimeException {
   }
 
   static DelphiFileConfig createConfig(
-      String encoding,
+      Charset charset,
       Charset ansiCharset,
       DelphiPreprocessorFactory preprocessorFactory,
       TypeFactory typeFactory,
       SearchPath searchPath,
       Set<String> definitions) {
     return createConfig(
-        encoding, ansiCharset, preprocessorFactory, typeFactory, searchPath, definitions, false);
+        charset, ansiCharset, preprocessorFactory, typeFactory, searchPath, definitions, false);
   }
 
   static DelphiFileConfig createConfig(
-      @Nullable String encoding,
+      Charset charset,
       Charset ansiCharset,
       DelphiPreprocessorFactory preprocessorFactory,
       TypeFactory typeFactory,
       SearchPath searchPath,
       Set<String> definitions,
       boolean shouldSkipImplementation) {
     return new DefaultDelphiFileConfig(
-        encoding,
+        charset,
         ansiCharset,
         preprocessorFactory,
         typeFactory,
@@ -150,10 +149,10 @@ private static void setupFile(
       DefaultDelphiFile delphiFile, File sourceFile, DelphiFileConfig config) {
     try {
       String filePath = sourceFile.getAbsolutePath();
-      DelphiFileStream fileStream = new DelphiFileStream(filePath, config.getEncoding());
+      DelphiFileStream fileStream = new DelphiFileStream(filePath, config.getCharset());
       DelphiPreprocessor preprocessor = preprocess(fileStream, config);
       delphiFile.setSourceCodeFile(sourceFile);
-      delphiFile.setSourceCodeEncoding(fileStream.getEncoding());
+      delphiFile.setSourceCodeCharset(fileStream.getCharset());
       delphiFile.setAnsiCharset(config.getAnsiCharset());
       delphiFile.setTypeFactory(config.getTypeFactory());
       delphiFile.setAst(createAST(delphiFile, preprocessor.getTokenStream(), config));

delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFileConfig.java

@@ -22,17 +22,15 @@
 import au.com.integradev.delphi.preprocessor.search.SearchPath;
 import java.nio.charset.Charset;
 import java.util.Set;
-import javax.annotation.Nullable;
 import org.sonar.plugins.communitydelphi.api.type.TypeFactory;
 
 public interface DelphiFileConfig {
   /**
-   * Returns the encoding that the source file is expected to be
+   * Returns the charset that the source file is expected to be read with
    *
-   * @return Name of encoding
+   * @return Source charset
    */
-  @Nullable
-  String getEncoding();
+  Charset getCharset();
 
   /**
    * Returns the charset that will be used for interpreting ANSI data in the source file

delphi-frontend/src/main/java/au/com/integradev/delphi/msbuild/DelphiProjectHelper.java

@@ -66,6 +66,8 @@
 @SonarLintSide
 public class DelphiProjectHelper {
   private static final Logger LOG = LoggerFactory.getLogger(DelphiProjectHelper.class);
+  private static final String SOURCE_ENCODING_KEY = "sonar.sourceEncoding";
+
   private final Configuration settings;
   private final FileSystem fs;
   private final EnvironmentVariableProvider environmentVariableProvider;
@@ -480,7 +482,10 @@ public InputFile getFile(String path) {
     return fs.inputFile(fs.predicates().hasURI(Paths.get(path).toUri()));
   }
 
-  public String encoding() {
-    return fs != null ? fs.encoding().name() : Charset.defaultCharset().name();
+  public Charset getCharset() {
+    if (fs != null && settings.get(SOURCE_ENCODING_KEY).isPresent()) {
+      return fs.encoding();
+    }
+    return getAnsiCharset();
   }
 }

delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessor.java

@@ -260,7 +260,7 @@ private List<Token> processIncludeFile(String filename, Path includePath, Delphi
   }
 
   private List<Token> preprocessIncludeFile(DelphiToken location, Path path) throws IOException {
-    var fileStream = new DelphiFileStream(path.toAbsolutePath().toString(), config.getEncoding());
+    var fileStream = new DelphiFileStream(path.toAbsolutePath().toString(), config.getCharset());
     DelphiLexer includeLexer = new DelphiIncludeLexer(fileStream, location);
 
     DelphiPreprocessor preprocessor =

delphi-frontend/src/main/java/au/com/integradev/delphi/reporting/DelphiIssueBuilderImpl.java

@@ -256,7 +256,7 @@ public void report() {
   private static DelphiFileStream getDelphiFileStream(DelphiFile delphiFile) {
     try {
       return new DelphiFileStream(
-          delphiFile.getSourceCodeFile().getAbsolutePath(), delphiFile.getSourceCodeFileEncoding());
+          delphiFile.getSourceCodeFile().getAbsolutePath(), delphiFile.getSourceCodeFileCharset());
     } catch (IOException e) {
       throw new UncheckedIOException(e);
     }

delphi-frontend/src/main/java/au/com/integradev/delphi/symbol/SymbolTableBuilder.java

@@ -74,7 +74,7 @@ public class SymbolTableBuilder {
   private final Set<UnitData> sourceFileUnits = new HashSet<>();
   private final HashMap<String, UnitData> allUnitsByName = new HashMap<>();
   private final Set<Path> unitPaths = new HashSet<>();
-  private String encoding;
+  private Charset charset = CharsetUtils.nativeCharset();
   private Charset ansiCharset = CharsetUtils.nativeCharset();
   private DelphiPreprocessorFactory preprocessorFactory;
   private TypeFactory typeFactory;
@@ -109,8 +109,8 @@ public SymbolTableBuilder referencedFiles(List<Path> referencedFiles) {
     return this;
   }
 
-  public SymbolTableBuilder encoding(String encoding) {
-    this.encoding = encoding;
+  public SymbolTableBuilder charset(Charset charset) {
+    this.charset = charset;
     return this;
   }
 
@@ -267,9 +267,9 @@ private UnitData findImportByName(UnitNameDeclaration unit, String importName) {
     return allUnitsByName.get(importName.toLowerCase());
   }
 
-  private DelphiFileConfig createFileConfig(UnitData unit, boolean shouldSkipImplementation) {
+  private DelphiFileConfig createFileConfig(boolean shouldSkipImplementation) {
     return DelphiFile.createConfig(
-        sourceFileUnits.contains(unit) ? encoding : null,
+        charset,
         ansiCharset,
         preprocessorFactory,
         typeFactory,
@@ -293,7 +293,7 @@ private void process(UnitData unit, ResolutionLevel resolutionLevel) {
       }
 
       boolean shouldSkipImplementation = (resolutionLevel != ResolutionLevel.COMPLETE);
-      DelphiFileConfig fileConfig = createFileConfig(unit, shouldSkipImplementation);
+      DelphiFileConfig fileConfig = createFileConfig(shouldSkipImplementation);
       DelphiFile delphiFile = DelphiFile.from(unit.unitFile.toFile(), fileConfig);
 
       if (unit.resolved == ResolutionLevel.NONE) {

delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java

@@ -275,7 +275,7 @@ void testDoubleAmpersands() {
   void testUndefinedInaccessibleNestedIfDef() {
     fileConfig =
         DelphiFile.createConfig(
-            StandardCharsets.UTF_8.name(),
+            StandardCharsets.UTF_8,
             StandardCharsets.UTF_8,
             new DelphiPreprocessorFactory(
                 DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS),

delphi-frontend/src/test/java/au/com/integradev/delphi/coverage/delphicodecoveragetool/DelphiCoverageToolParserTest.java

@@ -77,7 +77,7 @@ private void addFile(String fileName, InputFile.Type type) throws IOException {
         TestInputFileBuilder.create("", baseDir, file)
             .setLanguage(Delphi.KEY)
             .setType(type)
-            .setContents(FileUtils.readFileToString(file, delphiProjectHelper.encoding()))
+            .setContents(FileUtils.readFileToString(file, delphiProjectHelper.getCharset()))
             .build();
     context.fileSystem().add(inputFile);
   }

delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMasterExecutorTest.java

@@ -179,7 +179,7 @@ private static DelphiFileConfig mockConfig() {
         new TypeFactoryImpl(
             DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT, DelphiProperties.COMPILER_VERSION_DEFAULT);
     DelphiFileConfig mock = mock(DelphiFileConfig.class);
-    when(mock.getEncoding()).thenReturn(StandardCharsets.UTF_8.name());
+    when(mock.getCharset()).thenReturn(StandardCharsets.UTF_8);
     when(mock.getPreprocessorFactory())
         .thenReturn(
             new DelphiPreprocessorFactory(

delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMetricsExecutorTest.java

@@ -200,7 +200,7 @@ private static DelphiFileConfig mockConfig() {
         new TypeFactoryImpl(
             DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT, DelphiProperties.COMPILER_VERSION_DEFAULT);
     DelphiFileConfig mock = mock(DelphiFileConfig.class);
-    when(mock.getEncoding()).thenReturn(StandardCharsets.UTF_8.name());
+    when(mock.getCharset()).thenReturn(StandardCharsets.UTF_8);
     when(mock.getPreprocessorFactory())
         .thenReturn(
             new DelphiPreprocessorFactory(

delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiSymbolTableExecutorTest.java

@@ -1565,7 +1565,7 @@ private void execute(String filename, String... searchPaths) {
             DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT, DelphiProperties.COMPILER_VERSION_DEFAULT);
 
     DelphiFileConfig fileConfig = mock(DelphiFileConfig.class);
-    when(fileConfig.getEncoding()).thenReturn(StandardCharsets.UTF_8.name());
+    when(fileConfig.getCharset()).thenReturn(StandardCharsets.UTF_8);
     when(fileConfig.getPreprocessorFactory()).thenReturn(preprocessorFactory);
     when(fileConfig.getTypeFactory()).thenReturn(typeFactory);
     when(fileConfig.getSearchPath()).thenReturn(SearchPath.create(Collections.emptyList()));
@@ -1583,6 +1583,7 @@ private void execute(String filename, String... searchPaths) {
 
     symbolTable =
         SymbolTable.builder()
+            .charset(StandardCharsets.UTF_8)
             .preprocessorFactory(preprocessorFactory)
             .typeFactory(typeFactory)
             .sourceFiles(sourceFiles)

delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiTokenExecutorTest.java

@@ -231,7 +231,7 @@ private DelphiInputFile makeDelphiFile(String filePath) {
               DelphiProperties.COMPILER_VERSION_DEFAULT);
 
       DelphiFileConfig fileConfig = mock(DelphiFileConfig.class);
-      when(fileConfig.getEncoding()).thenReturn(StandardCharsets.UTF_8.name());
+      when(fileConfig.getCharset()).thenReturn(StandardCharsets.UTF_8);
       when(fileConfig.getPreprocessorFactory())
           .thenReturn(
               new DelphiPreprocessorFactory(