diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ea16dbce..8f8522064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,12 +22,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- The ANSI encoding configured by `sonar.delphi.codePage` or inferred from `DCC_CodePage` is now + preferred over the system encoding when parsing source files. - Metrics are no longer reported on test sources. - Duplications are no longer reported on test sources. - Coverage data is no longer reported on test sources. ### Fixed +- The configured `sonar.sourceEncoding` was never used for search path units. - Quick fixes removing too much surrounding code in `RedundantInherited`. ## [1.18.3] - 2025-11-11 diff --git a/delphi-checks-testkit/src/main/java/au/com/integradev/delphi/builders/AbstractDelphiTestFile.java b/delphi-checks-testkit/src/main/java/au/com/integradev/delphi/builders/AbstractDelphiTestFile.java index 7971298dc..594fe01b3 100644 --- a/delphi-checks-testkit/src/main/java/au/com/integradev/delphi/builders/AbstractDelphiTestFile.java +++ b/delphi-checks-testkit/src/main/java/au/com/integradev/delphi/builders/AbstractDelphiTestFile.java @@ -85,7 +85,7 @@ private static DelphiFileConfig mockConfig() { new TypeFactoryImpl( DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT, DelphiProperties.COMPILER_VERSION_DEFAULT); DelphiFileConfig mock = mock(DelphiFileConfig.class); - when(mock.getEncoding()).thenReturn(UTF_8.name()); + when(mock.getCharset()).thenReturn(UTF_8); when(mock.getTypeFactory()).thenReturn(typeFactory); when(mock.getSearchPath()).thenReturn(SearchPath.create(Collections.emptyList())); when(mock.getDefinitions()).thenReturn(Collections.emptySet()); diff --git a/delphi-checks-testkit/src/main/java/au/com/integradev/delphi/checks/verifier/CheckVerifierImpl.java b/delphi-checks-testkit/src/main/java/au/com/integradev/delphi/checks/verifier/CheckVerifierImpl.java index e7c5b2bdd..f56f01588 100644 --- a/delphi-checks-testkit/src/main/java/au/com/integradev/delphi/checks/verifier/CheckVerifierImpl.java +++ b/delphi-checks-testkit/src/main/java/au/com/integradev/delphi/checks/verifier/CheckVerifierImpl.java @@ -323,7 +323,7 @@ private static Supplier newFileStreamSupplier(DelphiFile delph try { return new DelphiFileStream( delphiFile.getSourceCodeFile().getAbsolutePath(), - delphiFile.getSourceCodeFileEncoding()); + delphiFile.getSourceCodeFileCharset()); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/delphi-frontend/src/main/java/au/com/integradev/delphi/antlr/DelphiFileStream.java b/delphi-frontend/src/main/java/au/com/integradev/delphi/antlr/DelphiFileStream.java index ad7f924d7..2848a3d51 100644 --- a/delphi-frontend/src/main/java/au/com/integradev/delphi/antlr/DelphiFileStream.java +++ b/delphi-frontend/src/main/java/au/com/integradev/delphi/antlr/DelphiFileStream.java @@ -29,34 +29,30 @@ public class DelphiFileStream extends ANTLRStringStream { private final String fileName; - private final String encoding; + private final Charset charset; - public DelphiFileStream(String fileName, String encoding) throws IOException { + public DelphiFileStream(String fileName, Charset charset) throws IOException { this.fileName = fileName; - this.encoding = this.load(fileName, encoding); + this.charset = this.load(fileName, charset); } - private String load(String fileName, String encoding) throws IOException { + private Charset load(String fileName, Charset charset) throws IOException { if (fileName != null) { File f = new File(fileName); int size = (int) f.length(); try (BOMInputStream input = bomInputStream(fileName).get()) { ByteOrderMark bom = input.getBOM(); if (bom != null) { - encoding = bom.getCharsetName(); + charset = Charset.forName(bom.getCharsetName()); } - if (encoding == null) { - encoding = Charset.defaultCharset().name(); - } - - try (InputStreamReader reader = new InputStreamReader(input, encoding)) { + try (InputStreamReader reader = new InputStreamReader(input, charset)) { this.data = new char[size]; super.n = reader.read(this.data); } } } - return encoding; + return charset; } @Override @@ -64,8 +60,8 @@ public String getSourceName() { return this.fileName; } - public String getEncoding() { - return this.encoding; + public Charset getCharset() { + return this.charset; } private static BOMInputStream.Builder bomInputStream(String fileName) throws IOException { diff --git a/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFile.java b/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFile.java index f2c1aa23e..8814a43b6 100644 --- a/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFile.java +++ b/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFile.java @@ -36,7 +36,7 @@ class DefaultDelphiFile implements DelphiFile { private CompilerSwitchRegistry switchRegistry; private TextBlockLineEndingModeRegistry textBlockLineEndingModeRegistry; private TypeFactory typeFactory; - private String encoding; + private Charset charset; private Charset ansiCharset; DefaultDelphiFile() { @@ -54,8 +54,8 @@ public List getSourceCodeFileLines() { } @Override - public String getSourceCodeFileEncoding() { - return encoding; + public Charset getSourceCodeFileCharset() { + return charset; } @Override @@ -101,8 +101,8 @@ void setSourceCodeLines(List sourceCodeLines) { this.sourceCodeLines = List.copyOf(sourceCodeLines); } - void setSourceCodeEncoding(String encoding) { - this.encoding = encoding; + void setSourceCodeCharset(Charset charset) { + this.charset = charset; } void setAnsiCharset(Charset ansiCharset) { diff --git a/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFileConfig.java b/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFileConfig.java index a0c14ad75..051a5fa7c 100644 --- a/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFileConfig.java +++ b/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DefaultDelphiFileConfig.java @@ -22,11 +22,10 @@ import au.com.integradev.delphi.preprocessor.search.SearchPath; import java.nio.charset.Charset; import java.util.Set; -import javax.annotation.Nullable; import org.sonar.plugins.communitydelphi.api.type.TypeFactory; public class DefaultDelphiFileConfig implements DelphiFileConfig { - private final String encoding; + private final Charset charset; private final Charset ansiCharset; private final DelphiPreprocessorFactory preprocessorFactory; private final TypeFactory typeFactory; @@ -35,14 +34,14 @@ public class DefaultDelphiFileConfig implements DelphiFileConfig { private final boolean skipImplementation; DefaultDelphiFileConfig( - String encoding, + Charset charset, Charset ansiCharset, DelphiPreprocessorFactory preprocessorFactory, TypeFactory typeFactory, SearchPath searchPath, Set definitions, boolean skipImplementation) { - this.encoding = encoding; + this.charset = charset; this.ansiCharset = ansiCharset; this.preprocessorFactory = preprocessorFactory; this.typeFactory = typeFactory; @@ -51,10 +50,9 @@ public class DefaultDelphiFileConfig implements DelphiFileConfig { this.skipImplementation = skipImplementation; } - @Nullable @Override - public String getEncoding() { - return encoding; + public Charset getCharset() { + return charset; } @Override diff --git a/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFile.java b/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFile.java index a205ddfa2..98517841f 100644 --- a/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFile.java +++ b/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFile.java @@ -39,7 +39,6 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import javax.annotation.Nullable; import org.antlr.runtime.BufferedTokenStream; import org.antlr.runtime.CommonToken; import org.antlr.runtime.RecognitionException; @@ -55,7 +54,7 @@ public interface DelphiFile { List getSourceCodeFileLines(); - String getSourceCodeFileEncoding(); + Charset getSourceCodeFileCharset(); DelphiAst getAst(); @@ -84,10 +83,10 @@ static DelphiInputFile from(InputFile inputFile, DelphiFileConfig config) { private static DelphiFileConfig useInputFileEncoding( InputFile inputFile, DelphiFileConfig config) { - if (inputFile.charset() != null && !inputFile.charset().name().equals(config.getEncoding())) { + if (inputFile.charset() != null && !inputFile.charset().equals(config.getCharset())) { config = DelphiFile.createConfig( - inputFile.charset().name(), + inputFile.charset(), config.getAnsiCharset(), config.getPreprocessorFactory(), config.getTypeFactory(), @@ -112,18 +111,18 @@ class EmptyDelphiFileException extends RuntimeException { } static DelphiFileConfig createConfig( - String encoding, + Charset charset, Charset ansiCharset, DelphiPreprocessorFactory preprocessorFactory, TypeFactory typeFactory, SearchPath searchPath, Set definitions) { return createConfig( - encoding, ansiCharset, preprocessorFactory, typeFactory, searchPath, definitions, false); + charset, ansiCharset, preprocessorFactory, typeFactory, searchPath, definitions, false); } static DelphiFileConfig createConfig( - @Nullable String encoding, + Charset charset, Charset ansiCharset, DelphiPreprocessorFactory preprocessorFactory, TypeFactory typeFactory, @@ -131,7 +130,7 @@ static DelphiFileConfig createConfig( Set definitions, boolean shouldSkipImplementation) { return new DefaultDelphiFileConfig( - encoding, + charset, ansiCharset, preprocessorFactory, typeFactory, @@ -150,10 +149,10 @@ private static void setupFile( DefaultDelphiFile delphiFile, File sourceFile, DelphiFileConfig config) { try { String filePath = sourceFile.getAbsolutePath(); - DelphiFileStream fileStream = new DelphiFileStream(filePath, config.getEncoding()); + DelphiFileStream fileStream = new DelphiFileStream(filePath, config.getCharset()); DelphiPreprocessor preprocessor = preprocess(fileStream, config); delphiFile.setSourceCodeFile(sourceFile); - delphiFile.setSourceCodeEncoding(fileStream.getEncoding()); + delphiFile.setSourceCodeCharset(fileStream.getCharset()); delphiFile.setAnsiCharset(config.getAnsiCharset()); delphiFile.setTypeFactory(config.getTypeFactory()); delphiFile.setAst(createAST(delphiFile, preprocessor.getTokenStream(), config)); diff --git a/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFileConfig.java b/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFileConfig.java index 21fa614b4..76ad91577 100644 --- a/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFileConfig.java +++ b/delphi-frontend/src/main/java/au/com/integradev/delphi/file/DelphiFileConfig.java @@ -22,17 +22,15 @@ import au.com.integradev.delphi.preprocessor.search.SearchPath; import java.nio.charset.Charset; import java.util.Set; -import javax.annotation.Nullable; import org.sonar.plugins.communitydelphi.api.type.TypeFactory; public interface DelphiFileConfig { /** - * Returns the encoding that the source file is expected to be + * Returns the charset that the source file is expected to be read with * - * @return Name of encoding + * @return Source charset */ - @Nullable - String getEncoding(); + Charset getCharset(); /** * Returns the charset that will be used for interpreting ANSI data in the source file diff --git a/delphi-frontend/src/main/java/au/com/integradev/delphi/msbuild/DelphiProjectHelper.java b/delphi-frontend/src/main/java/au/com/integradev/delphi/msbuild/DelphiProjectHelper.java index 6e87d4212..f73216e73 100644 --- a/delphi-frontend/src/main/java/au/com/integradev/delphi/msbuild/DelphiProjectHelper.java +++ b/delphi-frontend/src/main/java/au/com/integradev/delphi/msbuild/DelphiProjectHelper.java @@ -66,6 +66,8 @@ @SonarLintSide public class DelphiProjectHelper { private static final Logger LOG = LoggerFactory.getLogger(DelphiProjectHelper.class); + private static final String SOURCE_ENCODING_KEY = "sonar.sourceEncoding"; + private final Configuration settings; private final FileSystem fs; private final EnvironmentVariableProvider environmentVariableProvider; @@ -480,7 +482,10 @@ public InputFile getFile(String path) { return fs.inputFile(fs.predicates().hasURI(Paths.get(path).toUri())); } - public String encoding() { - return fs != null ? fs.encoding().name() : Charset.defaultCharset().name(); + public Charset getCharset() { + if (fs != null && settings.get(SOURCE_ENCODING_KEY).isPresent()) { + return fs.encoding(); + } + return getAnsiCharset(); } } diff --git a/delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessor.java b/delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessor.java index f5fba1861..e8372d07e 100644 --- a/delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessor.java +++ b/delphi-frontend/src/main/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessor.java @@ -260,7 +260,7 @@ private List processIncludeFile(String filename, Path includePath, Delphi } private List preprocessIncludeFile(DelphiToken location, Path path) throws IOException { - var fileStream = new DelphiFileStream(path.toAbsolutePath().toString(), config.getEncoding()); + var fileStream = new DelphiFileStream(path.toAbsolutePath().toString(), config.getCharset()); DelphiLexer includeLexer = new DelphiIncludeLexer(fileStream, location); DelphiPreprocessor preprocessor = diff --git a/delphi-frontend/src/main/java/au/com/integradev/delphi/reporting/DelphiIssueBuilderImpl.java b/delphi-frontend/src/main/java/au/com/integradev/delphi/reporting/DelphiIssueBuilderImpl.java index 2bbec90f8..a27d813dc 100644 --- a/delphi-frontend/src/main/java/au/com/integradev/delphi/reporting/DelphiIssueBuilderImpl.java +++ b/delphi-frontend/src/main/java/au/com/integradev/delphi/reporting/DelphiIssueBuilderImpl.java @@ -256,7 +256,7 @@ public void report() { private static DelphiFileStream getDelphiFileStream(DelphiFile delphiFile) { try { return new DelphiFileStream( - delphiFile.getSourceCodeFile().getAbsolutePath(), delphiFile.getSourceCodeFileEncoding()); + delphiFile.getSourceCodeFile().getAbsolutePath(), delphiFile.getSourceCodeFileCharset()); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/delphi-frontend/src/main/java/au/com/integradev/delphi/symbol/SymbolTableBuilder.java b/delphi-frontend/src/main/java/au/com/integradev/delphi/symbol/SymbolTableBuilder.java index d7cde92f2..bc6557d1e 100644 --- a/delphi-frontend/src/main/java/au/com/integradev/delphi/symbol/SymbolTableBuilder.java +++ b/delphi-frontend/src/main/java/au/com/integradev/delphi/symbol/SymbolTableBuilder.java @@ -74,7 +74,7 @@ public class SymbolTableBuilder { private final Set sourceFileUnits = new HashSet<>(); private final HashMap allUnitsByName = new HashMap<>(); private final Set unitPaths = new HashSet<>(); - private String encoding; + private Charset charset = CharsetUtils.nativeCharset(); private Charset ansiCharset = CharsetUtils.nativeCharset(); private DelphiPreprocessorFactory preprocessorFactory; private TypeFactory typeFactory; @@ -109,8 +109,8 @@ public SymbolTableBuilder referencedFiles(List referencedFiles) { return this; } - public SymbolTableBuilder encoding(String encoding) { - this.encoding = encoding; + public SymbolTableBuilder charset(Charset charset) { + this.charset = charset; return this; } @@ -267,9 +267,9 @@ private UnitData findImportByName(UnitNameDeclaration unit, String importName) { return allUnitsByName.get(importName.toLowerCase()); } - private DelphiFileConfig createFileConfig(UnitData unit, boolean shouldSkipImplementation) { + private DelphiFileConfig createFileConfig(boolean shouldSkipImplementation) { return DelphiFile.createConfig( - sourceFileUnits.contains(unit) ? encoding : null, + charset, ansiCharset, preprocessorFactory, typeFactory, @@ -293,7 +293,7 @@ private void process(UnitData unit, ResolutionLevel resolutionLevel) { } boolean shouldSkipImplementation = (resolutionLevel != ResolutionLevel.COMPLETE); - DelphiFileConfig fileConfig = createFileConfig(unit, shouldSkipImplementation); + DelphiFileConfig fileConfig = createFileConfig(shouldSkipImplementation); DelphiFile delphiFile = DelphiFile.from(unit.unitFile.toFile(), fileConfig); if (unit.resolved == ResolutionLevel.NONE) { diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java index 1989ae645..1ec18c6eb 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/antlr/GrammarTest.java @@ -275,7 +275,7 @@ void testDoubleAmpersands() { void testUndefinedInaccessibleNestedIfDef() { fileConfig = DelphiFile.createConfig( - StandardCharsets.UTF_8.name(), + StandardCharsets.UTF_8, StandardCharsets.UTF_8, new DelphiPreprocessorFactory( DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS), diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/coverage/delphicodecoveragetool/DelphiCoverageToolParserTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/coverage/delphicodecoveragetool/DelphiCoverageToolParserTest.java index b7055de24..6e64c8558 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/coverage/delphicodecoveragetool/DelphiCoverageToolParserTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/coverage/delphicodecoveragetool/DelphiCoverageToolParserTest.java @@ -77,7 +77,7 @@ private void addFile(String fileName, InputFile.Type type) throws IOException { TestInputFileBuilder.create("", baseDir, file) .setLanguage(Delphi.KEY) .setType(type) - .setContents(FileUtils.readFileToString(file, delphiProjectHelper.encoding())) + .setContents(FileUtils.readFileToString(file, delphiProjectHelper.getCharset())) .build(); context.fileSystem().add(inputFile); } diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMasterExecutorTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMasterExecutorTest.java index 5c38ce212..898f635d3 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMasterExecutorTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMasterExecutorTest.java @@ -179,7 +179,7 @@ private static DelphiFileConfig mockConfig() { new TypeFactoryImpl( DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT, DelphiProperties.COMPILER_VERSION_DEFAULT); DelphiFileConfig mock = mock(DelphiFileConfig.class); - when(mock.getEncoding()).thenReturn(StandardCharsets.UTF_8.name()); + when(mock.getCharset()).thenReturn(StandardCharsets.UTF_8); when(mock.getPreprocessorFactory()) .thenReturn( new DelphiPreprocessorFactory( diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMetricsExecutorTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMetricsExecutorTest.java index ed5e97e07..980df6bdd 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMetricsExecutorTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiMetricsExecutorTest.java @@ -200,7 +200,7 @@ private static DelphiFileConfig mockConfig() { new TypeFactoryImpl( DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT, DelphiProperties.COMPILER_VERSION_DEFAULT); DelphiFileConfig mock = mock(DelphiFileConfig.class); - when(mock.getEncoding()).thenReturn(StandardCharsets.UTF_8.name()); + when(mock.getCharset()).thenReturn(StandardCharsets.UTF_8); when(mock.getPreprocessorFactory()) .thenReturn( new DelphiPreprocessorFactory( diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiSymbolTableExecutorTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiSymbolTableExecutorTest.java index 2d61a6014..638d925d1 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiSymbolTableExecutorTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiSymbolTableExecutorTest.java @@ -1565,7 +1565,7 @@ private void execute(String filename, String... searchPaths) { DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT, DelphiProperties.COMPILER_VERSION_DEFAULT); DelphiFileConfig fileConfig = mock(DelphiFileConfig.class); - when(fileConfig.getEncoding()).thenReturn(StandardCharsets.UTF_8.name()); + when(fileConfig.getCharset()).thenReturn(StandardCharsets.UTF_8); when(fileConfig.getPreprocessorFactory()).thenReturn(preprocessorFactory); when(fileConfig.getTypeFactory()).thenReturn(typeFactory); when(fileConfig.getSearchPath()).thenReturn(SearchPath.create(Collections.emptyList())); @@ -1583,6 +1583,7 @@ private void execute(String filename, String... searchPaths) { symbolTable = SymbolTable.builder() + .charset(StandardCharsets.UTF_8) .preprocessorFactory(preprocessorFactory) .typeFactory(typeFactory) .sourceFiles(sourceFiles) diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiTokenExecutorTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiTokenExecutorTest.java index 6842546ca..aab3a860d 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiTokenExecutorTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/executor/DelphiTokenExecutorTest.java @@ -231,7 +231,7 @@ private DelphiInputFile makeDelphiFile(String filePath) { DelphiProperties.COMPILER_VERSION_DEFAULT); DelphiFileConfig fileConfig = mock(DelphiFileConfig.class); - when(fileConfig.getEncoding()).thenReturn(StandardCharsets.UTF_8.name()); + when(fileConfig.getCharset()).thenReturn(StandardCharsets.UTF_8); when(fileConfig.getPreprocessorFactory()) .thenReturn( new DelphiPreprocessorFactory( diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/file/DelphiFileTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/file/DelphiFileTest.java index 440c43048..bbca8b183 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/file/DelphiFileTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/file/DelphiFileTest.java @@ -66,7 +66,7 @@ void testInputFileEncodingShouldOverrideProvidedEncoding() { DelphiFileConfig config = DelphiFile.createConfig( - StandardCharsets.UTF_8.name(), + StandardCharsets.UTF_8, StandardCharsets.UTF_8, new DelphiPreprocessorFactory( DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS), @@ -84,7 +84,7 @@ void testByteOrderMarkShouldOverrideProvidedEncoding() { DelphiFileConfig config = DelphiFile.createConfig( - StandardCharsets.UTF_8.name(), + StandardCharsets.UTF_8, StandardCharsets.UTF_8, new DelphiPreprocessorFactory( DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS), @@ -102,7 +102,7 @@ void testByteOrderMarkShouldBeStripped() { DelphiFileConfig config = DelphiFile.createConfig( - StandardCharsets.UTF_8.name(), + StandardCharsets.UTF_8, StandardCharsets.UTF_8, new DelphiPreprocessorFactory( DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS), @@ -115,6 +115,27 @@ void testByteOrderMarkShouldBeStripped() { assertThat(firstLine).doesNotStartWith("\ufeff"); } + @Test + void testCharsetShouldDecodeShiftJisFile() { + File file = DelphiUtils.getResource("/au/com/integradev/delphi/file/ShiftJis.pas"); + Charset shiftJis = Charset.forName("windows-31j"); + + DelphiFileConfig config = + DelphiFile.createConfig( + shiftJis, + shiftJis, + new DelphiPreprocessorFactory( + DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS), + TypeFactoryUtils.defaultFactory(), + SearchPath.create(Collections.emptyList()), + Collections.emptySet()); + + DelphiFile delphiFile = DelphiFile.from(file, config); + assertThat(delphiFile.getSourceCodeFileCharset()).isEqualTo(shiftJis); + assertThat(delphiFile.getSourceCodeFileLines().get(1)).isEqualTo("カスタマイズ"); + assertThat(delphiFile.getSourceCodeFileLines().get(3)).isEqualTo("unit Consts;"); + } + @ParameterizedTest @ValueSource(strings = {"SkipImplementation.pas", "SkipImplementationWithDirectiveNesting.pas"}) void testShouldSkipImplementation(String filename) { @@ -122,7 +143,7 @@ void testShouldSkipImplementation(String filename) { DelphiFileConfig config = DelphiFile.createConfig( - StandardCharsets.UTF_8.name(), + StandardCharsets.UTF_8, StandardCharsets.UTF_8, new DelphiPreprocessorFactory( DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS), diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/msbuild/DelphiProjectHelperTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/msbuild/DelphiProjectHelperTest.java index b89b61349..be3f349b2 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/msbuild/DelphiProjectHelperTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/msbuild/DelphiProjectHelperTest.java @@ -55,6 +55,7 @@ class DelphiProjectHelperTest { private static final String PROJECTS_PATH = "/au/com/integradev/delphi/projects/"; + private static final String SOURCE_ENCODING_KEY = "sonar.sourceEncoding"; private static final File BASE_DIR = DelphiUtils.getResource(PROJECTS_PATH); private Configuration settings; private DefaultFileSystem fs; @@ -78,6 +79,7 @@ void setup() throws IOException { when(settings.get(DelphiProperties.INSTALLATION_PATH_KEY)) .thenReturn(Optional.of(installationPath)); + when(settings.get(SOURCE_ENCODING_KEY)).thenReturn(Optional.empty()); when(environmentVariableProvider.getenv()).thenReturn(Collections.emptyMap()); when(environmentVariableProvider.getenv(anyString())).thenReturn(null); @@ -331,12 +333,12 @@ void testProjectCodePageProvidesAnsiCharset() { @Test void testConfiguredCodePageOverridesProjectCodePage() { addInputFile(PROJECTS_PATH + "CodePageProject/Utf8.dproj"); - when(settings.get(DelphiProperties.CODE_PAGE_KEY)).thenReturn(Optional.of("1252")); + when(settings.get(DelphiProperties.CODE_PAGE_KEY)).thenReturn(Optional.of("1251")); DelphiProjectHelper delphiProjectHelper = new DelphiProjectHelper(settings, fs, environmentVariableProvider); - assertThat(delphiProjectHelper.getAnsiCharset().name()).isEqualTo("windows-1252"); + assertThat(delphiProjectHelper.getAnsiCharset().name()).isEqualTo("windows-1251"); } @Test @@ -422,6 +424,28 @@ void testAcpProjectCodePageUsesNativeCharsetWithoutConflictWhenItMatchesExplicit assertThat(delphiProjectHelper.getAnsiCharset()).isEqualTo(Charset.forName("windows-1252")); } + @Test + void testConfiguredSourceEncodingProvidesCharset() { + fs.setEncoding(StandardCharsets.UTF_16LE); + when(settings.get(SOURCE_ENCODING_KEY)).thenReturn(Optional.of("UTF-16LE")); + + DelphiProjectHelper delphiProjectHelper = + new DelphiProjectHelper(settings, fs, environmentVariableProvider); + + assertThat(delphiProjectHelper.getCharset()).isEqualTo(StandardCharsets.UTF_16LE); + } + + @Test + void testMissingSourceEncodingFallsBackToAnsiCharset() { + addInputFile(PROJECTS_PATH + "CodePageProject/Windows1252.dproj"); + fs.setEncoding(StandardCharsets.UTF_8); + + DelphiProjectHelper delphiProjectHelper = + new DelphiProjectHelper(settings, fs, environmentVariableProvider); + + assertThat(delphiProjectHelper.getCharset().name()).isEqualTo("windows-1252"); + } + @Test void testNoFilesExist(@TempDir Path tempDir) { fs = new DefaultFileSystem(tempDir); diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessorTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessorTest.java index 181a8ab26..5a52681c5 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessorTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/preprocessor/DelphiPreprocessorTest.java @@ -133,7 +133,7 @@ void testCallingProcessTwiceShouldThrowException() throws Exception { String filePath = DelphiUtils.getResource(BASE_DIR + "includeTest/SameNameBacktrack.pas").getAbsolutePath(); DelphiFileConfig config = DelphiFileUtils.mockConfig(); - DelphiFileStream fileStream = new DelphiFileStream(filePath, config.getEncoding()); + DelphiFileStream fileStream = new DelphiFileStream(filePath, config.getCharset()); DelphiLexer lexer = new DelphiLexer(fileStream); DelphiPreprocessor preprocessor = @@ -147,7 +147,7 @@ void testCallingProcessTwiceShouldThrowException() throws Exception { private static void executeWithDefines(String filename, String... defines) { DelphiFileConfig config = DelphiFile.createConfig( - UTF_8.name(), + UTF_8, UTF_8, new DelphiPreprocessorFactory( DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS), @@ -168,7 +168,7 @@ private static void executeWithSearchPath(String filename, String... directories execute( filename, DelphiFile.createConfig( - UTF_8.name(), + UTF_8, UTF_8, new DelphiPreprocessorFactory( DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS), @@ -184,7 +184,7 @@ private static void execute(String filename) { private static void execute(String filename, DelphiFileConfig config) { try { String filePath = DelphiUtils.getResource(BASE_DIR + filename).getAbsolutePath(); - DelphiFileStream fileStream = new DelphiFileStream(filePath, config.getEncoding()); + DelphiFileStream fileStream = new DelphiFileStream(filePath, config.getCharset()); DelphiLexer lexer = new DelphiLexer(fileStream); DelphiPreprocessor preprocessor = diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/reporting/QuickFixEditTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/reporting/QuickFixEditTest.java index 2af945163..1175047e2 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/reporting/QuickFixEditTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/reporting/QuickFixEditTest.java @@ -56,7 +56,7 @@ class QuickFixEditTest { private static DelphiFileStream getTestStream() { try { return new DelphiFileStream( - DelphiUtils.getResource(TEST_UNIT_PATH).getAbsolutePath(), StandardCharsets.UTF_8.name()); + DelphiUtils.getResource(TEST_UNIT_PATH).getAbsolutePath(), StandardCharsets.UTF_8); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -144,7 +144,7 @@ private static DelphiAst getTestAst() { DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT, DelphiProperties.COMPILER_VERSION_DEFAULT); DelphiFileConfig fileConfig = mock(DelphiFileConfig.class); - when(fileConfig.getEncoding()).thenReturn(StandardCharsets.UTF_8.name()); + when(fileConfig.getCharset()).thenReturn(StandardCharsets.UTF_8); when(fileConfig.getPreprocessorFactory()).thenReturn(preprocessorFactory); when(fileConfig.getTypeFactory()).thenReturn(typeFactory); when(fileConfig.getSearchPath()).thenReturn(SearchPath.create(Collections.emptyList())); diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/symbol/SymbolTableBuilderTest.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/symbol/SymbolTableBuilderTest.java index c3036fdc2..a2786523c 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/symbol/SymbolTableBuilderTest.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/symbol/SymbolTableBuilderTest.java @@ -30,8 +30,10 @@ import au.com.integradev.delphi.preprocessor.DelphiPreprocessorFactory; import au.com.integradev.delphi.preprocessor.search.SearchPath; import au.com.integradev.delphi.symbol.SymbolTableBuilder.SymbolTableConstructionException; +import au.com.integradev.delphi.utils.DelphiUtils; import au.com.integradev.delphi.utils.types.TypeFactoryUtils; import java.io.IOException; +import java.nio.charset.Charset; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; @@ -263,6 +265,44 @@ void testRecognitionErrorInImportShouldNotThrow( assertThatCode(symbolTable::build).doesNotThrowAnyException(); } + @Test + void testSearchPathImportsUseConfiguredCharset( + @TempDir Path standardLibraryPath, @TempDir Path tempDir) throws IOException { + createStandardLibrary(standardLibraryPath); + + Path searchPathRoot = tempDir.resolve("include"); + Files.createDirectories(searchPathRoot); + + Path importUnitPath = searchPathRoot.resolve("Consts.pas"); + Files.copy( + DelphiUtils.getResource("/au/com/integradev/delphi/file/ShiftJis.pas").toPath(), + importUnitPath); + + Path sourceFilePath = tempDir.resolve("SourceFile.pas"); + Files.writeString( + sourceFilePath, + "unit SourceFile;\n" + + "interface\n" + + "uses\n" + + " Consts;\n" + + "implementation\n" + + "end."); + + SymbolTable symbolTable = + SymbolTable.builder() + .charset(Charset.forName("windows-31j")) + .preprocessorFactory( + new DelphiPreprocessorFactory( + DelphiProperties.COMPILER_VERSION_DEFAULT, Platform.WINDOWS)) + .typeFactory(TypeFactoryUtils.defaultFactory()) + .standardLibraryPath(standardLibraryPath) + .sourceFiles(List.of(sourceFilePath)) + .searchPath(SearchPath.create(List.of(searchPathRoot))) + .build(); + + assertThat(symbolTable.getUnitByPath(importUnitPath.toString())).isNotNull(); + } + private static void createStandardLibrary(Path path) throws IOException { Files.writeString( path.resolve("SysInit.pas"), diff --git a/delphi-frontend/src/test/java/au/com/integradev/delphi/utils/files/DelphiFileUtils.java b/delphi-frontend/src/test/java/au/com/integradev/delphi/utils/files/DelphiFileUtils.java index 139032118..3d200af8b 100644 --- a/delphi-frontend/src/test/java/au/com/integradev/delphi/utils/files/DelphiFileUtils.java +++ b/delphi-frontend/src/test/java/au/com/integradev/delphi/utils/files/DelphiFileUtils.java @@ -54,7 +54,7 @@ public static DelphiFile parse(String... lines) { public static DelphiFileConfig mockConfig() { DelphiFileConfig mock = mock(DelphiFileConfig.class); - when(mock.getEncoding()).thenReturn(StandardCharsets.UTF_8.name()); + when(mock.getCharset()).thenReturn(StandardCharsets.UTF_8); when(mock.getPreprocessorFactory()) .thenReturn( new DelphiPreprocessorFactory( diff --git a/delphi-frontend/src/test/resources/au/com/integradev/delphi/file/ShiftJis.pas b/delphi-frontend/src/test/resources/au/com/integradev/delphi/file/ShiftJis.pas new file mode 100644 index 000000000..e6e81b945 --- /dev/null +++ b/delphi-frontend/src/test/resources/au/com/integradev/delphi/file/ShiftJis.pas @@ -0,0 +1,13 @@ +{ +ƒJƒXƒ^ƒ}ƒCƒY +} +unit Consts; + +interface + +const + SUMMARY = 'sum'; + +implementation + +end. diff --git a/sonar-delphi-plugin/src/main/java/au/com/integradev/delphi/DelphiSensor.java b/sonar-delphi-plugin/src/main/java/au/com/integradev/delphi/DelphiSensor.java index c437b847f..0c1228a1c 100644 --- a/sonar-delphi-plugin/src/main/java/au/com/integradev/delphi/DelphiSensor.java +++ b/sonar-delphi-plugin/src/main/java/au/com/integradev/delphi/DelphiSensor.java @@ -113,7 +113,7 @@ private void executeOnFiles(SensorContext sensorContext) { .typeFactory(typeFactory) .sourceFiles(sourceFiles) .referencedFiles(referencedFiles) - .encoding(delphiProjectHelper.encoding()) + .charset(delphiProjectHelper.getCharset()) .ansiCharset(delphiProjectHelper.getAnsiCharset()) .searchPath(searchPath) .conditionalDefines(delphiProjectHelper.getConditionalDefines()) @@ -131,7 +131,7 @@ private void executeOnFiles(SensorContext sensorContext) { ExecutorContext executorContext = new ExecutorContext(sensorContext, symbolTable); DelphiFileConfig config = DelphiFile.createConfig( - delphiProjectHelper.encoding(), + delphiProjectHelper.getCharset(), delphiProjectHelper.getAnsiCharset(), preprocessorFactory, typeFactory, diff --git a/sonar-delphi-plugin/src/test/java/au/com/integradev/delphi/DelphiSensorTest.java b/sonar-delphi-plugin/src/test/java/au/com/integradev/delphi/DelphiSensorTest.java index b4c0f55f0..b76e57959 100644 --- a/sonar-delphi-plugin/src/test/java/au/com/integradev/delphi/DelphiSensorTest.java +++ b/sonar-delphi-plugin/src/test/java/au/com/integradev/delphi/DelphiSensorTest.java @@ -32,9 +32,12 @@ import au.com.integradev.delphi.msbuild.DelphiProjectHelper; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.AfterEach; @@ -58,10 +61,20 @@ void setup() throws IOException { baseDir = Files.createTempDirectory("baseDir"); sensor = new DelphiSensor(delphiProjectHelper, executor); + when(delphiProjectHelper.getCharset()).thenReturn(StandardCharsets.UTF_8); + when(delphiProjectHelper.getAnsiCharset()).thenReturn(StandardCharsets.UTF_8); when(delphiProjectHelper.getToolchain()) .thenReturn(DelphiProperties.COMPILER_TOOLCHAIN_DEFAULT); when(delphiProjectHelper.getCompilerVersion()) .thenReturn(DelphiProperties.COMPILER_VERSION_DEFAULT); + when(delphiProjectHelper.getConditionalDefines()).thenReturn(Set.of()); + when(delphiProjectHelper.getReferencedFiles()).thenReturn(List.of()); + when(delphiProjectHelper.getSearchDirectories()).thenReturn(List.of()); + when(delphiProjectHelper.getDebugSourceDirectories()).thenReturn(List.of()); + when(delphiProjectHelper.getLibraryPathDirectories()).thenReturn(List.of()); + when(delphiProjectHelper.getBrowsingPathDirectories()).thenReturn(List.of()); + when(delphiProjectHelper.getUnitScopeNames()).thenReturn(Set.of()); + when(delphiProjectHelper.getUnitAliases()).thenReturn(Map.of()); Path standardLibraryPath = Files.createDirectories(baseDir.resolve("bds/source"));