Skip to content

DeepL: added Glossary and HTML functionality #100

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
thornch wants to merge 3 commits into
base: main
Choose a base branch
from
Open

DeepL: added Glossary and HTML functionality #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1cadd60
added DeepL config options for using glossary by id or name and tag h…
aproda-sso Jun 18, 2026
71cc4e7
DeepL: invalidate resolved glossary if lookup by name and changed id
aproda-sso Jun 18, 2026
b811bb8
DeepL: added dictionary cache for different glossary languages with s…
aproda-sso Jun 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions text/Squidex.Text/Translations/DeepL/DeepLTranslationOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@ public sealed class DeepLTranslationOptions
{
public string AuthKey { get; set; }

public string? GlossaryById { get; set; }

public string? GlossaryByName { get; set; }

public string? TagHandling { get; set; }

public decimal CostsPerCharacterInEUR { get; set; } = 20m / 1_000_000;

public Dictionary<string, string> Mapping { get; set; }
Expand Down
122 changes: 119 additions & 3 deletions text/Squidex.Text/Translations/DeepL/DeepLTranslationService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================

using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using System.Net;
using System.Text.Json;
Expand All @@ -19,6 +20,8 @@ public sealed class DeepLTranslationService(IHttpClientFactory httpClientFactory
private const string UrlPaid = "https://api.deepl.com/v2/translate";
private const string UrlFree = "https://api-free.deepl.com/v2/translate";
private readonly DeepLTranslationOptions options = options.Value;
private readonly ConcurrentDictionary<string, string?> glossaryCache = new ();

@SebastianStehle SebastianStehle Jun 18, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What exactly is this cache and how much entries can we expect here? I have not digged into this glossary stuff.

@thornch thornch Jun 20, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only the glossary ids per source/target language. It depends on how much translation glossaries are stored with the same glossary name.
E.g.:
DE_FR:{GUID-like-glossary-id}
DE_EN:...
DE_IT:...
FR_DE:...

I guess most will have one source language and create a dictionary for each translations. In our case there are mostly just one or two glossary maintained.

private bool glossaryResolved;

private sealed class TranslationsDto
{
Expand All @@ -35,6 +38,36 @@ private sealed class TranslationDto
public string DetectedSourceLanguage { get; set; }
}

private sealed class GlossariesDto
{
[JsonPropertyName("glossaries")]
public GlossaryDto[] Glossaries { get; set; }
}

private sealed class GlossaryDto
{
[JsonPropertyName("glossary_id")]
public string GlossaryId { get; set; } // "def3a26b-3e84-45b3-84ae-0c0aaf3525f7"

[JsonPropertyName("name")]
public string Name { get; set; } // "My Glossary"

[JsonPropertyName("ready")]
public bool Ready { get; set; } // true

[JsonPropertyName("source_lang")]
public string SourceLang { get; set; } // "en"

[JsonPropertyName("target_lang")]
public string TargetLang { get; set; } // "de"

[JsonPropertyName("creation_time")]
public DateTime CreationTime { get; set; } // "2021-08-03T14:16:18.329Z"

[JsonPropertyName("entry_count")]
public int EntryCount { get; set; } // 1
}

public bool IsConfigured { get; } = !string.IsNullOrWhiteSpace(options.Value.AuthKey);

public async Task<IReadOnlyList<TranslationResult>> TranslateAsync(IEnumerable<string> texts, string targetLanguage, string? sourceLanguage = null,
Expand All @@ -59,19 +92,22 @@ public async Task<IReadOnlyList<TranslationResult>> TranslateAsync(IEnumerable<s
return results;
}

var sourceCode = sourceLanguage != null ? GetLanguageCode(sourceLanguage) : null;
var targetCode = GetLanguageCode(targetLanguage);

var parameters = new List<KeyValuePair<string, string>>
{
new KeyValuePair<string, string>("target_lang", GetLanguageCode(targetLanguage)),
new KeyValuePair<string, string>("target_lang", targetCode),
};

foreach (var text in textsArray)
{
parameters.Add(new KeyValuePair<string, string>("text", text));
}

if (sourceLanguage != null)
if (sourceCode != null)
{
parameters.Add(new KeyValuePair<string, string>("source_lang", GetLanguageCode(sourceLanguage)));
parameters.Add(new KeyValuePair<string, string>("source_lang", sourceCode));
}

var url =
Expand All @@ -80,6 +116,62 @@ public async Task<IReadOnlyList<TranslationResult>> TranslateAsync(IEnumerable<s
UrlPaid;

using var httpClient = CreateClient();

var glossaryId = options.GlossaryById;

if (string.IsNullOrWhiteSpace(glossaryId) && sourceCode != null)
{
var langPairKey = $"{sourceCode}_{targetCode}";

if (glossaryResolved && glossaryCache.TryGetValue(langPairKey, out var cachedId))
{
glossaryId = cachedId;
}

if (!string.IsNullOrWhiteSpace(options.GlossaryByName) && string.IsNullOrWhiteSpace(glossaryId) && !glossaryResolved)
{
glossaryResolved = true;

var glossaryUrl = url.Replace("/translate", "/glossaries");
using var glossaryHttpRequest = new HttpRequestMessage(HttpMethod.Get, glossaryUrl);
using var glossaryHttpResponse = await httpClient.SendAsync(glossaryHttpRequest, ct);

try
{
glossaryHttpResponse.EnsureSuccessStatusCode();

var glossaryJsonString = await glossaryHttpResponse.Content.ReadAsStringAsync(ct);
var glossaryJsonResponse = JsonSerializer.Deserialize<GlossariesDto>(glossaryJsonString)!;

foreach (var glossary in glossaryJsonResponse.Glossaries)
{
if (options.GlossaryByName.Equals(glossary.Name, StringComparison.Ordinal) && glossary.Ready)
{
var key = $"{GetLanguageCode(glossary.SourceLang)}_{GetLanguageCode(glossary.TargetLang)}";
glossaryCache[key] = glossary.GlossaryId;
}
}

glossaryCache.TryGetValue(langPairKey, out glossaryId);
}
catch (Exception ex)
{
glossaryResolved = false;
AddError(TranslationResult.Failed(ex));
}
}
}

if (!string.IsNullOrWhiteSpace(glossaryId))
{
parameters.Add(new KeyValuePair<string, string>("glossary_id", glossaryId));
}

if (!string.IsNullOrWhiteSpace(options.TagHandling))
{
parameters.Add(new KeyValuePair<string, string>("tag_handling", options.TagHandling));
}

using var httpRequest = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = new FormUrlEncodedContent(parameters!),
Expand Down Expand Up @@ -108,8 +200,14 @@ public async Task<IReadOnlyList<TranslationResult>> TranslateAsync(IEnumerable<s
}
catch (HttpRequestException ex) when (ex.StatusCode is HttpStatusCode.BadRequest)
{
InvalidateGlossaryCacheIfResolved(glossaryId);
AddError(TranslationResult.LanguageNotSupported);
}
catch (HttpRequestException ex) when (ex.StatusCode is HttpStatusCode.NotFound)
{
InvalidateGlossaryCacheIfResolved(glossaryId);
AddError(TranslationResult.Failed(ex));
}
catch (HttpRequestException ex) when (ex.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden)
{
AddError(TranslationResult.Unauthorized);
Expand Down Expand Up @@ -139,6 +237,24 @@ private HttpClient CreateClient()
return httpClient;
}

private void InvalidateGlossaryCacheIfResolved(string? glossaryId)
{
if (string.IsNullOrWhiteSpace(glossaryId))
{
return;
}

foreach (var kvp in glossaryCache)
{
if (kvp.Value == glossaryId)
{
glossaryCache.TryRemove(kvp.Key, out _);
glossaryResolved = false;
break;
}
}
}

private static string GetSourceLanguage(string language, string? fallback)
{
var result = language?.ToLowerInvariant();
Expand Down
Loading