diff --git a/samples/cs/Directory.Packages.props b/samples/cs/Directory.Packages.props index 6fcae3448..3d5d4d244 100644 --- a/samples/cs/Directory.Packages.props +++ b/samples/cs/Directory.Packages.props @@ -6,9 +6,19 @@ + + + + + + + + + + - + \ No newline at end of file diff --git a/samples/cs/ModelManagement/ModelManagement.slnx b/samples/cs/ModelManagement/ModelManagement.slnx new file mode 100644 index 000000000..9ce720b6a --- /dev/null +++ b/samples/cs/ModelManagement/ModelManagement.slnx @@ -0,0 +1,3 @@ + + + diff --git a/samples/cs/ModelManagement/ModelManagement/CatalogManagement.cs b/samples/cs/ModelManagement/ModelManagement/CatalogManagement.cs new file mode 100644 index 000000000..da609e681 --- /dev/null +++ b/samples/cs/ModelManagement/ModelManagement/CatalogManagement.cs @@ -0,0 +1,130 @@ +using Microsoft.AI.Foundry.Local; +using Microsoft.Extensions.Logging; +using ModelManagement.Interfaces; +using System; +using System.Collections.Generic; +using System.Text; + +namespace ModelManagement +{ + public class CatalogManagement : ICatalogManagement + { + private readonly FoundryLocalManager _mgr; + private readonly ILogger _logger; + + public CatalogManagement(FoundryLocalManager mgr, ILogger logger) + { + _mgr = mgr; + _logger = logger; + } + + public async Task DownloadModelsAsync(List modelNames, CancellationToken ct = default) + { + ICatalog catalog = await _mgr.GetCatalogAsync(ct); + + List cachedModels = await catalog.GetCachedModelsAsync(ct); + List availableModels = await catalog.ListModelsAsync(ct); + + foreach (var modelName in modelNames) + { + var model = availableModels.Find(m => m.Alias == modelName); + if (model != null) + { + if (cachedModels.Find(m => m.Alias == modelName) != null) + { + _logger.LogInformation($"Model already cached: {model.Alias}"); + continue; + } + + _logger.LogInformation($"Downloading model: {model.Alias}"); + await model.DownloadAsync(); + } + else + { + _logger.LogWarning($"Model not found in catalog: {modelName}"); + } + } + } + + public async Task> LoadModelsAsync(List modelNames, CancellationToken ct = default) + { + List<(string, IModel)> result = new List<(string, IModel)>(); + + ICatalog catalog = await _mgr.GetCatalogAsync(ct); + + List cachedModels = await catalog.GetCachedModelsAsync(ct); + List loadedModels = await catalog.GetLoadedModelsAsync(ct); + List availableModels = await catalog.ListModelsAsync(ct); + + foreach (var modelName in modelNames) + { + var model = availableModels.Find(m => m.Alias == modelName); + if (model != null) + { + if (loadedModels.Find(m => m.Alias == modelName) == null) + { + _logger.LogInformation($"Model not loaded: {model.Alias}"); + + if (cachedModels.Find(m => m.Alias == modelName) == null) + { + _logger.LogInformation($"Downloading model: {model.Alias}"); + await model.DownloadAsync(); + } + else + { + _logger.LogInformation($"Model already cached: {model.Alias}"); + } + + _logger.LogInformation($"Loading model: {model.Alias}"); + await model.LoadAsync(); + } + else + { + _logger.LogInformation($"Model already loaded: {model.Alias}"); + } + + result.Add((modelName, model)); + } + else + { + _logger.LogWarning($"Model not found in catalog: {modelName}"); + } + + } + + return result; + } + + + public async Task UnloadModels(List models, CancellationToken ct = default) + { + foreach (var model in models) + { + _logger.LogInformation($"Unloading model: {model.Alias}"); + await model.UnloadAsync(ct); + } + } + + public async Task ClearCacheAsync(CancellationToken ct = default) + { + ICatalog catalog = await _mgr.GetCatalogAsync(ct); + + List cachedModels = await catalog.GetCachedModelsAsync(ct); + foreach(var model in cachedModels) + { + _logger.LogInformation($"Removing model from cache: {model.Alias}"); + await model.RemoveFromCacheAsync(ct); + } + } + + public async Task RemoveModelsFromCacheAsync(List models, CancellationToken ct = default) + { + foreach (var model in models) + { + _logger.LogInformation($"Removing model from cache: {model.Alias}"); + await model.RemoveFromCacheAsync(ct); + } + } + + } +} diff --git a/samples/cs/ModelManagement/ModelManagement/EPManagement.cs b/samples/cs/ModelManagement/ModelManagement/EPManagement.cs new file mode 100644 index 000000000..8c01b0332 --- /dev/null +++ b/samples/cs/ModelManagement/ModelManagement/EPManagement.cs @@ -0,0 +1,54 @@ +using Microsoft.AI.Foundry.Local; +using Microsoft.Extensions.Logging; +using ModelManagement.Interfaces; +using System; +using System.Collections.Generic; +using System.Text; + +namespace ModelManagement +{ + public class EPManagement : IEPManagement + { + private readonly FoundryLocalManager _mgr; + private readonly ILogger _logger; + + public EPManagement(FoundryLocalManager mgr, ILogger logger) + { + _mgr = mgr; + _logger = logger; + } + + public async Task DownloadAndRegisterEpsAsync(CancellationToken ct = default) + { + // Discover what EPs are available + var discoveredEps = _mgr.DiscoverEps(); + List epNames = new List(); + foreach (var ep in discoveredEps) + { + _logger.LogInformation($"{ep.Name} — registered: {ep.IsRegistered}"); + if (!ep.IsRegistered) + { + epNames.Add(ep.Name); + } + } + + // Download and register all EPs + string currentEp = ""; + var result = await _mgr.DownloadAndRegisterEpsAsync(epNames, (epName, percent) => + { + if (epName != currentEp) + { + if (currentEp != "") + { + _logger.LogInformation(""); + } + currentEp = epName; + } + _logger.LogInformation($"\r {epName} {percent,6:F1}%"); + }, ct); + _logger.LogInformation(""); + + _logger.LogInformation($"Success: {result.Success}, Status: {result.Status}"); + } + } +} diff --git a/samples/cs/ModelManagement/ModelManagement/Interfaces/ICatalogManagement.cs b/samples/cs/ModelManagement/ModelManagement/Interfaces/ICatalogManagement.cs new file mode 100644 index 000000000..54555602b --- /dev/null +++ b/samples/cs/ModelManagement/ModelManagement/Interfaces/ICatalogManagement.cs @@ -0,0 +1,10 @@ +using Microsoft.AI.Foundry.Local; + +namespace ModelManagement.Interfaces +{ + public interface ICatalogManagement + { + Task DownloadModelsAsync(List modelNames, CancellationToken ct = default); + Task> LoadModelsAsync(List modelNames, CancellationToken ct = default); + } +} \ No newline at end of file diff --git a/samples/cs/ModelManagement/ModelManagement/Interfaces/IEPManagement.cs b/samples/cs/ModelManagement/ModelManagement/Interfaces/IEPManagement.cs new file mode 100644 index 000000000..cd72343a7 --- /dev/null +++ b/samples/cs/ModelManagement/ModelManagement/Interfaces/IEPManagement.cs @@ -0,0 +1,7 @@ +namespace ModelManagement.Interfaces +{ + public interface IEPManagement + { + public Task DownloadAndRegisterEpsAsync(CancellationToken ct = default); + } +} \ No newline at end of file diff --git a/samples/cs/ModelManagement/ModelManagement/ModelManagement.csproj b/samples/cs/ModelManagement/ModelManagement/ModelManagement.csproj new file mode 100644 index 000000000..4453711d4 --- /dev/null +++ b/samples/cs/ModelManagement/ModelManagement/ModelManagement.csproj @@ -0,0 +1,24 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + + + + + + diff --git a/samples/cs/ModelManagement/ModelManagement/Program.cs b/samples/cs/ModelManagement/ModelManagement/Program.cs new file mode 100644 index 000000000..e9e55c77f --- /dev/null +++ b/samples/cs/ModelManagement/ModelManagement/Program.cs @@ -0,0 +1,81 @@ +using Microsoft.AI.Foundry.Local; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ModelManagement.Interfaces; +using Serilog; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace ModelManagement; + +internal class Program +{ + private static async Task Main(string[] args) + { + var _loggerFactory = Microsoft.Extensions.Logging.LoggerFactory.Create(builder => + { + builder.SetMinimumLevel(Microsoft.Extensions.Logging.LogLevel.Information); + }); + var _logger = _loggerFactory.CreateLogger("ModelManagement"); + + // Bootstrapping Serilog from appsettings.json explicitly so logging is active + var configuration = new ConfigurationBuilder() + .SetBasePath(AppContext.BaseDirectory) + .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .Build(); + + Log.Logger = new LoggerConfiguration() + .ReadFrom.Configuration(configuration) + .Enrich.FromLogContext() + .CreateLogger(); + + try + { + // Initialize the manager first (see Quick Start) + await FoundryLocalManager.CreateAsync( + new Configuration { AppName = "my-app" }, + _logger); + var mgr = FoundryLocalManager.Instance; + + Log.Information("Starting host"); + + var builder = Host.CreateDefaultBuilder(args) + .ConfigureAppConfiguration((hostingContext, config) => + { + // allow host to read configuration too + config.AddConfiguration(configuration); + }) + .UseSerilog() // use the static Log.Logger we created + .ConfigureServices((context, services) => + { + // Register the FoundryLocalManager singleton instance created earlier + services.AddSingleton(FoundryLocalManager.Instance); + + // Register EPManagement for IEPManagement so it can be injected + services.AddSingleton(); + services.AddScoped(); + + services.AddHostedService(); + }); + + using var host = builder.Build(); + + await host.RunAsync(); + } + catch (Exception ex) + { + Log.Fatal(ex, "Host terminated unexpectedly"); + throw; + } + finally + { + Log.CloseAndFlush(); + } + } +} \ No newline at end of file diff --git a/samples/cs/ModelManagement/ModelManagement/Worker.cs b/samples/cs/ModelManagement/ModelManagement/Worker.cs new file mode 100644 index 000000000..7e8a0264c --- /dev/null +++ b/samples/cs/ModelManagement/ModelManagement/Worker.cs @@ -0,0 +1,37 @@ +using Microsoft.AI.Foundry.Local; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ModelManagement.Interfaces; + +namespace ModelManagement; + +internal class Worker : BackgroundService +{ + private readonly ILogger _logger; + private readonly ICatalogManagement _catalogManagement; + private readonly IEPManagement _epManagement; + + public Worker(IEPManagement epManagement, ICatalogManagement catalogManagement, ILogger logger) + { + _epManagement = epManagement; + _catalogManagement = catalogManagement; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Worker running at: {time}", DateTimeOffset.Now); + + await _epManagement.DownloadAndRegisterEpsAsync(stoppingToken); + await _catalogManagement.DownloadModelsAsync(new List { "qwen3-embedding-0.6b", "phi-3.5-mini" }, stoppingToken); + List<(string, IModel)> result = await _catalogManagement.LoadModelsAsync(new List { "qwen3-embedding-0.6b", "phi-3.5-mini" }, stoppingToken); + + while (!stoppingToken.IsCancellationRequested) + { + _logger.LogInformation("Heartbeat at: {time}", DateTimeOffset.Now); + await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); + } + + _logger.LogInformation("Worker stopping"); + } +} diff --git a/samples/cs/ModelManagement/ModelManagement/appsettings.json b/samples/cs/ModelManagement/ModelManagement/appsettings.json new file mode 100644 index 000000000..0fb7406d3 --- /dev/null +++ b/samples/cs/ModelManagement/ModelManagement/appsettings.json @@ -0,0 +1,19 @@ +{ + "Serilog": { + "Using": [ "Serilog.Sinks.Console" ], + "MinimumLevel": { + "Default": "Debug", + "Override": { + "Microsoft": "Warning", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" } + ], + "Enrich": [ "FromLogContext", "WithMachineName", "WithThreadId" ], + "Properties": { + "Application": "ModelManagementSample" + } + } +} diff --git a/samples/cs/embeddings/Embeddings.slnx b/samples/cs/embeddings/Embeddings.slnx new file mode 100644 index 000000000..ae346257e --- /dev/null +++ b/samples/cs/embeddings/Embeddings.slnx @@ -0,0 +1,3 @@ + + + diff --git a/samples/cs/rag/rag.slnx b/samples/cs/rag/rag.slnx new file mode 100644 index 000000000..807c517a3 --- /dev/null +++ b/samples/cs/rag/rag.slnx @@ -0,0 +1,3 @@ + + + diff --git a/samples/cs/rag/rag/Program.cs b/samples/cs/rag/rag/Program.cs new file mode 100644 index 000000000..292c49f13 --- /dev/null +++ b/samples/cs/rag/rag/Program.cs @@ -0,0 +1,206 @@ +using Betalgo.Ranul.OpenAI.ObjectModels.RequestModels; +using Betalgo.Ranul.OpenAI.ObjectModels.ResponseModels; +using Microsoft.AI.Foundry.Local; +using Microsoft.ML.OnnxRuntimeGenAI; +using static Betalgo.Ranul.OpenAI.ObjectModels.StaticValues.AssistantsStatics.MessageStatics; +using static System.Runtime.InteropServices.JavaScript.JSType; + +internal class Program +{ + private static async Task Main(string[] args) + { + CancellationToken ct = new CancellationToken(); + + var config = new Configuration + { + AppName = "foundry_local_rag", + LogLevel = LogLevel.Information + }; + + // Initialize the singleton instance. + await FoundryLocalManager.CreateAsync(config, Utils.GetAppLogger()); + var mgr = FoundryLocalManager.Instance; + + // Download and register all execution providers. + var currentEp = ""; + await mgr.DownloadAndRegisterEpsAsync((epName, percent) => + { + if (epName != currentEp) + { + if (currentEp != "") Console.WriteLine(); + currentEp = epName; + } + Console.Write($"\r {epName.PadRight(30)} {percent,6:F1}%"); + }); + if (currentEp != "") Console.WriteLine(); + + + // Get the model catalog + var catalog = await mgr.GetCatalogAsync(); + + // Get an embedding model + var embeddingModel = await catalog.GetModelAsync("qwen3-embedding-0.6b") ?? throw new Exception("Embedding model not found"); + + // Download the model (the method skips download if already cached) + await embeddingModel.DownloadAsync(progress => + { + Console.Write($"\rDownloading model: {progress:F2}%"); + if (progress >= 100f) + { + Console.WriteLine(); + } + }); + + // Load the model + Console.Write($"Loading embedding model {embeddingModel.Id}..."); + await embeddingModel.LoadAsync(); + + + // Get an embedding client + var embeddingClient = await embeddingModel.GetEmbeddingClientAsync(); + + // Generate embeddings for multiple inputs + + // Knowledge base — each string represents a document + var documents = new List + { + "Foundry Local runs AI models directly on your device without cloud connectivity.", + "The Foundry Local SDK supports Python, C#, JavaScript, and Rust.", + "Embedding models convert text into numerical vectors for similarity search.", + "Foundry Local uses ONNX Runtime for efficient model inference on CPUs and GPUs.", + "The model catalog provides pre-optimized models that you can download and run locally.", + "Retrieval-augmented generation grounds model responses in your own data.", + "Vector similarity search finds documents that are semantically close to a query.", + "Chat completions generate natural language responses from a prompt and context.", + }; + + Console.WriteLine("\n--- Batch Embeddings ---"); + var response = await embeddingClient.GenerateEmbeddingsAsync(documents); + + Console.WriteLine($"Number of embeddings: {response.Data.Count}"); + for (var i = 0; i < response.Data.Count; i++) + { + Console.WriteLine($" [{i}] Dimensions: {response.Data[i].Embedding.Count}"); + } + + Console.WriteLine($"Indexed {response.Data.Count} documents."); + + + // Get a model using an alias. + var chatModel = await catalog.GetModelAsync("qwen2.5-0.5b") ?? throw new Exception("Model not found"); + + // Download the model (the method skips download if already cached) + await chatModel.DownloadAsync(progress => + { + Console.Write($"\rDownloading model: {progress:F2}%"); + if (progress >= 100f) + { + Console.WriteLine(); + } + }); + + // Load the model + Console.Write($"Loading model {chatModel.Id}..."); + await chatModel.LoadAsync(); + + // + // Get a chat client + var chatClient = await chatModel.GetChatClientAsync(); + + Console.WriteLine("\nModels loaded. Ready for questions."); + Console.WriteLine("\nThe knowledge base contains information about:"); + Console.WriteLine(" - Foundry Local features and architecture"); + Console.WriteLine(" - Supported programming languages"); + Console.WriteLine(" - Embedding models and vector search"); + Console.WriteLine(" - ONNX Runtime inference"); + Console.WriteLine(" - The model catalog"); + Console.WriteLine(" - RAG and chat completions"); + Console.WriteLine("\nExample questions:"); + Console.WriteLine(" \"What programming languages does the SDK support?\""); + Console.WriteLine(" \"How does Foundry Local run models?\""); + Console.WriteLine(" \"What is retrieval-augmented generation?\""); + Console.WriteLine("\nType \"quit\" to exit.\n"); + + // Interactive query loop + while (true) + { + Console.WriteLine("Question:"); + var query = Console.ReadLine()?.Trim(); + if (string.IsNullOrEmpty(query) || query.ToLower() == "quit") + { + break; + } + + // Embed the query + var queryResponse = await embeddingClient.GenerateEmbeddingAsync(query); + var queryEmbedding = queryResponse.Data[0].Embedding; + + // Retrieve the most relevant documents + var results = FindRelevant(queryEmbedding, response.Data, topK: 2); + string context = string.Join("\n", results.Select(r => $"- {documents[r.Item1]}")); + + // Build the prompt with retrieved context + string content = + $$""" + Answer the user's question using only the provided context. + If the context doesn't contain enough information, say so. + + Context: + {{context}} + """; + + // Create chat messages + List messages = new() + { + new ChatMessage { Role = "system", Content = content }, + new ChatMessage { Role = "user", Content = query } + }; + + // Get a streaming chat completion response + Console.WriteLine("Answer:"); + var streamingResponse = chatClient.CompleteChatStreamingAsync(messages, ct); + await foreach (var chunk in streamingResponse) + { + Console.Write(chunk.Choices[0].Message.Content); + Console.Out.Flush(); + } + Console.WriteLine(); + } + + // Tidy up - unload the models + await chatModel.UnloadAsync(); + await embeddingModel.UnloadAsync(); + } + + private static double CosineSimilarity(List a, List b) + { + double dot = 0; + double norm_a = 0; + double norm_b = 0; + + for (int i = 0; i < a.Count; i++) + { + dot += a[i] * b[i]; + norm_a += a[i] * a[i]; + norm_b += b[i] * b[i]; + } + + norm_a = Math.Sqrt(norm_a); + norm_b = Math.Sqrt(norm_b); + + return norm_a * norm_b != 0 ? dot / (norm_a * norm_b) : 0.0; + } + + + private static (int, double)[] FindRelevant(List queryEmbedding, List docEmbeddings, int topK = 2) + { + var scores = new List<(int, double)>(); + for (int i = 0; i < docEmbeddings.Count; i++) + { + double score = CosineSimilarity(queryEmbedding, docEmbeddings[i].Embedding); + scores.Add((i, score)); + } + scores.Sort((x, y) => y.Item2.CompareTo(x.Item2)); + return scores.Take(topK).ToArray(); + } +} \ No newline at end of file diff --git a/samples/cs/rag/rag/rag.csproj b/samples/cs/rag/rag/rag.csproj new file mode 100644 index 000000000..02f715278 --- /dev/null +++ b/samples/cs/rag/rag/rag.csproj @@ -0,0 +1,18 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + diff --git a/samples/cs/tutorial-document-summarizer/Program.cs b/samples/cs/tutorial-document-summarizer/Program.cs index 333d5c964..6c45e77aa 100644 --- a/samples/cs/tutorial-document-summarizer/Program.cs +++ b/samples/cs/tutorial-document-summarizer/Program.cs @@ -56,22 +56,30 @@ await model.DownloadAsync(progress => // // -var systemPrompt = +var systemPrompt1 = "Summarize the following document into concise bullet points. " + "Focus on the key points and main ideas."; +var systemPrompt2 = + "Summarize the following document in a single, concise paragraph. " + + "Capture the main argument and supporting points."; + +var systemPrompt3 = + "Extract the three most important takeaways from the following document. " + + "Number each takeaway and keep each to one or two sentences."; + // -var target = args.Length > 0 ? args[0] : "document.txt"; +var target = args.Length > 0 ? args[0] : Path.Combine(AppContext.BaseDirectory, "document.txt"); // if (Directory.Exists(target)) { - await SummarizeDirectoryAsync(chatClient, target, systemPrompt, ct); + await SummarizeDirectoryAsync(chatClient, target, systemPrompt1, ct); } else { Console.WriteLine($"--- {Path.GetFileName(target)} ---"); - await SummarizeFileAsync(chatClient, target, systemPrompt, ct); + await SummarizeFileAsync(chatClient, target, systemPrompt1, ct); } // diff --git a/samples/cs/tutorial-document-summarizer/TutorialDocumentSummarizer.csproj b/samples/cs/tutorial-document-summarizer/TutorialDocumentSummarizer.csproj index 972ec6210..d8ae135f9 100644 --- a/samples/cs/tutorial-document-summarizer/TutorialDocumentSummarizer.csproj +++ b/samples/cs/tutorial-document-summarizer/TutorialDocumentSummarizer.csproj @@ -23,5 +23,10 @@ + + + PreserveNewest + + diff --git a/samples/cs/tutorial-document-summarizer/document.txt b/samples/cs/tutorial-document-summarizer/document.txt new file mode 100644 index 000000000..905faf409 --- /dev/null +++ b/samples/cs/tutorial-document-summarizer/document.txt @@ -0,0 +1,18 @@ +Automated testing is a practice in software development where tests are written and executed +by specialized tools rather than performed manually. There are several categories of automated +tests, including unit tests, integration tests, and end-to-end tests. Unit tests verify that +individual functions or methods behave correctly in isolation. Integration tests check that +multiple components work together as expected. End-to-end tests simulate real user workflows +across the entire application. + +Adopting automated testing brings measurable benefits to a development team. It catches +regressions early, before they reach production. It reduces the time spent on repetitive +manual verification after each code change. It serves as living documentation of expected +behavior, which helps new team members understand the codebase. Continuous integration +pipelines rely on automated tests to gate deployments and maintain release quality. + +Effective test suites follow a few guiding principles. Tests should be deterministic, meaning +they produce the same result every time they run. Tests should be independent, so that one +failing test does not cascade into false failures elsewhere. Tests should run fast, because +slow tests discourage developers from running them frequently. Finally, tests should be +maintained alongside production code so they stay accurate as the application evolves. \ No newline at end of file diff --git a/samples/cs/verify-winml/Program.cs b/samples/cs/verify-winml/Program.cs index 27a141296..cc32c387c 100644 --- a/samples/cs/verify-winml/Program.cs +++ b/samples/cs/verify-winml/Program.cs @@ -241,7 +241,7 @@ await candidate.DownloadAsync(progress => { var chatClient = await chosen.GetChatClientAsync(); chatClient.Settings.Temperature = 0; - chatClient.Settings.MaxTokens = 16; + chatClient.Settings.MaxTokens = 500; var messages = new List { new() { Role = "system", Content = "You are a helpful assistant." }, diff --git a/samples/cs/verify-winml/VerifyWinML.slnx b/samples/cs/verify-winml/VerifyWinML.slnx new file mode 100644 index 000000000..0fb9b4212 --- /dev/null +++ b/samples/cs/verify-winml/VerifyWinML.slnx @@ -0,0 +1,3 @@ + + +